diff --git a/CLAUDE.md b/CLAUDE.md index 295ee63..38eee8c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -207,6 +207,9 @@ conductor.createTask( pollInterval: 1000, // How often to poll for new executions in ms (default: 1000) partition: false, // Enable partitioning (default: false) window: ["09:00", "17:00"], // Time window for execution [start, end] + fifo: true, // Durable strict FIFO (exclusive with concurrency) + concurrency: 10, // Soft task-level limit + groupConcurrency: 2, // Soft task/group-level limit }, handler, ); diff --git a/docs/content/api/conductor.md b/docs/content/api/conductor.md index a8fbd7e..26ee861 100644 --- a/docs/content/api/conductor.md +++ b/docs/content/api/conductor.md @@ -67,7 +67,9 @@ const task = conductor.createTask( name: string; // Task name (required) queue?: string; // Queue name (default: "default") maxAttempts?: number; // Max retry attempts (default: 3) - concurrency?: number; // Max concurrent executions (default: unlimited) + fifo?: boolean; // Strict durable FIFO lane + concurrency?: number; // Soft max concurrent executions + groupConcurrency?: number; // Soft max per invocation group window?: [string, string]; // Time window (e.g., ["09:00", "17:00"]) removeOnComplete?: { days: number } | false; // Retention policy removeOnFail?: { days: number } | false; // Retention policy diff --git a/docs/content/task-execution/concurrency.md b/docs/content/task-execution/concurrency.md index eb93dd5..173537b 100644 --- a/docs/content/task-execution/concurrency.md +++ b/docs/content/task-execution/concurrency.md @@ -10,7 +10,8 @@ Control the maximum number of concurrent executions for a specific task: const processVideo = conductor.createTask( { name: "process-video", - concurrency: 3, // Max 3 videos processing at once + concurrency: 3, // Soft max of 3 videos at once + groupConcurrency: 1, // Soft max of 1 per invocation group }, { invocable: true }, async (event, ctx) => { @@ -24,12 +25,7 @@ When the limit is reached, additional executions wait in the queue until a slot ## How It Works -Postgres Conductor uses a slot-based system to enforce concurrency limits: - -1. **Slot allocation**: When a task has `concurrency: N`, Postgres creates N slots in the `_private_concurrency_slots` table -2. **Claiming slots**: Workers claim available slots using `FOR UPDATE SKIP LOCKED` -3. **Execution**: Task runs while holding the slot -4. **Release**: Slot is released when execution completes or fails +Postgres Conductor evaluates active executions when claiming work. Task and group limits are coordinated with `FOR UPDATE SKIP LOCKED`; limits are intentionally soft across concurrent workers. Grouped candidates whose group is full do not consume task-level capacity, so another available group can be claimed in the same batch. This happens entirely in Postgres - no external coordination needed. @@ -50,8 +46,14 @@ This happens entirely in Postgres - no external coordination needed. - Set on worker/queue with `config: { concurrency }` - Independent per worker instance +Child invocations inherit the group supplied to `ctx.invoke`. Dynamic cron schedules accept `group` alongside `cron`, and each next cron execution preserves the group. + ## What's Next? - [Worker Configuration](../api/worker-config.md) - Configure worker-level concurrency - [Priority](priority.md) - Control execution order when waiting for slots - [Batching](batching.md) - Process multiple executions together + +## Group concurrency + +`group` may be supplied when invoking a task. `groupConcurrency` limits active executions within each `(queue, task, group)` scope; ungrouped invocations bypass that limit. Task and group limits compose and are intentionally soft across concurrent workers. diff --git a/docs/content/task-execution/fifo.md b/docs/content/task-execution/fifo.md new file mode 100644 index 0000000..279e3e2 --- /dev/null +++ b/docs/content/task-execution/fifo.md @@ -0,0 +1,31 @@ +# FIFO execution + +Set `fifo: true` on a task to serialize its executions in strict enqueue order: + +```ts +const updateAccount = conductor.createTask( + { name: "update-account", fifo: true }, + { invocable: true }, + async (event, ctx) => { /* ... */ }, +); +``` + +FIFO is scoped to `(queue, task)`. It uses a durable database lane owner, so only +one execution can be active at a time across all workers. The owner remains with +the execution during retries, sleeps, child waits, and `waitForEvent` waits, and +is released on completion, cancellation, or permanent failure. A worker crash +therefore resumes the owner before admitting its successor. + +Priorities are ignored for FIFO tasks. A future, never-started scheduled or +delayed execution does not block currently runnable work; once selected, it +retains the lane even when it becomes delayed. Different FIFO tasks (and queue +identities) run independently. + +FIFO cannot be combined with `concurrency` or `groupConcurrency`: + +```ts +{ name: "invalid", fifo: true, concurrency: 2 } +``` + +Use soft `concurrency` or `groupConcurrency` instead when strict ordering is +not required. diff --git a/docs/zensical.toml b/docs/zensical.toml index 56b1789..bb1ea8b 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -73,6 +73,7 @@ nav = [ { "Cancellation" = "task-execution/cancellation.md" }, { "Priority" = "task-execution/priority.md" }, { "Concurrency" = "task-execution/concurrency.md" }, + { "FIFO" = "task-execution/fifo.md" }, { "Deduplication" = "task-execution/deduplication.md" }, { "Rate Limiting" = "task-execution/rate-limiting.md" }, { "Batching" = "task-execution/batching.md" }, diff --git a/justfile b/justfile index f3c2532..e781d1d 100644 --- a/justfile +++ b/justfile @@ -8,7 +8,7 @@ alias r := ready alias t := test build-migrations: - sh ./scripts/build-migrations.sh + bash ./scripts/build-migrations.sh lint: bun run oxlint --type-aware --deny-warnings diff --git a/migrations/0000000001_setup.sql b/migrations/0000000001_setup.sql index abe054c..14122be 100644 --- a/migrations/0000000001_setup.sql +++ b/migrations/0000000001_setup.sql @@ -49,6 +49,8 @@ begin end; $$; +create sequence pgconductor._private_enqueue_position_seq as bigint; + create table pgconductor._private_orchestrators ( id uuid default pgconductor._private_portable_uuidv7() primary key, last_heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), @@ -95,6 +97,9 @@ create table pgconductor._private_executions ( run_at timestamptz default pgconductor._private_current_time() not null, locked_at timestamptz, locked_by uuid, + claim_token uuid, + "group" text, + enqueue_position bigint not null default nextval('pgconductor._private_enqueue_position_seq'), is_available boolean generated always as (locked_at is null and failed_at is null and completed_at is null) stored not null, attempts integer default 0 not null, last_error text, @@ -116,7 +121,7 @@ create unique index on pgconductor._private_executions (task_key, singleton_on, where singleton_on is not null and completed_at is null and failed_at is null and cancelled = false; create table pgconductor._private_tasks ( - key text primary key, + key text not null, -- queue that this task belongs to (used for queue-based worker assignment) queue text default 'default' not null, @@ -142,22 +147,24 @@ create table pgconductor._private_tasks ( ) ), - -- concurrency control: maximum number of concurrent executions across all workers + -- FIFO is a strict, persistent single-lane policy. It cannot be combined + -- with soft concurrency controls. + fifo boolean default false not null, + -- concurrency controls are intentionally soft and coordinated at claim time -- NULL means no limit (unlimited concurrency) - concurrency_limit integer -); + concurrency_limit integer, + group_concurrency_limit integer, + constraint positive_concurrency_limits check ( + (concurrency_limit is null or concurrency_limit > 0) and + (group_concurrency_limit is null or group_concurrency_limit > 0) + ), + constraint fifo_excludes_concurrency check ( + not fifo or (concurrency_limit is null and group_concurrency_limit is null) + ), -create table pgconductor._private_concurrency_slots ( - task_key text not null, - slot_group_number integer not null, - capacity integer not null, - used integer default 0 not null, - primary key (task_key, slot_group_number) + primary key (queue, key) ); -create index idx_slots_claim - on pgconductor._private_concurrency_slots (task_key, capacity, used); - create table pgconductor._private_steps ( id uuid default pgconductor._private_portable_uuidv7() primary key, key text not null, @@ -171,6 +178,19 @@ create table pgconductor._private_steps ( create index idx_steps_execution_id on pgconductor._private_steps (execution_id); +-- A FIFO task owns one durable lane. The owner survives every non-terminal +-- execution state so another worker cannot pass a sleeping or waiting owner. +create table pgconductor._private_fifo_owners ( + queue text not null, + task_key text not null, + execution_id uuid not null, + primary key (queue, task_key), + constraint fk_fifo_owner_execution foreign key (execution_id, queue) + references pgconductor._private_executions(id, queue) on delete cascade +); + +create index idx_fifo_owners_execution on pgconductor._private_fifo_owners (execution_id, queue); + -- Trigger function to manage executions partitions per queue -- Automatically creates partition when queue is inserted create or replace function pgconductor._private_manage_queue_partition() @@ -196,7 +216,7 @@ begin -- main index for fetching available executions execute format( - 'create index if not exists %I on pgconductor.%I (priority, run_at) include (id, task_key) where is_available = true', + 'create index if not exists %I on pgconductor.%I (priority, run_at, enqueue_position) include (id, task_key) where is_available = true', 'idx_' || v_partition_name || '_get_executions', v_partition_name ); @@ -250,6 +270,19 @@ begin v_partition_name ); + -- indexes used to count active executions for soft concurrency limits + execute format( + 'create index if not exists %I on pgconductor.%I (task_key) where locked_at is not null and failed_at is null and completed_at is null', + 'idx_' || v_partition_name || '_active_task', + v_partition_name + ); + + execute format( + 'create index if not exists %I on pgconductor.%I (task_key, "group") where "group" is not null and locked_at is not null and failed_at is null and completed_at is null', + 'idx_' || v_partition_name || '_active_task_group', + v_partition_name + ); + RETURN NEW; elsif tg_op = 'UPDATE' then @@ -312,7 +345,8 @@ create type pgconductor.execution_spec as ( dedupe_seconds integer, dedupe_next_slot boolean, cron_expression text, - priority integer + priority integer, + "group" text ); create type pgconductor.task_spec as ( @@ -323,7 +357,9 @@ create type pgconductor.task_spec as ( remove_on_fail_days integer, window_start timetz, window_end timetz, - concurrency_limit integer + concurrency_limit integer, + fifo boolean, + group_concurrency_limit integer ); create type pgconductor._private_event_operation as enum ( @@ -362,7 +398,7 @@ begin on conflict (name) do nothing; -- step 2: register/update tasks - insert into pgconductor._private_tasks (key, queue, max_attempts, remove_on_complete_days, remove_on_fail_days, window_start, window_end, concurrency_limit) + insert into pgconductor._private_tasks (key, queue, max_attempts, remove_on_complete_days, remove_on_fail_days, window_start, window_end, fifo, concurrency_limit, group_concurrency_limit) select spec.key, coalesce(spec.queue, 'default'), @@ -371,9 +407,11 @@ begin spec.remove_on_fail_days, spec.window_start, spec.window_end, - spec.concurrency_limit + coalesce(spec.fifo, false), + spec.concurrency_limit, + spec.group_concurrency_limit from unnest(p_task_specs) as spec - on conflict (key) + on conflict (queue, key) do update set queue = coalesce(excluded.queue, pgconductor._private_tasks.queue), max_attempts = coalesce(excluded.max_attempts, pgconductor._private_tasks.max_attempts), @@ -381,51 +419,34 @@ begin remove_on_fail_days = excluded.remove_on_fail_days, window_start = excluded.window_start, window_end = excluded.window_end, - concurrency_limit = excluded.concurrency_limit; - - -- step 2a: manage concurrency slots - -- create one row per slot (capacity=1 each) - insert into pgconductor._private_concurrency_slots (task_key, slot_group_number, capacity, used) - select - spec.key, - slot_num, - 1 as capacity, - 0 as used - from unnest(p_task_specs) as spec - cross join lateral generate_series(1, spec.concurrency_limit) as slot_num - where spec.concurrency_limit is not null - on conflict (task_key, slot_group_number) - do update set - capacity = excluded.capacity, - used = least(pgconductor._private_concurrency_slots.used, excluded.capacity); - - -- clean up orphaned slots (tasks removed or concurrency_limit set to null) - delete from pgconductor._private_concurrency_slots - where task_key not in ( - select key from pgconductor._private_tasks - where concurrency_limit is not null - ); - - -- clean up excess slots when concurrency decreased - delete from pgconductor._private_concurrency_slots cs - where cs.slot_group_number > ( - select concurrency_limit - from pgconductor._private_tasks t - where t.key = cs.task_key - ); - - -- step 3: insert scheduled cron executions (on conflict do nothing) - insert into pgconductor._private_executions (task_key, queue, payload, run_at, dedupe_key, cron_expression) + fifo = excluded.fifo, + concurrency_limit = excluded.concurrency_limit, + group_concurrency_limit = excluded.group_concurrency_limit; + + -- A task leaving FIFO must release its lane immediately. Re-enabling FIFO + -- intentionally leaves pending work ownerless; the claim query elects its + -- current head on the next poll. + delete from pgconductor._private_fifo_owners o + using pgconductor._private_tasks t + where t.queue = o.queue and t.key = o.task_key and t.fifo = false; + + -- step 3: insert scheduled cron executions + insert into pgconductor._private_executions (task_key, queue, payload, run_at, dedupe_key, cron_expression, "group") select spec.task_key, coalesce(spec.queue, 'default'), coalesce(spec.payload, '{}'::jsonb), coalesce(spec.run_at, pgconductor._private_current_time()), spec.dedupe_key, - spec.cron_expression + spec.cron_expression, + spec."group" from unnest(p_cron_schedules) as spec where spec.dedupe_key is not null - on conflict (task_key, dedupe_key, queue) do nothing; + on conflict (task_key, dedupe_key, queue) do update set + payload = excluded.payload, + run_at = excluded.run_at, + cron_expression = excluded.cron_expression, + "group" = excluded."group"; -- step 4: clean up stale schedules for this queue -- delete future executions for schedules that no longer exist @@ -525,20 +546,38 @@ declare begin v_now := pgconductor._private_current_time(); + insert into pgconductor._private_queues (name) + select distinct coalesce(spec.queue, 'default') + from unnest(specs) as spec + on conflict (name) do nothing; + -- clear locked dedupe keys before batch insert - update pgconductor._private_executions as e + with superseded as ( + select e.id, e.queue, e.task_key + from pgconductor._private_executions as e + cross join unnest(specs) as spec + where e.dedupe_key = spec.dedupe_key + and e.task_key = spec.task_key + and e.queue = coalesce(spec.queue, 'default') + and e.locked_at is not null + and spec.dedupe_key is not null + for update of e + ) + update pgconductor._private_executions e set dedupe_key = null, locked_by = null, locked_at = null, + claim_token = null, failed_at = v_now, last_error = 'superseded by reinvoke' - from unnest(specs) as spec - where e.dedupe_key = spec.dedupe_key - and e.task_key = spec.task_key - and e.queue = coalesce(spec.queue, 'default') - and e.locked_at is not null - and spec.dedupe_key is not null; + from superseded s + where e.id = s.id; + + delete from pgconductor._private_fifo_owners o + using pgconductor._private_executions e + where o.execution_id = e.id and o.queue = e.queue + and e.failed_at = v_now and e.last_error = 'superseded by reinvoke'; -- batch insert all executions -- note: duplicate dedupe_keys within same batch will cause error @@ -553,7 +592,8 @@ begin dedupe_key, singleton_on, cron_expression, - priority + priority, + "group" ) select pgconductor._private_portable_uuidv7(), @@ -572,14 +612,16 @@ begin else null end, spec.cron_expression, - coalesce(spec.priority, 0) + coalesce(spec.priority, 0), + spec."group" from unnest(specs) as spec on conflict (task_key, dedupe_key, queue) do update set payload = excluded.payload, run_at = excluded.run_at, priority = excluded.priority, cron_expression = excluded.cron_expression, - singleton_on = excluded.singleton_on + singleton_on = excluded.singleton_on, + "group" = excluded."group" returning pgconductor._private_executions.id; end; $function$ @@ -594,7 +636,8 @@ create or replace function pgconductor.invoke( p_dedupe_seconds integer default null, p_dedupe_next_slot boolean default false, p_cron_expression text default null, - p_priority integer default null + p_priority integer default null, + p_group text default null ) returns table(id uuid) language plpgsql @@ -611,19 +654,36 @@ begin v_now := pgconductor._private_current_time(); v_run_at := coalesce(p_run_at, v_now); + insert into pgconductor._private_queues (name) + values (p_queue) + on conflict (name) do nothing; + -- clear locked dedupe key before insert (supersede pattern) if p_dedupe_key is not null then - update pgconductor._private_executions + with superseded as ( + select e.id, e.queue, e.task_key + from pgconductor._private_executions e + where e.dedupe_key = p_dedupe_key + and e.task_key = p_task_key + and e.queue = p_queue + and e.locked_at is not null + for update of e + ) + update pgconductor._private_executions e set dedupe_key = null, locked_by = null, locked_at = null, + claim_token = null, failed_at = v_now, last_error = 'superseded by reinvoke' - where dedupe_key = p_dedupe_key - and task_key = p_task_key - and queue = p_queue - and locked_at is not null; + from superseded s + where e.id = s.id; + + delete from pgconductor._private_fifo_owners o + using pgconductor._private_executions e + where o.execution_id = e.id and o.queue = e.queue + and e.failed_at = v_now and e.last_error = 'superseded by reinvoke'; end if; -- singleton throttle/debounce logic @@ -647,7 +707,8 @@ begin dedupe_key, singleton_on, cron_expression, - priority + priority, + "group" ) values ( pgconductor._private_portable_uuidv7(), p_task_key, @@ -657,7 +718,8 @@ begin p_dedupe_key, v_singleton_on, p_cron_expression, - coalesce(p_priority, 0) + coalesce(p_priority, 0), + p_group ) on conflict (task_key, singleton_on, coalesce(dedupe_key, ''), queue) where singleton_on is not null and completed_at is null and failed_at is null and cancelled = false @@ -678,7 +740,8 @@ begin dedupe_key, singleton_on, cron_expression, - priority + priority, + "group" ) values ( pgconductor._private_portable_uuidv7(), p_task_key, @@ -688,35 +751,23 @@ begin p_dedupe_key, v_next_singleton_on, p_cron_expression, - coalesce(p_priority, 0) + coalesce(p_priority, 0), + p_group ) on conflict (task_key, singleton_on, coalesce(dedupe_key, ''), queue) where singleton_on is not null and completed_at is null and failed_at is null and cancelled = false do update set payload = excluded.payload, run_at = excluded.run_at, - priority = excluded.priority + priority = excluded.priority, + cron_expression = excluded.cron_expression, + "group" = excluded."group" returning _private_executions.id; return; end if; end if; -- regular invoke (no singleton) - if p_dedupe_key is not null then - -- clear keys that are currently locked so a subsequent insert can succeed. - update pgconductor._private_executions as e - set - dedupe_key = null, - locked_by = null, - locked_at = null, - failed_at = pgconductor._private_current_time(), - last_error = 'superseded by reinvoke' - where e.dedupe_key = p_dedupe_key - and e.task_key = p_task_key - and e.queue = p_queue - and e.locked_at is not null; - end if; - return query insert into pgconductor._private_executions as e ( id, task_key, @@ -725,7 +776,8 @@ begin run_at, dedupe_key, cron_expression, - priority + priority, + "group" ) values ( pgconductor._private_portable_uuidv7(), p_task_key, @@ -734,13 +786,20 @@ begin v_run_at, p_dedupe_key, p_cron_expression, - coalesce(p_priority, 0) + coalesce(p_priority, 0), + p_group ) on conflict (task_key, dedupe_key, queue) do update set payload = excluded.payload, run_at = excluded.run_at, priority = excluded.priority, - cron_expression = excluded.cron_expression + cron_expression = excluded.cron_expression, + "group" = excluded."group", + failed_at = case when e.last_error = 'superseded by reinvoke' then null else e.failed_at end, + locked_by = case when e.last_error = 'superseded by reinvoke' then null else e.locked_by end, + locked_at = case when e.last_error = 'superseded by reinvoke' then null else e.locked_at end, + claim_token = case when e.last_error = 'superseded by reinvoke' then null else e.claim_token end, + cancelled = case when e.last_error = 'superseded by reinvoke' then false else e.cancelled end returning e.id; end; $function$ @@ -758,37 +817,108 @@ set search_path to '' as $function$ declare v_orchestrator_id uuid; + v_claim_token uuid; v_queue text; + v_child_id uuid; + v_child_orchestrator_id uuid; + v_child_queue text; v_completed boolean; v_failed boolean; v_rows_affected integer; begin select locked_by, + claim_token, queue, + waiting_on_execution_id, completed_at is not null, failed_at is not null - into v_orchestrator_id, v_queue, v_completed, v_failed + into v_orchestrator_id, v_claim_token, v_queue, v_child_id, v_completed, v_failed from pgconductor._private_executions - where id = p_execution_id; + where id = p_execution_id + for update; if not found or v_completed or v_failed then return false; end if; if v_orchestrator_id is null then - -- pending: fail immediately + -- pending: fail immediately. If this is a waiting parent, resolve its + -- child relationship in the same transaction so the child cannot become + -- orphaned or leave the workflow stranded. + if v_child_id is not null then + select locked_by, queue + into v_child_orchestrator_id, v_child_queue + from pgconductor._private_executions + where id = v_child_id + for update; + + if found and v_child_orchestrator_id is null then + update pgconductor._private_executions + set + failed_at = pgconductor._private_current_time(), + last_error = 'Cancelled: parent execution was cancelled', + locked_by = null, + locked_at = null, + claim_token = null, + waiting_on_execution_id = null, + waiting_step_key = null + where id = v_child_id + and completed_at is null + and failed_at is null; + + get diagnostics v_rows_affected = row_count; + if v_rows_affected > 0 then + delete from pgconductor._private_fifo_owners + where execution_id = v_child_id and queue = v_child_queue; + end if; + elsif found then + update pgconductor._private_executions + set cancelled = true, last_error = p_reason + where id = v_child_id + and completed_at is null + and failed_at is null + and cancelled = false; + + get diagnostics v_rows_affected = row_count; + if v_rows_affected > 0 then + -- A running child keeps its FIFO owner until its cancellation is + -- fenced and settled by return/recovery. + insert into pgconductor._private_orchestrator_signals + (orchestrator_id, type, execution_id, payload) + values ( + v_child_orchestrator_id, + 'cancel_execution', + v_child_id, + jsonb_build_object('queue', v_child_queue, 'reason', p_reason) + ) + on conflict (orchestrator_id, execution_id) + where type = 'cancel_execution' and execution_id is not null + do nothing; + end if; + end if; + end if; + update pgconductor._private_executions set failed_at = pgconductor._private_current_time(), last_error = p_reason, locked_by = null, - locked_at = null + locked_at = null, + claim_token = null, + waiting_on_execution_id = null, + waiting_step_key = null where id = p_execution_id and completed_at is null - and failed_at is null; + and failed_at is null + and locked_by is null + and locked_at is null; get diagnostics v_rows_affected = row_count; + if v_rows_affected > 0 then + delete from pgconductor._private_fifo_owners + where execution_id = p_execution_id and queue = v_queue; + end if; return v_rows_affected > 0; else -- running: signal orchestrator + set cancelled flag @@ -797,6 +927,9 @@ begin cancelled = true, last_error = p_reason where id = p_execution_id + and queue = v_queue + and locked_by = v_orchestrator_id + and claim_token = v_claim_token and completed_at is null and cancelled = false; diff --git a/migrations/0000000002_events.sql b/migrations/0000000002_events.sql index fc29f37..2d3b223 100644 --- a/migrations/0000000002_events.sql +++ b/migrations/0000000002_events.sql @@ -92,7 +92,7 @@ begin sub.id ), e'\n') from pgconductor._private_event_subscriptions as sub - join pgconductor._private_tasks as t on t.key = sub.task_key + join pgconductor._private_tasks as t on t.key = sub.task_key and t.queue = sub.queue where sub.event_key is not null ); @@ -257,7 +257,7 @@ begin sub.id ), e'\n') from pgconductor._private_event_subscriptions as sub - join pgconductor._private_tasks as t on t.key = sub.task_key + join pgconductor._private_tasks as t on t.key = sub.task_key and t.queue = sub.queue where sub.table_name = v_table_name and sub.schema_name = v_schema_name and sub.operation = v_op diff --git a/packages/pgconductor-js/src/conductor.ts b/packages/pgconductor-js/src/conductor.ts index ce75afc..f62e6fd 100644 --- a/packages/pgconductor-js/src/conductor.ts +++ b/packages/pgconductor-js/src/conductor.ts @@ -8,6 +8,7 @@ import { type ValidateTasksQueue, type BatchConfig, type ExecuteFunction, + type ValidateTaskConfiguration, } from "./task"; import type { TaskContext, BatchTaskContext } from "./task-context"; import { @@ -162,7 +163,7 @@ export class Conductor< }, const TTriggers extends object | readonly object[], >( - definition: TDef, + definition: TDef & ValidateTaskConfiguration, triggers: TTriggers & ValidateTriggers>, fn: TDef extends { readonly batch: BatchConfig } ? ResolvedReturns extends void @@ -270,6 +271,7 @@ export class Conductor< debounce: item.debounce, cron_expression: item.cron_expression, priority: item.priority, + group: item.group, })); return this.db.invokeBatch(specs); } diff --git a/packages/pgconductor-js/src/database-client.ts b/packages/pgconductor-js/src/database-client.ts index c388fa2..a3b9561 100644 --- a/packages/pgconductor-js/src/database-client.ts +++ b/packages/pgconductor-js/src/database-client.ts @@ -33,6 +33,7 @@ export interface ExecutionSpec { debounce?: { seconds: number } | null; cron_expression?: string | null; priority?: number | null; + group?: string | null; parent_execution_id?: string | null; parent_step_key?: string | null; parent_timeout_ms?: number | null; @@ -45,7 +46,9 @@ export interface TaskSpec { removeOnCompleteDays?: number | null; removeOnFailDays?: number | null; window?: [string, string] | null; + fifo?: boolean | null; concurrency?: number | null; + groupConcurrency?: number | null; } export interface Execution { @@ -55,11 +58,13 @@ export interface Execution { payload: Payload; waiting_on_execution_id: string | null; waiting_step_key: string | null; + locked_by: string; + claim_token: string; cancelled: boolean; last_error: string | null; dedupe_key?: string | null; cron_expression?: string | null; - slot_group_number?: number | null; + group?: string | null; } // todo: move all of this to query-builder too or create new types.ts file @@ -73,6 +78,7 @@ export type ExecutionResult = export type GroupedExecutionResults = { count: number; + orchestratorId: string; completed: ExecutionCompleted[]; failed: (ExecutionFailed | ExecutionPermamentlyFailed)[]; released: ExecutionReleased[]; @@ -83,43 +89,50 @@ export type GroupedExecutionResults = { export interface ExecutionCompleted { execution_id: string; queue: string; + orchestrator_id: string; + claim_token: string; task_key: string; status: "completed"; result?: Payload; - slot_group_number?: number | null; } export interface ExecutionFailed { execution_id: string; queue: string; + orchestrator_id: string; + claim_token: string; task_key: string; status: "failed"; error: string; - slot_group_number?: number | null; } export interface ExecutionReleased { execution_id: string; queue: string; + orchestrator_id: string; + claim_token: string; task_key: string; status: "released"; reschedule_in_ms?: number | "infinity"; step_key?: string; - slot_group_number?: number | null; } export interface ExecutionPermamentlyFailed { execution_id: string; queue: string; + orchestrator_id: string; + claim_token: string; task_key: string; status: "permanently_failed"; error: string; - slot_group_number?: number | null; } export interface ExecutionInvokeChild { + group?: string | null; execution_id: string; queue: string; + orchestrator_id: string; + claim_token: string; task_key: string; status: "invoke_child"; timeout_ms: number | "infinity"; @@ -127,7 +140,6 @@ export interface ExecutionInvokeChild { child_task_name: string; child_task_queue: string; child_payload: Payload | null; - slot_group_number?: number | null; } export interface EventSubscriptionSpec { diff --git a/packages/pgconductor-js/src/generated/sql.ts b/packages/pgconductor-js/src/generated/sql.ts index 94bedab..9825a07 100644 --- a/packages/pgconductor-js/src/generated/sql.ts +++ b/packages/pgconductor-js/src/generated/sql.ts @@ -65,6 +65,8 @@ begin end; $$; +create sequence pgconductor._private_enqueue_position_seq as bigint; + create table pgconductor._private_orchestrators ( id uuid default pgconductor._private_portable_uuidv7() primary key, last_heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), @@ -111,6 +113,9 @@ create table pgconductor._private_executions ( run_at timestamptz default pgconductor._private_current_time() not null, locked_at timestamptz, locked_by uuid, + claim_token uuid, + "group" text, + enqueue_position bigint not null default nextval('pgconductor._private_enqueue_position_seq'), is_available boolean generated always as (locked_at is null and failed_at is null and completed_at is null) stored not null, attempts integer default 0 not null, last_error text, @@ -132,7 +137,7 @@ create unique index on pgconductor._private_executions (task_key, singleton_on, where singleton_on is not null and completed_at is null and failed_at is null and cancelled = false; create table pgconductor._private_tasks ( - key text primary key, + key text not null, -- queue that this task belongs to (used for queue-based worker assignment) queue text default 'default' not null, @@ -158,22 +163,24 @@ create table pgconductor._private_tasks ( ) ), - -- concurrency control: maximum number of concurrent executions across all workers + -- FIFO is a strict, persistent single-lane policy. It cannot be combined + -- with soft concurrency controls. + fifo boolean default false not null, + -- concurrency controls are intentionally soft and coordinated at claim time -- NULL means no limit (unlimited concurrency) - concurrency_limit integer -); + concurrency_limit integer, + group_concurrency_limit integer, + constraint positive_concurrency_limits check ( + (concurrency_limit is null or concurrency_limit > 0) and + (group_concurrency_limit is null or group_concurrency_limit > 0) + ), + constraint fifo_excludes_concurrency check ( + not fifo or (concurrency_limit is null and group_concurrency_limit is null) + ), -create table pgconductor._private_concurrency_slots ( - task_key text not null, - slot_group_number integer not null, - capacity integer not null, - used integer default 0 not null, - primary key (task_key, slot_group_number) + primary key (queue, key) ); -create index idx_slots_claim - on pgconductor._private_concurrency_slots (task_key, capacity, used); - create table pgconductor._private_steps ( id uuid default pgconductor._private_portable_uuidv7() primary key, key text not null, @@ -187,6 +194,19 @@ create table pgconductor._private_steps ( create index idx_steps_execution_id on pgconductor._private_steps (execution_id); +-- A FIFO task owns one durable lane. The owner survives every non-terminal +-- execution state so another worker cannot pass a sleeping or waiting owner. +create table pgconductor._private_fifo_owners ( + queue text not null, + task_key text not null, + execution_id uuid not null, + primary key (queue, task_key), + constraint fk_fifo_owner_execution foreign key (execution_id, queue) + references pgconductor._private_executions(id, queue) on delete cascade +); + +create index idx_fifo_owners_execution on pgconductor._private_fifo_owners (execution_id, queue); + -- Trigger function to manage executions partitions per queue -- Automatically creates partition when queue is inserted create or replace function pgconductor._private_manage_queue_partition() @@ -212,7 +232,7 @@ begin -- main index for fetching available executions execute format( - 'create index if not exists %I on pgconductor.%I (priority, run_at) include (id, task_key) where is_available = true', + 'create index if not exists %I on pgconductor.%I (priority, run_at, enqueue_position) include (id, task_key) where is_available = true', 'idx_' || v_partition_name || '_get_executions', v_partition_name ); @@ -266,6 +286,19 @@ begin v_partition_name ); + -- indexes used to count active executions for soft concurrency limits + execute format( + 'create index if not exists %I on pgconductor.%I (task_key) where locked_at is not null and failed_at is null and completed_at is null', + 'idx_' || v_partition_name || '_active_task', + v_partition_name + ); + + execute format( + 'create index if not exists %I on pgconductor.%I (task_key, "group") where "group" is not null and locked_at is not null and failed_at is null and completed_at is null', + 'idx_' || v_partition_name || '_active_task_group', + v_partition_name + ); + RETURN NEW; elsif tg_op = 'UPDATE' then @@ -328,7 +361,8 @@ create type pgconductor.execution_spec as ( dedupe_seconds integer, dedupe_next_slot boolean, cron_expression text, - priority integer + priority integer, + "group" text ); create type pgconductor.task_spec as ( @@ -339,7 +373,9 @@ create type pgconductor.task_spec as ( remove_on_fail_days integer, window_start timetz, window_end timetz, - concurrency_limit integer + concurrency_limit integer, + fifo boolean, + group_concurrency_limit integer ); create type pgconductor._private_event_operation as enum ( @@ -378,7 +414,7 @@ begin on conflict (name) do nothing; -- step 2: register/update tasks - insert into pgconductor._private_tasks (key, queue, max_attempts, remove_on_complete_days, remove_on_fail_days, window_start, window_end, concurrency_limit) + insert into pgconductor._private_tasks (key, queue, max_attempts, remove_on_complete_days, remove_on_fail_days, window_start, window_end, fifo, concurrency_limit, group_concurrency_limit) select spec.key, coalesce(spec.queue, 'default'), @@ -387,9 +423,11 @@ begin spec.remove_on_fail_days, spec.window_start, spec.window_end, - spec.concurrency_limit + coalesce(spec.fifo, false), + spec.concurrency_limit, + spec.group_concurrency_limit from unnest(p_task_specs) as spec - on conflict (key) + on conflict (queue, key) do update set queue = coalesce(excluded.queue, pgconductor._private_tasks.queue), max_attempts = coalesce(excluded.max_attempts, pgconductor._private_tasks.max_attempts), @@ -397,51 +435,34 @@ begin remove_on_fail_days = excluded.remove_on_fail_days, window_start = excluded.window_start, window_end = excluded.window_end, - concurrency_limit = excluded.concurrency_limit; - - -- step 2a: manage concurrency slots - -- create one row per slot (capacity=1 each) - insert into pgconductor._private_concurrency_slots (task_key, slot_group_number, capacity, used) - select - spec.key, - slot_num, - 1 as capacity, - 0 as used - from unnest(p_task_specs) as spec - cross join lateral generate_series(1, spec.concurrency_limit) as slot_num - where spec.concurrency_limit is not null - on conflict (task_key, slot_group_number) - do update set - capacity = excluded.capacity, - used = least(pgconductor._private_concurrency_slots.used, excluded.capacity); - - -- clean up orphaned slots (tasks removed or concurrency_limit set to null) - delete from pgconductor._private_concurrency_slots - where task_key not in ( - select key from pgconductor._private_tasks - where concurrency_limit is not null - ); - - -- clean up excess slots when concurrency decreased - delete from pgconductor._private_concurrency_slots cs - where cs.slot_group_number > ( - select concurrency_limit - from pgconductor._private_tasks t - where t.key = cs.task_key - ); - - -- step 3: insert scheduled cron executions (on conflict do nothing) - insert into pgconductor._private_executions (task_key, queue, payload, run_at, dedupe_key, cron_expression) + fifo = excluded.fifo, + concurrency_limit = excluded.concurrency_limit, + group_concurrency_limit = excluded.group_concurrency_limit; + + -- A task leaving FIFO must release its lane immediately. Re-enabling FIFO + -- intentionally leaves pending work ownerless; the claim query elects its + -- current head on the next poll. + delete from pgconductor._private_fifo_owners o + using pgconductor._private_tasks t + where t.queue = o.queue and t.key = o.task_key and t.fifo = false; + + -- step 3: insert scheduled cron executions + insert into pgconductor._private_executions (task_key, queue, payload, run_at, dedupe_key, cron_expression, "group") select spec.task_key, coalesce(spec.queue, 'default'), coalesce(spec.payload, '{}'::jsonb), coalesce(spec.run_at, pgconductor._private_current_time()), spec.dedupe_key, - spec.cron_expression + spec.cron_expression, + spec."group" from unnest(p_cron_schedules) as spec where spec.dedupe_key is not null - on conflict (task_key, dedupe_key, queue) do nothing; + on conflict (task_key, dedupe_key, queue) do update set + payload = excluded.payload, + run_at = excluded.run_at, + cron_expression = excluded.cron_expression, + "group" = excluded."group"; -- step 4: clean up stale schedules for this queue -- delete future executions for schedules that no longer exist @@ -541,20 +562,38 @@ declare begin v_now := pgconductor._private_current_time(); + insert into pgconductor._private_queues (name) + select distinct coalesce(spec.queue, 'default') + from unnest(specs) as spec + on conflict (name) do nothing; + -- clear locked dedupe keys before batch insert - update pgconductor._private_executions as e + with superseded as ( + select e.id, e.queue, e.task_key + from pgconductor._private_executions as e + cross join unnest(specs) as spec + where e.dedupe_key = spec.dedupe_key + and e.task_key = spec.task_key + and e.queue = coalesce(spec.queue, 'default') + and e.locked_at is not null + and spec.dedupe_key is not null + for update of e + ) + update pgconductor._private_executions e set dedupe_key = null, locked_by = null, locked_at = null, + claim_token = null, failed_at = v_now, last_error = 'superseded by reinvoke' - from unnest(specs) as spec - where e.dedupe_key = spec.dedupe_key - and e.task_key = spec.task_key - and e.queue = coalesce(spec.queue, 'default') - and e.locked_at is not null - and spec.dedupe_key is not null; + from superseded s + where e.id = s.id; + + delete from pgconductor._private_fifo_owners o + using pgconductor._private_executions e + where o.execution_id = e.id and o.queue = e.queue + and e.failed_at = v_now and e.last_error = 'superseded by reinvoke'; -- batch insert all executions -- note: duplicate dedupe_keys within same batch will cause error @@ -569,7 +608,8 @@ begin dedupe_key, singleton_on, cron_expression, - priority + priority, + "group" ) select pgconductor._private_portable_uuidv7(), @@ -588,14 +628,16 @@ begin else null end, spec.cron_expression, - coalesce(spec.priority, 0) + coalesce(spec.priority, 0), + spec."group" from unnest(specs) as spec on conflict (task_key, dedupe_key, queue) do update set payload = excluded.payload, run_at = excluded.run_at, priority = excluded.priority, cron_expression = excluded.cron_expression, - singleton_on = excluded.singleton_on + singleton_on = excluded.singleton_on, + "group" = excluded."group" returning pgconductor._private_executions.id; end; $function$ @@ -610,7 +652,8 @@ create or replace function pgconductor.invoke( p_dedupe_seconds integer default null, p_dedupe_next_slot boolean default false, p_cron_expression text default null, - p_priority integer default null + p_priority integer default null, + p_group text default null ) returns table(id uuid) language plpgsql @@ -627,19 +670,36 @@ begin v_now := pgconductor._private_current_time(); v_run_at := coalesce(p_run_at, v_now); + insert into pgconductor._private_queues (name) + values (p_queue) + on conflict (name) do nothing; + -- clear locked dedupe key before insert (supersede pattern) if p_dedupe_key is not null then - update pgconductor._private_executions + with superseded as ( + select e.id, e.queue, e.task_key + from pgconductor._private_executions e + where e.dedupe_key = p_dedupe_key + and e.task_key = p_task_key + and e.queue = p_queue + and e.locked_at is not null + for update of e + ) + update pgconductor._private_executions e set dedupe_key = null, locked_by = null, locked_at = null, + claim_token = null, failed_at = v_now, last_error = 'superseded by reinvoke' - where dedupe_key = p_dedupe_key - and task_key = p_task_key - and queue = p_queue - and locked_at is not null; + from superseded s + where e.id = s.id; + + delete from pgconductor._private_fifo_owners o + using pgconductor._private_executions e + where o.execution_id = e.id and o.queue = e.queue + and e.failed_at = v_now and e.last_error = 'superseded by reinvoke'; end if; -- singleton throttle/debounce logic @@ -663,7 +723,8 @@ begin dedupe_key, singleton_on, cron_expression, - priority + priority, + "group" ) values ( pgconductor._private_portable_uuidv7(), p_task_key, @@ -673,7 +734,8 @@ begin p_dedupe_key, v_singleton_on, p_cron_expression, - coalesce(p_priority, 0) + coalesce(p_priority, 0), + p_group ) on conflict (task_key, singleton_on, coalesce(dedupe_key, ''), queue) where singleton_on is not null and completed_at is null and failed_at is null and cancelled = false @@ -694,7 +756,8 @@ begin dedupe_key, singleton_on, cron_expression, - priority + priority, + "group" ) values ( pgconductor._private_portable_uuidv7(), p_task_key, @@ -704,35 +767,23 @@ begin p_dedupe_key, v_next_singleton_on, p_cron_expression, - coalesce(p_priority, 0) + coalesce(p_priority, 0), + p_group ) on conflict (task_key, singleton_on, coalesce(dedupe_key, ''), queue) where singleton_on is not null and completed_at is null and failed_at is null and cancelled = false do update set payload = excluded.payload, run_at = excluded.run_at, - priority = excluded.priority + priority = excluded.priority, + cron_expression = excluded.cron_expression, + "group" = excluded."group" returning _private_executions.id; return; end if; end if; -- regular invoke (no singleton) - if p_dedupe_key is not null then - -- clear keys that are currently locked so a subsequent insert can succeed. - update pgconductor._private_executions as e - set - dedupe_key = null, - locked_by = null, - locked_at = null, - failed_at = pgconductor._private_current_time(), - last_error = 'superseded by reinvoke' - where e.dedupe_key = p_dedupe_key - and e.task_key = p_task_key - and e.queue = p_queue - and e.locked_at is not null; - end if; - return query insert into pgconductor._private_executions as e ( id, task_key, @@ -741,7 +792,8 @@ begin run_at, dedupe_key, cron_expression, - priority + priority, + "group" ) values ( pgconductor._private_portable_uuidv7(), p_task_key, @@ -750,13 +802,20 @@ begin v_run_at, p_dedupe_key, p_cron_expression, - coalesce(p_priority, 0) + coalesce(p_priority, 0), + p_group ) on conflict (task_key, dedupe_key, queue) do update set payload = excluded.payload, run_at = excluded.run_at, priority = excluded.priority, - cron_expression = excluded.cron_expression + cron_expression = excluded.cron_expression, + "group" = excluded."group", + failed_at = case when e.last_error = 'superseded by reinvoke' then null else e.failed_at end, + locked_by = case when e.last_error = 'superseded by reinvoke' then null else e.locked_by end, + locked_at = case when e.last_error = 'superseded by reinvoke' then null else e.locked_at end, + claim_token = case when e.last_error = 'superseded by reinvoke' then null else e.claim_token end, + cancelled = case when e.last_error = 'superseded by reinvoke' then false else e.cancelled end returning e.id; end; $function$ @@ -774,37 +833,108 @@ set search_path to '' as $function$ declare v_orchestrator_id uuid; + v_claim_token uuid; v_queue text; + v_child_id uuid; + v_child_orchestrator_id uuid; + v_child_queue text; v_completed boolean; v_failed boolean; v_rows_affected integer; begin select locked_by, + claim_token, queue, + waiting_on_execution_id, completed_at is not null, failed_at is not null - into v_orchestrator_id, v_queue, v_completed, v_failed + into v_orchestrator_id, v_claim_token, v_queue, v_child_id, v_completed, v_failed from pgconductor._private_executions - where id = p_execution_id; + where id = p_execution_id + for update; if not found or v_completed or v_failed then return false; end if; if v_orchestrator_id is null then - -- pending: fail immediately + -- pending: fail immediately. If this is a waiting parent, resolve its + -- child relationship in the same transaction so the child cannot become + -- orphaned or leave the workflow stranded. + if v_child_id is not null then + select locked_by, queue + into v_child_orchestrator_id, v_child_queue + from pgconductor._private_executions + where id = v_child_id + for update; + + if found and v_child_orchestrator_id is null then + update pgconductor._private_executions + set + failed_at = pgconductor._private_current_time(), + last_error = 'Cancelled: parent execution was cancelled', + locked_by = null, + locked_at = null, + claim_token = null, + waiting_on_execution_id = null, + waiting_step_key = null + where id = v_child_id + and completed_at is null + and failed_at is null; + + get diagnostics v_rows_affected = row_count; + if v_rows_affected > 0 then + delete from pgconductor._private_fifo_owners + where execution_id = v_child_id and queue = v_child_queue; + end if; + elsif found then + update pgconductor._private_executions + set cancelled = true, last_error = p_reason + where id = v_child_id + and completed_at is null + and failed_at is null + and cancelled = false; + + get diagnostics v_rows_affected = row_count; + if v_rows_affected > 0 then + -- A running child keeps its FIFO owner until its cancellation is + -- fenced and settled by return/recovery. + insert into pgconductor._private_orchestrator_signals + (orchestrator_id, type, execution_id, payload) + values ( + v_child_orchestrator_id, + 'cancel_execution', + v_child_id, + jsonb_build_object('queue', v_child_queue, 'reason', p_reason) + ) + on conflict (orchestrator_id, execution_id) + where type = 'cancel_execution' and execution_id is not null + do nothing; + end if; + end if; + end if; + update pgconductor._private_executions set failed_at = pgconductor._private_current_time(), last_error = p_reason, locked_by = null, - locked_at = null + locked_at = null, + claim_token = null, + waiting_on_execution_id = null, + waiting_step_key = null where id = p_execution_id and completed_at is null - and failed_at is null; + and failed_at is null + and locked_by is null + and locked_at is null; get diagnostics v_rows_affected = row_count; + if v_rows_affected > 0 then + delete from pgconductor._private_fifo_owners + where execution_id = p_execution_id and queue = v_queue; + end if; return v_rows_affected > 0; else -- running: signal orchestrator + set cancelled flag @@ -813,6 +943,9 @@ begin cancelled = true, last_error = p_reason where id = p_execution_id + and queue = v_queue + and locked_by = v_orchestrator_id + and claim_token = v_claim_token and completed_at is null and cancelled = false; @@ -935,7 +1068,7 @@ begin sub.id ), e'\n') from pgconductor._private_event_subscriptions as sub - join pgconductor._private_tasks as t on t.key = sub.task_key + join pgconductor._private_tasks as t on t.key = sub.task_key and t.queue = sub.queue where sub.event_key is not null ); @@ -1100,7 +1233,7 @@ begin sub.id ), e'\n') from pgconductor._private_event_subscriptions as sub - join pgconductor._private_tasks as t on t.key = sub.task_key + join pgconductor._private_tasks as t on t.key = sub.task_key and t.queue = sub.queue where sub.table_name = v_table_name and sub.schema_name = v_schema_name and sub.operation = v_op diff --git a/packages/pgconductor-js/src/lib/map-concurrent.ts b/packages/pgconductor-js/src/lib/map-concurrent.ts index 4eb0808..d51b128 100644 --- a/packages/pgconductor-js/src/lib/map-concurrent.ts +++ b/packages/pgconductor-js/src/lib/map-concurrent.ts @@ -10,72 +10,55 @@ export async function* mapConcurrent( ): AsyncGenerator { const it = source[Symbol.asyncIterator](); let sourceDone = false; + let pendingRead: Promise> | null = null; let nextId = 0; - // Track promises with unique IDs - type Task = { id: number; promise: Promise }; - const active = new Map(); + type ActiveTask = { id: number; promise: Promise }; + const active = new Map(); - const nextItem = async (): Promise => { - if (sourceDone) return null; - const { value, done } = await it.next(); - if (done) { - sourceDone = true; - return null; + const startRead = () => { + if (!sourceDone && !pendingRead && active.size < limit) { + pendingRead = it.next(); } - return value; }; - const fillSlots = async () => { - while (!sourceDone && active.size < limit) { - let item: T | null; + try { + startRead(); - if (active.size === 0) { - // No active tasks - MUST block to get at least one - item = await nextItem(); - } else { - // Try non-blocking poll - const polled = source.tryNext(); - if (polled === undefined) { - // Queue empty, stop filling - break; - } - item = polled; - } + const getPendingRead = (): Promise> | null => pendingRead; - if (item === null) break; + while (active.size > 0 || pendingRead) { + const read = getPendingRead(); + const reads = read ? [read.then((result) => ({ kind: "read" as const, result }))] : []; + const tasks = [...active.values()].map((task) => + task.promise.then((result) => ({ kind: "result" as const, id: task.id, result })), + ); - const id = nextId++; - active.set(id, { id, promise: mapper(item) }); + const event = await Promise.race([...reads, ...tasks]); + if (event.kind === "read") { + pendingRead = null; + if (event.result.done) { + sourceDone = true; + } else { + const id = nextId++; + active.set(id, { id, promise: mapper(event.result.value) }); + } + startRead(); + } else { + active.delete(event.id); + yield event.result; + startRead(); + } } - }; - - await fillSlots(); - - while (active.size > 0) { - // Wrap each promise to include its ID - const wrappedPromises = Array.from(active.values()).map(async (task) => ({ - id: task.id, - result: await task.promise, - })); - - // Race to get first completed task - const { id, result } = await Promise.race(wrappedPromises); - - // Remove the completed task - active.delete(id); - - yield result; - - // Refill slots - await fillSlots(); - } - - if (typeof it.return === "function") { - try { - await it.return(); - } catch { - // ignore + } finally { + // A mapper or consumer can abort iteration while a read is pending. Always + // close the source so live queues do not retain a dangling waiter. + if (typeof it.return === "function") { + try { + await it.return(); + } catch { + // ignore cleanup errors + } } } } diff --git a/packages/pgconductor-js/src/query-builder.ts b/packages/pgconductor-js/src/query-builder.ts index a7f3a0f..d7d62f9 100644 --- a/packages/pgconductor-js/src/query-builder.ts +++ b/packages/pgconductor-js/src/query-builder.ts @@ -37,7 +37,6 @@ export type GetExecutionsArgs = { queueName: string; batchSize: number; filterTaskKeys: string[]; - taskKeysWithConcurrency: string[]; }; export type RemoveExecutionsArgs = { @@ -65,12 +64,17 @@ export type UnscheduleCronExecutionArgs = { export type LoadStepArgs = { executionId: string; + queue: string; + orchestratorId: string; + claimToken: string; key: string; }; export type SaveStepArgs = { executionId: string; queue: string; + orchestratorId: string; + claimToken: string; key: string; result: Payload | null; runAtMs?: number; @@ -78,6 +82,9 @@ export type SaveStepArgs = { export type ClearWaitingStateArgs = { executionId: string; + queue: string; + orchestratorId: string; + claimToken: string; }; export type EmitEventArgs = { @@ -169,21 +176,24 @@ export class QueryBuilder { -- fail cancelled executions from expired orchestrators failed_cancelled as ( update pgconductor._private_executions e - set - failed_at = pgconductor._private_current_time(), - locked_by = null, - locked_at = null + set failed_at = pgconductor._private_current_time(), locked_by = null, + locked_at = null, claim_token = null from expired - where e.locked_by = expired.id - and e.cancelled = true - and e.failed_at is null - and e.completed_at is null + where e.locked_by = expired.id and e.cancelled = true + and e.failed_at is null and e.completed_at is null + returning e.id, e.queue + ), released_fifo_owners as ( + delete from pgconductor._private_fifo_owners o + using failed_cancelled f + where o.execution_id = f.id and o.queue = f.queue + returning o.execution_id ) -- unlock remaining (non-cancelled) executions update pgconductor._private_executions e set locked_by = null, - locked_at = null + locked_at = null, + claim_token = null from expired where e.locked_by = expired.id and e.cancelled = false @@ -224,21 +234,23 @@ export class QueryBuilder { -- fail cancelled executions from this orchestrator failed_cancelled as ( update pgconductor._private_executions e - set - failed_at = pgconductor._private_current_time(), - locked_by = null, - locked_at = null + set failed_at = pgconductor._private_current_time(), locked_by = null, + locked_at = null, claim_token = null from deleted - where e.locked_by = deleted.id - and e.cancelled = true - and e.failed_at is null - and e.completed_at is null + where e.locked_by = deleted.id and e.cancelled = true + and e.failed_at is null and e.completed_at is null + returning e.id, e.queue + ), released_fifo_owners as ( + delete from pgconductor._private_fifo_owners o using failed_cancelled f + where o.execution_id = f.id and o.queue = f.queue + returning o.execution_id ) -- unlock remaining (non-cancelled) executions update pgconductor._private_executions e set locked_by = null, - locked_at = null + locked_at = null, + claim_token = null from deleted where e.locked_by = deleted.id and e.cancelled = false @@ -250,567 +262,389 @@ export class QueryBuilder { queueName, batchSize, filterTaskKeys, - taskKeysWithConcurrency, }: GetExecutionsArgs): PendingQuery { - // fast path: no tasks have concurrency limits - if (!taskKeysWithConcurrency.length) { - return this.sql` - with e as ( - select - e.id, - e.task_key - from pgconductor._private_executions e - where e.queue = ${queueName}::text - ${filterTaskKeys?.length ? this.sql`and e.task_key != any(${this.sql.array(filterTaskKeys)}::text[])` : this.sql``} - and e.run_at <= pgconductor._private_current_time() - and e.is_available = true - order by e.priority asc, e.run_at asc - limit ${batchSize}::integer - for update skip locked - ) - - update pgconductor._private_executions - set - attempts = _private_executions.attempts + 1, - locked_by = ${orchestratorId}::uuid, - locked_at = pgconductor._private_current_time() - from e - where _private_executions.id = e.id - and _private_executions.queue = ${queueName}::text - returning - _private_executions.id, - _private_executions.task_key, - _private_executions.queue, - _private_executions.payload, - _private_executions.waiting_on_execution_id, - _private_executions.waiting_step_key, - _private_executions.cancelled, - _private_executions.last_error, - _private_executions.dedupe_key, - _private_executions.cron_expression, - null as slot_group_number - `; - } - - // slow path: some tasks have concurrency limits (OPTIMIZED) return this.sql` - with - -- lock up to batchSize slots per concurrency task - locked_slots_raw as ( - select t.task_key, ls.slot_group_number - from unnest(${this.sql.array(taskKeysWithConcurrency)}::text[]) as t(task_key) - cross join lateral ( - select s.slot_group_number - from pgconductor._private_concurrency_slots s - where s.task_key = t.task_key - and s.used = 0 - order by s.slot_group_number - limit ${batchSize}::integer - for update skip locked - ) as ls - ), - - -- count slots per task - slots_per_task as ( - select task_key, count(*) as slot_count - from locked_slots_raw - group by task_key - ), - - -- lock jobs for concurrency tasks (limit by slot count) - concurrency_execs as ( - select t.task_key, le.* - from slots_per_task t - cross join lateral ( - select - e.id, - e.task_key as exec_task_key, - e.queue, - e.payload, - e.waiting_on_execution_id, - e.waiting_step_key, - e.cancelled, - e.last_error, - e.dedupe_key, - e.cron_expression, - e.priority, - e.run_at - from pgconductor._private_executions e - where e.is_available = true - and e.run_at <= pgconductor._private_current_time() - and e.queue = ${queueName}::text - and e.task_key = t.task_key - ${filterTaskKeys.length ? this.sql`and not (e.task_key = any(${this.sql.array(filterTaskKeys)}::text[]))` : this.sql``} - order by e.priority asc, e.run_at asc, e.id asc - limit t.slot_count - for update skip locked - ) as le - ), - - -- lock jobs from non-concurrency tasks - unlimited_execs as ( - select - e.id, - e.task_key, - e.queue, - e.payload, - e.waiting_on_execution_id, - e.waiting_step_key, - e.cancelled, - e.last_error, - e.dedupe_key, - e.cron_expression, - e.priority, - e.run_at - from pgconductor._private_executions e - where e.is_available = true - and e.run_at <= pgconductor._private_current_time() - and e.queue = ${queueName}::text - and not (e.task_key = any(${this.sql.array(taskKeysWithConcurrency)}::text[])) - ${filterTaskKeys.length > 0 ? this.sql`and not (e.task_key = any(${this.sql.array(filterTaskKeys)}::text[]))` : this.sql``} - order by e.priority asc, e.run_at asc, e.id asc - limit ${batchSize}::integer - for update skip locked - ), - - -- row number concurrency executions by task - concurrency_execs_rn as ( - select - ce.*, - row_number() over (partition by ce.task_key order by ce.priority asc, ce.run_at asc, ce.id asc) as exec_rn - from concurrency_execs ce - ), - - -- row number slots by task - slots_rn as ( - select - ls.*, - row_number() over (partition by ls.task_key order by ls.slot_group_number) as slot_rn - from locked_slots_raw ls - ), - - -- pair concurrency executions with slots - concurrency_paired as ( - select - e.id, - e.exec_task_key as task_key, - e.queue, - e.payload, - e.waiting_on_execution_id, - e.waiting_step_key, - e.cancelled, - e.last_error, - e.dedupe_key, - e.cron_expression, - s.slot_group_number - from concurrency_execs_rn e - join slots_rn s - on e.task_key = s.task_key - and e.exec_rn = s.slot_rn - ), - - -- unlimited executions don't need slots - unlimited_paired as ( - select - id, - task_key, - queue, - payload, - waiting_on_execution_id, - waiting_step_key, - cancelled, - last_error, - dedupe_key, - cron_expression, - null::integer as slot_group_number - from unlimited_execs - ), - - -- merge all paired executions - paired as ( - select * from concurrency_paired - union all - select * from unlimited_paired - ), - - -- mark slots as used - mark_used as ( - update pgconductor._private_concurrency_slots cs - set used = 1 - from paired p - where cs.task_key = p.task_key - and cs.slot_group_number = p.slot_group_number - and p.slot_group_number is not null - ) - - -- update and return executions - update pgconductor._private_executions e - set - attempts = e.attempts + 1, - locked_by = ${orchestratorId}::uuid, - locked_at = pgconductor._private_current_time() - from paired p - where e.id = p.id - returning - e.id, - e.task_key, - e.queue, - e.payload, - e.waiting_on_execution_id, - e.waiting_step_key, - e.cancelled, - e.last_error, - e.dedupe_key, - e.cron_expression, - p.slot_group_number + with active_tasks as ( + select e.task_key, count(*)::integer as active_count + from pgconductor._private_executions e + where e.queue = ${queueName}::text and e.locked_at is not null + and e.failed_at is null and e.completed_at is null + group by e.task_key + ), active_groups as ( + select e.task_key, e."group", count(*)::integer as active_count + from pgconductor._private_executions e + where e.queue = ${queueName}::text and e."group" is not null + and e.locked_at is not null and e.failed_at is null and e.completed_at is null + group by e.task_key, e."group" + ), ranked as ( + select e.id, e.task_key, e.queue, e.priority, e.run_at, e.enqueue_position, e."group", + t.concurrency_limit, t.group_concurrency_limit, + coalesce(at.active_count, 0) as active_task_count, + coalesce(ag.active_count, 0) as active_group_count, + row_number() over (partition by e.task_key + order by e.priority asc, e.run_at asc, e.enqueue_position asc) as task_rank, + row_number() over (partition by e.task_key, e."group" + order by e.priority asc, e.run_at asc, e.enqueue_position asc) as group_rank + from pgconductor._private_executions e + join pgconductor._private_tasks t on t.key = e.task_key and t.queue = e.queue + left join active_tasks at on at.task_key = e.task_key + left join active_groups ag on ag.task_key = e.task_key and ag."group" = e."group" + where e.queue = ${queueName}::text and t.fifo = false + and e.run_at <= pgconductor._private_current_time() and e.is_available = true + ${filterTaskKeys?.length ? this.sql`and not (e.task_key = any(${this.sql.array(filterTaskKeys)}::text[]))` : this.sql``} + ), group_eligible as ( + select r.*, row_number() over (partition by r.task_key + order by r.priority asc, r.run_at asc, r.enqueue_position asc) as available_task_rank + from ranked r + where r.group_concurrency_limit is null or r."group" is null + or r.active_group_count + r.group_rank <= r.group_concurrency_limit + ), standard_eligible as ( + select r.id, r.priority, r.run_at, r.enqueue_position + from group_eligible r + where r.concurrency_limit is null + or r.active_task_count + r.available_task_rank <= r.concurrency_limit + ), stale_fifo_owners as ( + delete from pgconductor._private_fifo_owners o + where o.queue = ${queueName}::text + and not exists ( + select 1 from pgconductor._private_executions e + where e.id = o.execution_id and e.queue = o.queue + and e.completed_at is null and e.failed_at is null + ) + returning o.queue, o.task_key + ), fifo_ranked as ( + -- Future executions do not block ready work, but locked rows remain in + -- the ranking so a racing client cannot elect a later head. + select e.id, e.task_key, e.queue, e.priority, e.run_at, e.enqueue_position, + e.is_available, o.execution_id as owner_execution_id, + row_number() over (partition by e.task_key order by e.enqueue_position asc) as fifo_rank + from pgconductor._private_executions e + join pgconductor._private_tasks t on t.key = e.task_key and t.queue = e.queue + left join pgconductor._private_fifo_owners o + on o.queue = e.queue and o.task_key = e.task_key + left join stale_fifo_owners stale on stale.queue = e.queue and stale.task_key = e.task_key + where e.queue = ${queueName}::text and t.fifo = true + and e.run_at <= pgconductor._private_current_time() + and e.completed_at is null and e.failed_at is null + and stale.queue is null + ${filterTaskKeys?.length ? this.sql`and not (e.task_key = any(${this.sql.array(filterTaskKeys)}::text[]))` : this.sql``} + ), fifo_candidates as ( + select id, task_key, queue, priority, run_at, enqueue_position, owner_execution_id + from fifo_ranked + where (owner_execution_id = id or (owner_execution_id is null and fifo_rank = 1)) + and is_available + ), eligible as ( + select id, null::text as task_key, false as is_fifo, priority, run_at, enqueue_position, + null::uuid as owner_execution_id + from standard_eligible + union all + select id, task_key, true, priority, run_at, enqueue_position, owner_execution_id + from fifo_candidates + ), locked_candidates as ( + -- The global batch limit and row locks happen before creating FIFO + -- owners. Thus an unclaimed FIFO row outside this batch gets no owner. + select e.id, c.task_key, c.is_fifo, c.owner_execution_id + from pgconductor._private_executions e + join eligible c on c.id = e.id + where e.queue = ${queueName}::text and e.is_available = true + order by c.priority asc, c.run_at asc, c.enqueue_position asc + limit ${batchSize}::integer + for update of e skip locked + ), new_fifo_owners as ( + insert into pgconductor._private_fifo_owners (queue, task_key, execution_id) + select ${queueName}::text, c.task_key, c.id + from locked_candidates c + where c.is_fifo and c.owner_execution_id is null + on conflict (queue, task_key) do nothing + returning queue, task_key, execution_id + ), claimed as ( + update pgconductor._private_executions e + set attempts = e.attempts + 1, locked_by = ${orchestratorId}::uuid, + claim_token = pgconductor._private_portable_uuidv7(), + locked_at = pgconductor._private_current_time() + from locked_candidates c + where e.id = c.id and e.queue = ${queueName}::text and e.is_available = true + and (not c.is_fifo + or c.owner_execution_id = e.id + or exists ( + select 1 from new_fifo_owners n + where n.queue = e.queue and n.task_key = e.task_key and n.execution_id = e.id + )) + returning e.id, e.task_key, e.queue, e.payload, e.waiting_on_execution_id, + e.waiting_step_key, e.cancelled, e.last_error, e.dedupe_key, e.cron_expression, + e.locked_by, e.claim_token, e."group", e.priority, e.run_at, e.enqueue_position + ) + select id, task_key, queue, payload, waiting_on_execution_id, waiting_step_key, + cancelled, last_error, dedupe_key, cron_expression, locked_by, claim_token, "group" + from claimed order by priority asc, run_at asc, enqueue_position asc `; } buildReturnExecutions(grouped: GroupedExecutionResults): PendingQuery | null { - const completed = grouped.completed; - const failed = grouped.failed; - const released = grouped.released; - const invokeChild = grouped.invokeChild; + const allResults = [ + ...grouped.completed, + ...grouped.failed, + ...grouped.released, + ...grouped.invokeChild, + ]; - if (grouped.count === 0) { - return null; - } + if (allResults.length === 0) return null; const ctes: PendingQuery[] = []; - - // Precompute timestamp once ctes.push(this.sql`now_ts as (select pgconductor._private_current_time() as ts)`); - - // Load task configs once for all task_keys we're processing + ctes.push(this.sql`result_data as ( + select * from jsonb_to_recordset(${this.sql.json(JSON.parse(JSON.stringify(allResults)))}::jsonb) + as r( + execution_id uuid, queue text, task_key text, status text, + orchestrator_id uuid, claim_token uuid, result jsonb, error text, + reschedule_in_ms text, step_key text, timeout_ms text, + child_task_name text, child_task_queue text, child_payload jsonb, + "group" text + ) + )`); + // Lock the claimed rows for the whole statement. This prevents recovery or a + // new claim from racing the side effects below. + ctes.push(this.sql`valid_results as materialized ( + select r.*, + e.cancelled as execution_cancelled, e.last_error as execution_last_error + from result_data r + join pgconductor._private_executions e + on e.id = r.execution_id + and e.queue = r.queue + and e.task_key = r.task_key + and e.locked_by = r.orchestrator_id + and e.claim_token = r.claim_token + for update of e + )`); ctes.push(this.sql`task_configs as ( - select key, max_attempts, remove_on_complete_days, remove_on_fail_days + select queue, key, max_attempts, remove_on_complete_days, remove_on_fail_days from pgconductor._private_tasks - where key = any(${this.sql.array(Array.from(grouped.taskKeys))}::text[]) + where queue = any(${this.sql.array(Array.from(new Set(allResults.map((r) => r.queue))))}::text[]) + )`); + ctes.push(this.sql`completed_results as ( + select * from valid_results where status = 'completed' and not execution_cancelled + )`); + ctes.push(this.sql`failed_results as ( + select * from valid_results + where status in ('failed', 'permanently_failed') + or (status in ('completed', 'released') and execution_cancelled) + )`); + ctes.push(this.sql`released_results as ( + select * from valid_results where status = 'released' and not execution_cancelled + )`); + ctes.push(this.sql`invoke_child_data as ( + select * from valid_results where status = 'invoke_child' )`); - // Release concurrency slots for all results with slot_group_number - const allResults = [ - ...completed, - ...failed, - ...released, - ...invokeChild, - // ...waitForCustomEvent, - // ...waitForDbEvent, - ]; - const slotsToRelease = allResults - .filter((r) => r.slot_group_number != null) - .map((r) => ({ - task_key: r.task_key, - slot_group_number: r.slot_group_number, - })); - - if (slotsToRelease.length > 0) { - ctes.push(this.sql`released_slots as ( - update pgconductor._private_concurrency_slots cs - set used = 0 - from jsonb_to_recordset(${this.sql.json(slotsToRelease)}::jsonb) - as r(task_key text, slot_group_number integer) - where cs.task_key = r.task_key - and cs.slot_group_number = r.slot_group_number - )`); - } - - // Completed results - if (completed.length > 0) { - const completedData = completed.map((r) => ({ - execution_id: r.execution_id, - task_key: r.task_key, - result: r.result || null, - })); - - ctes.push(this.sql`completed_results as ( - select * from jsonb_to_recordset(${this.sql.json(completedData)}::jsonb) - as x(execution_id uuid, task_key text, result jsonb) - )`); - - // Insert parent steps for all completed - ctes.push(this.sql`parent_steps_all as ( - insert into pgconductor._private_steps (execution_id, queue, key, result) - select - parent_e.id, - parent_e.queue, - parent_e.waiting_step_key, - r.result - from completed_results r - join pgconductor._private_executions parent_e on parent_e.waiting_on_execution_id = r.execution_id - on conflict (execution_id, key) do nothing - returning execution_id - )`); - - // Mark orphaned children (completed but no parent waiting) as failed - ctes.push(this.sql`orphaned_children as ( - update pgconductor._private_executions e - set - failed_at = nt.ts, - completed_at = null, - last_error = 'Parent timed out before child completed', - locked_by = null, - locked_at = null - from now_ts nt, completed_results r - where e.id = r.execution_id - -- Only apply to child executions (has a parent) - and e.parent_execution_id is not null - -- No parent is waiting for this child - and not exists ( - select 1 from pgconductor._private_executions parent - where parent.waiting_on_execution_id = r.execution_id - ) - returning e.id - )`); - - // Update parents for all completed - ctes.push(this.sql`updated_parents_all as ( - update pgconductor._private_executions e - set - run_at = nt.ts, - waiting_on_execution_id = null, - waiting_step_key = null, - locked_by = null, - locked_at = null - from now_ts nt, completed_results r - where e.waiting_on_execution_id = r.execution_id - )`); - - // Delete completed where remove_on_complete_days = 0 (excluding orphaned) - ctes.push(this.sql`deleted_completed as ( - delete from pgconductor._private_executions e - using completed_results r, task_configs tc - where e.id = r.execution_id - and tc.key = r.task_key - and tc.remove_on_complete_days = 0 - and not exists (select 1 from orphaned_children oc where oc.id = e.id) - )`); - - // Update completed where remove_on_complete_days != 0 (keep) - ctes.push(this.sql`updated_completed as ( - update pgconductor._private_executions e - set - completed_at = nt.ts, - locked_by = null, - locked_at = null - from now_ts nt, completed_results r, task_configs tc - where e.id = r.execution_id - and tc.key = r.task_key - and (tc.remove_on_complete_days is null or tc.remove_on_complete_days != 0) - and not exists (select 1 from orphaned_children oc where oc.id = e.id) - )`); - } - - // Failed results - if (failed.length > 0) { - ctes.push(this.sql`failed_results as ( - select * from jsonb_to_recordset(${this.sql.json(failed as unknown as JsonValue)}::jsonb) - as x(execution_id uuid, task_key text, status text, error text) - )`); - - // Permanently failed children (attempts >= max_attempts) - ctes.push(this.sql`permanently_failed_children as ( - select - r.execution_id, - r.task_key, - r.error as child_error, - tc.remove_on_fail_days = 0 as should_remove - from failed_results r, pgconductor._private_executions e, task_configs tc - where e.id = r.execution_id - and tc.key = r.task_key - and (e.attempts >= tc.max_attempts or r.status = 'permanently_failed') - )`); - - // Delete permanently failed children and their parents - ctes.push(this.sql`deleted_failed as ( - delete from pgconductor._private_executions e - using permanently_failed_children p - where - (e.id = p.execution_id and p.should_remove) - or ( - e.waiting_on_execution_id = p.execution_id - and exists ( - select 1 from pgconductor._private_tasks t - where t.key = e.task_key and t.remove_on_fail_days = 0 - ) - ) - )`); - - // Failed updates (permanently failed children + parents) - ctes.push(this.sql`failed_updates as ( - select - e.id as target_id, - p.child_error as error, - true as is_child - from permanently_failed_children p - join pgconductor._private_executions e on e.id = p.execution_id - where p.should_remove = false - union all - select - e.id as target_id, - p.child_error as error, - false as is_child - from permanently_failed_children p - join pgconductor._private_executions e on e.waiting_on_execution_id = p.execution_id - where not exists ( - select 1 from pgconductor._private_tasks t - where t.key = e.task_key and t.remove_on_fail_days = 0 - ) - )`); - - // Update all permanently failed - ctes.push(this.sql`updated_failed_all as ( - update pgconductor._private_executions e - set - failed_at = nt.ts, - last_error = case - when f.is_child then coalesce(f.error, 'unknown error') - else 'Child execution failed: ' || coalesce(f.error, 'unknown error') - end, - waiting_on_execution_id = null, - waiting_step_key = null, - locked_by = null, - locked_at = null - from now_ts nt, failed_updates f - where e.id = f.target_id - )`); - - // Retry failed (not permanently failed) - ctes.push(this.sql`retried as ( - update pgconductor._private_executions e - set - last_error = coalesce(r.error, 'unknown error'), - run_at = greatest(nt.ts, coalesce(e.run_at, nt.ts)) - + ((array[15, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200])[least(greatest(e.attempts, 1), 10)] * interval '1 second'), - locked_by = null, - locked_at = null - from now_ts nt, failed_results r, task_configs tc - where e.id = r.execution_id - and tc.key = r.task_key - and e.attempts < tc.max_attempts - )`); - } - - // Released - if (released.length > 0) { - // Save steps for released executions with step_key (e.g., sleep) - const releasedWithSteps = released.filter((r) => r.step_key !== undefined); - if (releasedWithSteps.length > 0) { - ctes.push(this.sql`released_steps as ( - insert into pgconductor._private_steps (execution_id, queue, key, result) - select - r.execution_id, - r.queue, - r.step_key, - null::jsonb - from jsonb_to_recordset(${this.sql.json(releasedWithSteps as unknown as JsonValue)}::jsonb) - as r(execution_id uuid, queue text, step_key text) - on conflict (execution_id, key) do nothing - returning id - )`); - } - - const shouldRescheduleSome = released.some((r) => r.reschedule_in_ms !== undefined); - - if (shouldRescheduleSome) { - const releasedData = released.map((r) => ({ - execution_id: r.execution_id, - reschedule_in_ms: r.reschedule_in_ms === "infinity" ? -1 : r.reschedule_in_ms, - })); - - ctes.push(this.sql`updated_released as ( - update pgconductor._private_executions e - set - attempts = greatest(attempts - 1, 0), - run_at = case - when r.reschedule_in_ms = -1 then - 'infinity'::timestamptz - when r.reschedule_in_ms is not null then - nt.ts + (r.reschedule_in_ms::integer || ' milliseconds')::interval - else - nt.ts - end, - locked_by = null, - locked_at = null - from now_ts nt, jsonb_to_recordset(${this.sql.json(releasedData)}::jsonb) - as r(execution_id uuid, reschedule_in_ms integer) - where e.id = r.execution_id - )`); - } else { - const releasedIds = released.map((r) => r.execution_id); - - ctes.push(this.sql`updated_released as ( - update pgconductor._private_executions - set - attempts = greatest(attempts - 1, 0), - locked_by = null, - locked_at = null - where id = any(${this.sql.array(releasedIds)}::uuid[]) - )`); - } - } - - if (invokeChild.length > 0) { - ctes.push(this.sql`invoke_child_data as ( - select * from jsonb_to_recordset(${this.sql.json(invokeChild as unknown as JsonValue)}::jsonb) - as x( - execution_id uuid, - task_key text, - queue text, - step_key text, - timeout_ms text, - child_task_name text, - child_task_queue text, - child_payload jsonb + // A completed child may wake only a parent which is still waiting and is not + // currently claimed. The row lock makes this check race-safe. + ctes.push(this.sql`completed_parents as materialized ( + select parent.id as parent_id, parent.queue, parent.waiting_step_key, r.result + from completed_results r + join pgconductor._private_executions parent + on parent.waiting_on_execution_id = r.execution_id + where parent.completed_at is null + and parent.failed_at is null + and parent.locked_by is null + for update of parent + )`); + ctes.push(this.sql`parent_steps_all as ( + insert into pgconductor._private_steps (execution_id, queue, key, result) + select parent_id, queue, waiting_step_key, result + from completed_parents + where waiting_step_key is not null + on conflict (execution_id, key) do nothing + returning execution_id + )`); + ctes.push(this.sql`orphaned_children as ( + update pgconductor._private_executions e + set failed_at = nt.ts, completed_at = null, + last_error = 'Parent timed out before child completed', + locked_by = null, locked_at = null, claim_token = null + from now_ts nt, completed_results r + where e.id = r.execution_id and e.queue = r.queue + and e.locked_by = r.orchestrator_id and e.claim_token = r.claim_token + and e.parent_execution_id is not null + and not exists ( + select 1 from pgconductor._private_executions parent + where parent.waiting_on_execution_id = r.execution_id ) - )`); + returning e.id + )`); + ctes.push(this.sql`updated_parents_all as ( + update pgconductor._private_executions e + set run_at = nt.ts, waiting_on_execution_id = null, waiting_step_key = null, + locked_by = null, locked_at = null, claim_token = null + from now_ts nt, completed_parents p + where e.id = p.parent_id and e.queue = p.queue + returning e.id + )`); + ctes.push(this.sql`deleted_completed as ( + delete from pgconductor._private_executions e + using completed_results r, task_configs tc + where e.id = r.execution_id and e.queue = r.queue + and e.locked_by = r.orchestrator_id and e.claim_token = r.claim_token + and tc.key = r.task_key and tc.queue = r.queue and tc.remove_on_complete_days = 0 + and not exists (select 1 from orphaned_children oc where oc.id = e.id) + returning e.id + )`); + ctes.push(this.sql`updated_completed as ( + update pgconductor._private_executions e + set completed_at = nt.ts, locked_by = null, locked_at = null, + claim_token = null + from now_ts nt, completed_results r, task_configs tc + where e.id = r.execution_id and e.queue = r.queue + and e.locked_by = r.orchestrator_id and e.claim_token = r.claim_token + and tc.key = r.task_key and tc.queue = r.queue + and (tc.remove_on_complete_days is null or tc.remove_on_complete_days != 0) + and not exists (select 1 from orphaned_children oc where oc.id = e.id) + returning e.id, e.queue + )`); + ctes.push(this.sql`terminal_fifo_owners as ( + delete from pgconductor._private_fifo_owners o + where exists ( + select 1 from completed_results r + where r.execution_id = o.execution_id and r.queue = o.queue + ) + returning o.execution_id + )`); - // Insert child executions with parent reference - ctes.push(this.sql`inserted_children as ( - insert into pgconductor._private_executions ( - id, - task_key, - queue, - payload, - run_at, - parent_execution_id - ) - select - pgconductor._private_portable_uuidv7(), - icd.child_task_name, - icd.child_task_queue, - icd.child_payload, - nt.ts, - icd.execution_id - from invoke_child_data icd, now_ts nt - returning id, parent_execution_id - )`); + ctes.push(this.sql`permanently_failed_children as materialized ( + select r.execution_id, r.queue, r.task_key, r.orchestrator_id, r.claim_token, + coalesce(r.error, r.execution_last_error, 'unknown error') as child_error, + tc.remove_on_fail_days = 0 as should_remove + from failed_results r + join pgconductor._private_executions e on e.id = r.execution_id and e.queue = r.queue + join task_configs tc on tc.key = r.task_key and tc.queue = r.queue + where e.attempts >= tc.max_attempts + or r.status = 'permanently_failed' + or r.execution_cancelled + )`); + ctes.push(this.sql`failed_parent_targets as materialized ( + select p.execution_id as child_id, p.queue as child_queue, p.child_error, + parent.id as parent_id, parent.queue as parent_queue, + pt.remove_on_fail_days = 0 as parent_should_remove + from permanently_failed_children p + join pgconductor._private_executions parent + on parent.waiting_on_execution_id = p.execution_id + join pgconductor._private_tasks pt + on pt.key = parent.task_key and pt.queue = parent.queue + where parent.completed_at is null and parent.failed_at is null and parent.locked_by is null + for update of parent + )`); + ctes.push(this.sql`failed_updates as ( + select p.execution_id as target_id, p.queue, p.child_error, true as is_child + from permanently_failed_children p + where p.should_remove is not true + union all + select p.parent_id, p.parent_queue, p.child_error, false + from failed_parent_targets p + where p.parent_should_remove is not true + )`); + ctes.push(this.sql`deleted_failed as ( + delete from pgconductor._private_executions e + where exists ( + select 1 from permanently_failed_children p + where e.id = p.execution_id and e.queue = p.queue + and e.locked_by = p.orchestrator_id and e.claim_token = p.claim_token + and p.should_remove is true + ) + or exists ( + select 1 from failed_parent_targets p + where e.id = p.parent_id and e.queue = p.parent_queue + and p.parent_should_remove is true + ) + returning e.id + )`); + ctes.push(this.sql`updated_failed as ( + update pgconductor._private_executions e + set failed_at = nt.ts, + last_error = case when f.is_child then coalesce(f.child_error, 'unknown error') + else 'Child execution failed: ' || coalesce(f.child_error, 'unknown error') end, + waiting_on_execution_id = null, waiting_step_key = null, + locked_by = null, locked_at = null, claim_token = null + from now_ts nt, failed_updates f + where e.id = f.target_id and e.queue = f.queue + returning e.id, e.queue + )`); + ctes.push(this.sql`failed_fifo_owners as ( + delete from pgconductor._private_fifo_owners o + where exists ( + select 1 from permanently_failed_children p + where p.execution_id = o.execution_id and p.queue = o.queue + ) + or exists ( + select 1 from failed_parent_targets p + where p.parent_id = o.execution_id and p.parent_queue = o.queue + ) + returning o.execution_id + )`); + ctes.push(this.sql`retried as ( + update pgconductor._private_executions e + set last_error = coalesce(r.error, 'unknown error'), + run_at = greatest(nt.ts, coalesce(e.run_at, nt.ts)) + + ((array[15, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200])[least(greatest(e.attempts, 1), 10)] * interval '1 second'), + locked_by = null, locked_at = null, claim_token = null + from now_ts nt, failed_results r, task_configs tc + where e.id = r.execution_id and e.queue = r.queue + and e.locked_by = r.orchestrator_id and e.claim_token = r.claim_token + and tc.key = r.task_key and tc.queue = r.queue + and r.status <> 'permanently_failed' + and not r.execution_cancelled + and e.attempts < tc.max_attempts + returning e.id + )`); - // Update parent executions - ctes.push(this.sql`updated_invoke_parents as ( - update pgconductor._private_executions e - set - waiting_on_execution_id = ic.id, - waiting_step_key = icd.step_key, - run_at = case - when icd.timeout_ms = 'infinity' then 'infinity'::timestamptz - else nt.ts + (icd.timeout_ms::bigint || ' milliseconds')::interval - end, - locked_by = null, - locked_at = null - from now_ts nt, inserted_children ic - join invoke_child_data icd on icd.execution_id = ic.parent_execution_id - where e.id = ic.parent_execution_id - )`); - } + ctes.push(this.sql`released_steps as ( + insert into pgconductor._private_steps (execution_id, queue, key, result) + select execution_id, queue, step_key, null::jsonb from released_results + where step_key is not null + on conflict (execution_id, key) do nothing + returning id + )`); + ctes.push(this.sql`updated_released as ( + update pgconductor._private_executions e + set attempts = greatest(e.attempts - 1, 0), + run_at = case when lower(nullif(trim(r.reschedule_in_ms), '')) = 'infinity' then 'infinity'::timestamptz + when nullif(trim(r.reschedule_in_ms), '') is not null then + nt.ts + (nullif(trim(r.reschedule_in_ms), '')::bigint || ' milliseconds')::interval + else nt.ts end, + locked_by = null, locked_at = null, claim_token = null + from now_ts nt, released_results r + where e.id = r.execution_id and e.queue = r.queue + and e.locked_by = r.orchestrator_id and e.claim_token = r.claim_token + returning e.id + )`); - if (ctes.length <= 1) return null; // Only now_ts + ctes.push(this.sql`inserted_children as ( + insert into pgconductor._private_executions (id, task_key, queue, payload, run_at, parent_execution_id, "group") + select pgconductor._private_portable_uuidv7(), r.child_task_name, r.child_task_queue, r.child_payload, nt.ts, r.execution_id, r."group" + from invoke_child_data r, now_ts nt + where exists ( + select 1 from pgconductor._private_executions parent + where parent.id = r.execution_id and parent.queue = r.queue + and parent.locked_by = r.orchestrator_id and parent.claim_token = r.claim_token + ) + returning id, parent_execution_id + )`); + ctes.push(this.sql`updated_invoke_parents as ( + update pgconductor._private_executions e + set waiting_on_execution_id = ic.id, waiting_step_key = r.step_key, + run_at = case when lower(nullif(trim(r.timeout_ms), '')) = 'infinity' then 'infinity'::timestamptz + when nullif(trim(r.timeout_ms), '') is not null then + nt.ts + (nullif(trim(r.timeout_ms), '')::bigint || ' milliseconds')::interval + else nt.ts end, + locked_by = null, locked_at = null, claim_token = null + from now_ts nt, inserted_children ic + join invoke_child_data r on r.execution_id = ic.parent_execution_id + where e.id = r.execution_id and e.queue = r.queue + and e.locked_by = r.orchestrator_id and e.claim_token = r.claim_token + returning e.id + )`); const combined = ctes.reduce((acc, cte, i) => (i === 0 ? cte : this.sql`${acc}, ${cte}`)); - return this.sql<[{ result: number }]>`with ${combined} select 1 as result`; } - buildRemoveExecutions({ queueName, batchSize, @@ -819,7 +653,7 @@ export class QueryBuilder { with batch as ( select e.id from pgconductor._private_executions e - join pgconductor._private_tasks t on t.key = e.task_key + join pgconductor._private_tasks t on t.key = e.task_key and t.queue = e.queue where e.queue = ${queueName} and ( (e.completed_at is not null and t.remove_on_complete_days > 0 and e.completed_at < pgconductor._private_current_time() - t.remove_on_complete_days * interval '1 day') @@ -852,7 +686,9 @@ export class QueryBuilder { remove_on_fail_days: spec.removeOnFailDays ?? null, window_start: spec.window?.[0] || null, window_end: spec.window?.[1] || null, + fifo: spec.fifo || false, concurrency_limit: spec.concurrency || null, + group_concurrency_limit: spec.groupConcurrency || null, })); const cronScheduleRows = cronSchedules.map((spec) => { @@ -866,6 +702,7 @@ export class QueryBuilder { dedupe_key: spec.dedupe_key, cron_expression: spec.cron_expression, priority: spec.priority || null, + group: spec.group || null, }; }); @@ -923,7 +760,8 @@ export class QueryBuilder { p_dedupe_seconds := ${dedupe_seconds}::integer, p_dedupe_next_slot := ${dedupe_next_slot}::boolean, p_cron_expression := ${spec.cron_expression || null}::text, - p_priority := ${spec.priority || null}::integer + p_priority := ${spec.priority || null}::integer, + p_group := ${spec.group || null}::text ) `; } @@ -959,7 +797,8 @@ export class QueryBuilder { p_dedupe_seconds := null::integer, p_dedupe_next_slot := false::boolean, p_cron_expression := ${cronExpression}::text, - p_priority := ${spec.priority || 0}::integer + p_priority := ${spec.priority || 0}::integer, + p_group := ${spec.group || null}::text ) `; } @@ -1029,6 +868,7 @@ export class QueryBuilder { dedupe_next_slot, cron_expression: spec.cron_expression || null, priority: spec.priority, + group: spec.group || null, }; }); @@ -1041,10 +881,23 @@ export class QueryBuilder { `; } - buildLoadStep({ executionId, key }: LoadStepArgs): PendingQuery<[{ result: Payload | null }]> { + buildLoadStep({ + executionId, + queue, + orchestratorId, + claimToken, + key, + }: LoadStepArgs): PendingQuery<[{ result: Payload | null }]> { return this.sql<[{ result: Payload | null }]>` select result from pgconductor._private_steps - where execution_id = ${executionId}::uuid and key = ${key}::text + where execution_id = ${executionId}::uuid + and queue = ${queue}::text + and exists ( + select 1 from pgconductor._private_executions e + where e.id = ${executionId}::uuid and e.queue = ${queue}::text + and e.locked_by = ${orchestratorId}::uuid and e.claim_token = ${claimToken}::uuid + ) + and key = ${key}::text `; } @@ -1054,39 +907,72 @@ export class QueryBuilder { key, result, runAtMs, + orchestratorId, + claimToken, }: SaveStepArgs): PendingQuery> { if (runAtMs) { - return this.sql` - with inserted as ( + return this.sql>` + with claimed_execution as materialized ( + select e.id, e.queue + from pgconductor._private_executions e + where e.id = ${executionId}::uuid + and e.queue = ${queue}::text + and e.locked_by = ${orchestratorId}::uuid + and e.claim_token = ${claimToken}::uuid + for update + ), inserted as ( insert into pgconductor._private_steps (execution_id, queue, key, result) - values (${executionId}::uuid, ${queue}::text, ${key}::text, ${this.sql.json(result)}::jsonb) + select e.id, e.queue, ${key}::text, ${this.sql.json(result)}::jsonb + from claimed_execution e on conflict (execution_id, key) do nothing returning id ) - update pgconductor._private_executions + update pgconductor._private_executions e set run_at = pgconductor._private_current_time() + (${runAtMs}::integer || ' milliseconds')::interval - where id = ${executionId}::uuid - and queue = ${queue}::text + from claimed_execution c + where e.id = c.id and e.queue = c.queue and exists (select 1 from inserted) `; } - return this.sql` + return this.sql>` + with claimed_execution as materialized ( + select e.id, e.queue + from pgconductor._private_executions e + where e.id = ${executionId}::uuid + and e.queue = ${queue}::text + and e.locked_by = ${orchestratorId}::uuid + and e.claim_token = ${claimToken}::uuid + for update + ) insert into pgconductor._private_steps (execution_id, queue, key, result) - values (${executionId}::uuid, ${queue}::text, ${key}::text, ${this.sql.json(result)}::jsonb) + select e.id, e.queue, ${key}::text, ${this.sql.json(result)}::jsonb + from claimed_execution e on conflict (execution_id, key) do nothing `; } - buildClearWaitingState({ executionId }: ClearWaitingStateArgs): PendingQuery> { - return this.sql` - with child_info as ( - select - e.waiting_on_execution_id as child_id, - c.locked_by as child_locked_by + buildClearWaitingState({ + executionId, + queue, + orchestratorId, + claimToken, + }: ClearWaitingStateArgs): PendingQuery> { + return this.sql>` + with claimed_parent as materialized ( + select e.id, e.queue, e.waiting_on_execution_id from pgconductor._private_executions e - left join pgconductor._private_executions c on c.id = e.waiting_on_execution_id where e.id = ${executionId}::uuid + and e.queue = ${queue}::text + and e.locked_by = ${orchestratorId}::uuid + and e.claim_token = ${claimToken}::uuid + for update + ), child_info as ( + select + p.waiting_on_execution_id as child_id, + c.locked_by as child_locked_by + from claimed_parent p + left join pgconductor._private_executions c on c.id = p.waiting_on_execution_id ), -- Fail pending (not locked) children immediately failed_pending_child as ( @@ -1095,12 +981,14 @@ export class QueryBuilder { failed_at = pgconductor._private_current_time(), last_error = 'Cancelled: parent timed out', locked_by = null, - locked_at = null + locked_at = null, + claim_token = null from child_info ci where e.id = ci.child_id and ci.child_locked_by is null -- not currently executing and e.completed_at is null and e.failed_at is null + returning e.id, e.queue ), -- Signal executing (locked) children to cancel signaled_executing_child as ( @@ -1111,17 +999,30 @@ export class QueryBuilder { and ci.child_locked_by is not null -- currently executing and e.completed_at is null and e.failed_at is null + returning e.id + ), + released_pending_child_owner as ( + delete from pgconductor._private_fifo_owners o + using failed_pending_child c + where o.execution_id = c.id and o.queue = c.queue + returning o.execution_id ), -- Always clear parent's waiting state cleared_parent as ( - update pgconductor._private_executions + update pgconductor._private_executions e set waiting_on_execution_id = null, waiting_step_key = null - where id = ${executionId}::uuid - returning id + from claimed_parent p + where e.id = p.id + and e.queue = p.queue + returning e.id ) select id from cleared_parent + union all + select id from failed_pending_child + union all + select id from signaled_executing_child `; } diff --git a/packages/pgconductor-js/src/task-context.ts b/packages/pgconductor-js/src/task-context.ts index 509febe..e6721a3 100644 --- a/packages/pgconductor-js/src/task-context.ts +++ b/packages/pgconductor-js/src/task-context.ts @@ -37,6 +37,7 @@ export type TaskAbortReasons = step_key: string; task: TaskIdentifier; payload: Payload | null; + group?: string | null; __pgconductorTaskAborted: true; }; @@ -84,6 +85,7 @@ export type TaskContextOptions = { type ScheduleOptions = { cron: string; priority?: number; + group?: string; }; // second argument for tasks @@ -147,6 +149,9 @@ export class TaskContext< const cached = await this.opts.db.loadStep( { executionId: this.opts.execution.id, + queue: this.opts.execution.queue, + orchestratorId: this.opts.execution.locked_by, + claimToken: this.opts.execution.claim_token, key: name, }, { signal: this.signal }, @@ -163,6 +168,8 @@ export class TaskContext< { executionId: this.opts.execution.id, queue: this.opts.execution.queue, + orchestratorId: this.opts.execution.locked_by, + claimToken: this.opts.execution.claim_token, key: name, result: { result: result as JsonValue }, runAtMs: undefined, @@ -201,6 +208,9 @@ export class TaskContext< const cached = await this.opts.db.loadStep( { executionId: this.opts.execution.id, + queue: this.opts.execution.queue, + orchestratorId: this.opts.execution.locked_by, + claimToken: this.opts.execution.claim_token, key: id, }, { signal: this.signal }, @@ -230,10 +240,14 @@ export class TaskContext< task: TaskIdentifier, payload: InferPayload = {} as InferPayload, timeout?: number, + group?: string, ): Promise> { const cached = await this.opts.db.loadStep( { executionId: this.opts.execution.id, + queue: this.opts.execution.queue, + orchestratorId: this.opts.execution.locked_by, + claimToken: this.opts.execution.claim_token, key, }, { signal: this.signal }, @@ -251,6 +265,9 @@ export class TaskContext< await this.opts.db.clearWaitingState( { executionId: this.opts.execution.id, + queue: this.opts.execution.queue, + orchestratorId: this.opts.execution.locked_by, + claimToken: this.opts.execution.claim_token, }, { signal: this.signal }, ); @@ -266,6 +283,7 @@ export class TaskContext< task, step_key: key, payload, + group, }); } @@ -308,6 +326,7 @@ export class TaskContext< run_at: nextTimestamp, cron_expression: options.cron, priority: options.priority || null, + group: options.group || null, }, scheduleName, }, diff --git a/packages/pgconductor-js/src/task-definition.ts b/packages/pgconductor-js/src/task-definition.ts index f847e1b..cf7257b 100644 --- a/packages/pgconductor-js/src/task-definition.ts +++ b/packages/pgconductor-js/src/task-definition.ts @@ -118,7 +118,7 @@ export type FindTaskByIdentifier< // Trigger types export type InvocableTrigger = { invocable: true }; -export type CronTrigger = { cron: string; name: string }; +export type CronTrigger = { cron: string; name: string; group?: string }; // Event trigger - triggers when a custom event is emitted export type CustomEventTrigger< diff --git a/packages/pgconductor-js/src/task.ts b/packages/pgconductor-js/src/task.ts index 22973e5..2dec307 100644 --- a/packages/pgconductor-js/src/task.ts +++ b/packages/pgconductor-js/src/task.ts @@ -37,12 +37,29 @@ export type TaskConfiguration< window?: [string, string]; removeOnComplete?: RetentionSettings; removeOnFail?: RetentionSettings; + fifo?: boolean; concurrency?: number; + groupConcurrency?: number; batch?: BatchConfig; }; +/** Configuration validation used by the public task factory. */ +export type ValidateTaskConfiguration = T extends { readonly fifo: true } + ? T extends { readonly concurrency: number } | { readonly groupConcurrency: number } + ? "fifo cannot be combined with concurrency or groupConcurrency" + : unknown + : unknown; + export type RetentionSettings = boolean | { days: number }; +function validateConcurrency(value: number | undefined, name: string): number | undefined { + if (value === undefined) return undefined; + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return value; +} + export type TaskEvent

= | { name: "pgconductor.cron" } | { name: "pgconductor.invoke"; payload: P }; @@ -191,7 +208,9 @@ export class Task< public readonly window?: [string, string]; public readonly removeOnComplete: RetentionSettings; public readonly removeOnFail: RetentionSettings; + public readonly fifo?: boolean; public readonly concurrency?: number; + public readonly groupConcurrency?: number; public readonly batch?: BatchConfig; public readonly triggers: NonEmptyArray; @@ -209,7 +228,15 @@ export class Task< this.window = config.window; this.removeOnComplete = config.removeOnComplete ?? false; this.removeOnFail = config.removeOnFail ?? false; - this.concurrency = config.concurrency; + if ( + config.fifo && + (config.concurrency !== undefined || config.groupConcurrency !== undefined) + ) { + throw new Error("fifo cannot be combined with concurrency or groupConcurrency"); + } + this.fifo = config.fifo; + this.concurrency = validateConcurrency(config.concurrency, "concurrency"); + this.groupConcurrency = validateConcurrency(config.groupConcurrency, "groupConcurrency"); this.batch = config.batch; this.triggers = Array.isArray(triggers) ? triggers : [triggers]; diff --git a/packages/pgconductor-js/src/worker.ts b/packages/pgconductor-js/src/worker.ts index d18a18d..7c31f09 100644 --- a/packages/pgconductor-js/src/worker.ts +++ b/packages/pgconductor-js/src/worker.ts @@ -58,6 +58,7 @@ export const DEFAULT_WORKER_CONFIG: WorkerConfig = { * Encapsulates buffered execution results with internal counting and task key tracking. */ class BufferState { + orchestratorId = ""; completed: ExecutionCompleted[] = []; failed: (ExecutionFailed | ExecutionPermamentlyFailed)[] = []; released: ExecutionReleased[] = []; @@ -66,6 +67,7 @@ class BufferState { count = 0; add(result: ExecutionResult): void { + this.orchestratorId = result.orchestrator_id || this.orchestratorId; this.taskKeys.add(result.task_key); this.count++; @@ -100,6 +102,7 @@ class BufferState { this.failed.push(...other.failed); this.released.push(...other.released); this.invokeChild.push(...other.invokeChild); + this.orchestratorId = this.orchestratorId || other.orchestratorId; this.count += other.count; for (const key of other.taskKeys) { this.taskKeys.add(key); @@ -308,14 +311,16 @@ export class Worker< removeOnCompleteDays: retentionToDays(task.removeOnComplete), removeOnFailDays: retentionToDays(task.removeOnFail), window: task.window, + fifo: task.fifo, concurrency: task.concurrency, + groupConcurrency: task.groupConcurrency, })); const allTasks = Array.from(this.tasks.values()); const cronSchedules: ExecutionSpec[] = allTasks.flatMap((task) => task.triggers - .filter((t): t is { cron: string; name: string } => "cron" in t) + .filter((t): t is { cron: string; name: string; group?: string } => "cron" in t) .map((trigger) => { const interval = CronExpressionParser.parse(trigger.cron); const nextTimestamp = interval.next().toDate(); @@ -326,6 +331,7 @@ export class Worker< run_at: nextTimestamp, dedupe_key: `scheduled::${trigger.name}::${timestampSeconds}`, cron_expression: trigger.cron, + group: trigger.group || null, }; }), ); @@ -392,10 +398,6 @@ export class Worker< for (const task of allTasks) { taskMaxAttempts[task.name] = task.maxAttempts || 3; } - const taskKeysWithConcurrency = allTasks - .filter((task) => task.concurrency != null) - .map((task) => task.name); - // Check if any tasks have windows - only then do we need time-based filtering const tasksWithWindows = allTasks.filter((task) => task.window); @@ -436,7 +438,6 @@ export class Worker< queueName: this.queueName, batchSize: this.fetchBatchSize, filterTaskKeys: disallowedTaskKeys, - taskKeysWithConcurrency, }, { signal: this.signal }, ); @@ -476,10 +477,11 @@ export class Worker< return executions.map((exec) => ({ queue: exec.queue, execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, task_key: taskKey, status: "failed", error: `Task not found: ${taskKey}`, - slot_group_number: exec.slot_group_number, })) as ExecutionResult[]; } @@ -489,11 +491,12 @@ export class Worker< // All cancelled - return failures for all return executions.map((exec) => ({ execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, queue: exec.queue, task_key: taskKey, status: "permanently_failed", error: exec.last_error || "Execution was cancelled", - slot_group_number: exec.slot_group_number, })) as ExecutionResult[]; } @@ -589,6 +592,8 @@ export class Worker< execution: exec, logger: makeChildLogger(this.logger, { execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, task_key: exec.task_key, queue: exec.queue, }), @@ -605,6 +610,8 @@ export class Worker< case "child-invocation": return { execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, queue: exec.queue, task_key: exec.task_key, status: "invoke_child", @@ -613,27 +620,29 @@ export class Worker< child_task_name: output.task.name, child_task_queue: output.task.queue || "default", child_payload: output.payload, - slot_group_number: exec.slot_group_number, + group: output.group, } as const; case "cancelled": return { execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, queue: exec.queue, task_key: exec.task_key, status: "permanently_failed", error: exec.last_error || "Task was cancelled", - slot_group_number: exec.slot_group_number, } as const; case "released": case "parent-aborted": return { execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, queue: exec.queue, reschedule_in_ms: output.reason === "released" ? output.reschedule_in_ms : undefined, step_key: output.reason === "released" ? output.step_key : undefined, task_key: exec.task_key, status: "released", - slot_group_number: exec.slot_group_number, } as const; default: assert.never(output); @@ -642,20 +651,22 @@ export class Worker< return { execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, queue: exec.queue, task_key: exec.task_key, status: "completed", result: output, - slot_group_number: exec.slot_group_number, } as const; } catch (err) { return { execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, queue: exec.queue, task_key: exec.task_key, status: "failed", error: coerceError(err).message, - slot_group_number: exec.slot_group_number, } as const; } finally { // Clean up running task tracking @@ -725,23 +736,25 @@ export class Worker< // Batch sleep - reschedule all return executions.map((exec) => ({ execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, queue: exec.queue, task_key: taskKey, status: "released" as const, reschedule_in_ms: result.reschedule_in_ms, step_key: result.step_key, - slot_group_number: exec.slot_group_number, })); } // Other abort reasons return executions.map((exec) => ({ execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, queue: exec.queue, task_key: taskKey, status: "failed" as const, error: `Task aborted: ${result.reason}`, - slot_group_number: exec.slot_group_number, })); } @@ -749,11 +762,12 @@ export class Worker< if (result === undefined) { return executions.map((exec) => ({ execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, queue: exec.queue, task_key: taskKey, status: "completed" as const, result: undefined, - slot_group_number: exec.slot_group_number, })); } @@ -771,22 +785,24 @@ export class Worker< // Individual results return executions.map((exec, i) => ({ execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, queue: exec.queue, task_key: taskKey, status: "completed" as const, result: result[i], - slot_group_number: exec.slot_group_number, })); } catch (err) { // Handler threw: all fail together const errorMsg = coerceError(err).message; return executions.map((exec) => ({ execution_id: exec.id, + orchestrator_id: exec.locked_by, + claim_token: exec.claim_token, queue: exec.queue, task_key: taskKey, status: "failed" as const, error: errorMsg, - slot_group_number: exec.slot_group_number, })); } } @@ -819,6 +835,7 @@ export class Worker< run_at: nextTimestamp, dedupe_key: nextDedupeKey, cron_expression: execution.cron_expression, + group: execution.group || null, }, { signal: this.signal }, ); @@ -841,6 +858,7 @@ export class Worker< } try { + batch.orchestratorId = this.orchestratorId || batch.orchestratorId; await this.db.returnExecutions(batch, { signal: this.signal }); } catch (err) { this.logger.error("Error flushing results:", err); diff --git a/packages/pgconductor-js/tests/integration/concurrency.test.ts b/packages/pgconductor-js/tests/integration/concurrency.test.ts index 9446a83..1833b28 100644 --- a/packages/pgconductor-js/tests/integration/concurrency.test.ts +++ b/packages/pgconductor-js/tests/integration/concurrency.test.ts @@ -216,68 +216,6 @@ describe("Task-Level Concurrency", () => { await orchestrator.stopped; }, 30000); - test("high concurrency uses slot groups correctly", async () => { - const db = await pool.child(); - databases.push(db); - - const taskDef = defineTask({ name: "high-concurrency-task" }); - - const conductor = Conductor.create({ - sql: db.sql, - tasks: TaskSchemas.fromSchema([taskDef]), - context: {}, - }); - - let completed = 0; - - const highConcurrencyTask = conductor.createTask( - { name: "high-concurrency-task", concurrency: 500 }, - { invocable: true }, - async () => { - completed++; - }, - ); - - const orchestrator = Orchestrator.create({ - conductor, - tasks: [highConcurrencyTask], - defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50, concurrency: 50 }, - }); - - await orchestrator.start(); - - // queue 100 tasks - for (let i = 0; i < 100; i++) { - await conductor.invoke({ name: "high-concurrency-task" }, {}); - } - - // wait for completion - await new Promise((r) => setTimeout(r, 5000)); - - await orchestrator.stop(); - await orchestrator.stopped; - - // extra wait to ensure all flushes complete - await new Promise((r) => setTimeout(r, 1000)); - - // all tasks should complete - expect(completed).toBe(100); - - // verify slots were created (one row per slot, concurrency=500 → 500 rows) - const slots = await db.sql` - select * from pgconductor._private_concurrency_slots - where task_key = 'high-concurrency-task' - `; - - expect(slots.length).toBe(500); // one row per slot - expect(slots.every((s) => s.capacity === 1)).toBe(true); // each slot has capacity=1 - - // verify all slots are released (used = 0) - for (const slot of slots) { - expect(slot.used).toBe(0); - } - }, 30000); - test("mixed queue with concurrency and without", async () => { const db = await pool.child(); databases.push(db); diff --git a/packages/pgconductor-js/tests/integration/cron-scheduling.test.ts b/packages/pgconductor-js/tests/integration/cron-scheduling.test.ts index 6699181..409e7ab 100644 --- a/packages/pgconductor-js/tests/integration/cron-scheduling.test.ts +++ b/packages/pgconductor-js/tests/integration/cron-scheduling.test.ts @@ -7,6 +7,17 @@ import { TestDatabasePool } from "../fixtures/test-database"; import { waitFor } from "../../src/lib/wait-for"; import { TaskSchemas } from "../../src/schemas"; +async function waitForCondition( + condition: () => boolean | Promise, + timeoutMs = 5000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (!(await condition())) { + if (Date.now() >= deadline) throw new Error("condition was not met before timeout"); + await waitFor(50); + } +} + describe("Cron Scheduling", () => { let pool: TestDatabasePool; @@ -101,26 +112,20 @@ describe("Cron Scheduling", () => { await orchestrator.start(); - // Wait for first execution - await waitFor(4000); - expect(executions.mock.calls.length).toBeGreaterThanOrEqual(1); - - // Check that next execution is scheduled - const schedules = await db.sql>` - SELECT dedupe_key, run_at - FROM pgconductor._private_executions - WHERE task_key = 'frequent-sync' - AND dedupe_key LIKE 'scheduled::%' - AND run_at > pgconductor._private_current_time() - ORDER BY run_at - LIMIT 1 - `; + await waitForCondition(() => executions.mock.calls.length >= 1, 7000); - expect(schedules.length).toBe(1); + // A completed cron execution durably creates a distinct next occurrence. + await waitForCondition(async () => { + const schedules = await db.sql>` + SELECT count(*)::integer AS count + FROM pgconductor._private_executions + WHERE task_key = 'frequent-sync' + AND dedupe_key LIKE 'scheduled::%' + `; + return (schedules[0]?.count ?? 0) >= 2; + }, 7000); - // Wait for second execution - await waitFor(4000); - expect(executions.mock.calls.length).toBeGreaterThanOrEqual(2); + await waitForCondition(() => executions.mock.calls.length >= 2, 7000); await orchestrator.stop(); await db.destroy(); @@ -361,21 +366,23 @@ describe("Cron Scheduling", () => { await orchestrator.start(); await conductor.invoke({ name: "dynamic-scheduler" }, {}); - await waitFor(4000); - expect(targetExecutions.mock.calls.length).toBeGreaterThanOrEqual(1); - - const nextSchedules = await db.sql>` - SELECT dedupe_key - FROM pgconductor._private_executions - WHERE task_key = 'dynamic-target' - AND cron_expression IS NOT NULL - AND run_at > pgconductor._private_current_time() - ORDER BY run_at - LIMIT 1 - `; - - expect(nextSchedules.length).toBe(1); - expect(nextSchedules[0]!.dedupe_key).toMatch(/^scheduled::reporting::\d+$/); + await waitForCondition(() => targetExecutions.mock.calls.length >= 1, 20_000); + + let nextSchedule: { dedupe_key: string } | undefined; + await waitForCondition(async () => { + [nextSchedule] = await db.sql>` + SELECT dedupe_key + FROM pgconductor._private_executions + WHERE task_key = 'dynamic-target' + AND cron_expression IS NOT NULL + AND run_at > pgconductor._private_current_time() + ORDER BY run_at + LIMIT 1 + `; + return nextSchedule !== undefined; + }, 20_000); + + expect(nextSchedule?.dedupe_key).toMatch(/^scheduled::reporting::\d+$/); await orchestrator.stop(); await db.destroy(); @@ -424,11 +431,13 @@ describe("Cron Scheduling", () => { }, ); + let unscheduled = false; const unschedulerTask = conductor.createTask( { name: "dynamic-unscheduler" }, { invocable: true }, async (_event, ctx) => { await ctx.unschedule({ name: "dynamic-target" }, "reporting"); + unscheduled = true; }, ); @@ -443,12 +452,22 @@ describe("Cron Scheduling", () => { await orchestrator.start(); await conductor.invoke({ name: "dynamic-scheduler" }, {}); - await waitFor(4000); - expect(targetExecutions.mock.calls.length).toBeGreaterThanOrEqual(1); + await waitForCondition(() => targetExecutions.mock.calls.length >= 1, 7000); + await waitForCondition(async () => { + const active = await db.sql>` + SELECT count(*)::integer AS count + FROM pgconductor._private_executions + WHERE task_key = 'dynamic-target' + AND locked_at IS NOT NULL + AND completed_at IS NULL + AND failed_at IS NULL + `; + return (active[0]?.count ?? 0) === 0; + }); const runsBeforeUnschedule = targetExecutions.mock.calls.length; await conductor.invoke({ name: "dynamic-unscheduler" }, {}); - await waitFor(2000); + await waitForCondition(() => unscheduled); const futureSchedules = await db.sql>` SELECT id @@ -558,18 +577,21 @@ describe("Cron Scheduling", () => { // Should have at least attempted twice (fail + success) expect(attemptCount).toBeGreaterThanOrEqual(2); - // Verify next cron execution is scheduled - const nextExecution = await db.sql>` - SELECT run_at, dedupe_key - FROM pgconductor._private_executions - WHERE task_key = 'flaky-cron' - AND dedupe_key LIKE 'scheduled::%' - AND run_at > pgconductor._private_current_time() - ORDER BY run_at - LIMIT 1 - `; + // Verify next cron execution is scheduled without sampling at a claim boundary. + let nextExecution: Array<{ run_at: Date; dedupe_key: string }> = []; + await waitForCondition(async () => { + nextExecution = await db.sql>` + SELECT run_at, dedupe_key + FROM pgconductor._private_executions + WHERE task_key = 'flaky-cron' + AND dedupe_key LIKE 'scheduled::%' + AND run_at > pgconductor._private_current_time() + ORDER BY run_at + LIMIT 1 + `; + return nextExecution.length === 1; + }); - expect(nextExecution.length).toBe(1); expect(nextExecution[0]?.dedupe_key).toMatch(/^scheduled::.*::\d+$/); await orchestrator.stop(); diff --git a/packages/pgconductor-js/tests/integration/execution-foundations.test.ts b/packages/pgconductor-js/tests/integration/execution-foundations.test.ts new file mode 100644 index 0000000..81a3469 --- /dev/null +++ b/packages/pgconductor-js/tests/integration/execution-foundations.test.ts @@ -0,0 +1,581 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import { Conductor } from "../../src/conductor"; +import { TestDatabasePool, type TestDatabase } from "../fixtures/test-database"; + +describe("execution foundations", () => { + let pool: TestDatabasePool; + const databases: TestDatabase[] = []; + + beforeAll(async () => { + pool = await TestDatabasePool.create(); + }, 60000); + + afterEach(async () => { + await Promise.all(databases.map((database) => database.destroy())); + databases.length = 0; + }); + + afterAll(async () => { + await pool?.destroy(); + }); + + async function database(): Promise { + const database = await pool.child(); + databases.push(database); + const conductor = Conductor.create({ sql: database.sql, context: {} }); + await conductor.ensureInstalled(); + return database; + } + + function grouped( + result: Parameters[0]["completed"][number], + ) { + return { + count: 1, + orchestratorId: result.orchestrator_id, + completed: [result], + failed: [], + released: [], + invokeChild: [], + taskKeys: new Set([result.task_key]), + }; + } + + test("registers and executes the same task key independently in two queues", async () => { + const db = await database(); + + await db.client.registerWorker({ + queueName: "queue-a", + taskSpecs: [ + { + key: "same-task", + queue: "queue-a", + maxAttempts: 2, + removeOnCompleteDays: 0, + }, + ], + cronSchedules: [], + eventSubscriptions: [], + }); + await db.client.registerWorker({ + queueName: "queue-b", + taskSpecs: [ + { + key: "same-task", + queue: "queue-b", + maxAttempts: 7, + removeOnCompleteDays: 1, + }, + ], + cronSchedules: [], + eventSubscriptions: [], + }); + + const tasks = await db.sql< + { queue: string; key: string; max_attempts: number; remove_on_complete_days: number | null }[] + >` + select queue, key, max_attempts, remove_on_complete_days + from pgconductor._private_tasks + where key = 'same-task' + order by queue + `; + expect([...tasks]).toEqual([ + { queue: "queue-a", key: "same-task", max_attempts: 2, remove_on_complete_days: 0 }, + { queue: "queue-b", key: "same-task", max_attempts: 7, remove_on_complete_days: 1 }, + ]); + + const firstId = await db.client.invoke({ task_key: "same-task", queue: "queue-a" }); + const secondId = await db.client.invoke({ task_key: "same-task", queue: "queue-b" }); + expect(firstId).not.toBeNull(); + expect(secondId).not.toBeNull(); + + const first = ( + await db.client.getExecutions({ + orchestratorId: crypto.randomUUID(), + queueName: "queue-a", + batchSize: 1, + filterTaskKeys: [], + }) + )[0]; + const second = ( + await db.client.getExecutions({ + orchestratorId: crypto.randomUUID(), + queueName: "queue-b", + batchSize: 1, + filterTaskKeys: [], + }) + )[0]; + expect(first?.queue).toBe("queue-a"); + expect(second?.queue).toBe("queue-b"); + if (!first || !second) throw new Error("expected both executions to be claimed"); + + await db.client.returnExecutions( + grouped({ + execution_id: first.id, + queue: first.queue, + task_key: first.task_key, + orchestrator_id: first.locked_by, + claim_token: first.claim_token, + status: "completed", + }), + ); + await db.client.returnExecutions( + grouped({ + execution_id: second.id, + queue: second.queue, + task_key: second.task_key, + orchestrator_id: second.locked_by, + claim_token: second.claim_token, + status: "completed", + }), + ); + + const remaining = await db.sql<{ queue: string; completed_at: Date | null }[]>` + select queue, completed_at + from pgconductor._private_executions + where task_key = 'same-task' + order by queue + `; + expect([...remaining]).toHaveLength(1); + expect(remaining[0]?.queue).toBe("queue-b"); + expect(remaining[0]?.completed_at).not.toBeNull(); + }); + + test("retains a parent when its permanently failed child is configured for removal", async () => { + const db = await database(); + await db.client.registerWorker({ + queueName: "parent-retention", + taskSpecs: [ + { key: "parent", queue: "parent-retention", removeOnFailDays: 1 }, + { key: "child", queue: "parent-retention", maxAttempts: 1, removeOnFailDays: 0 }, + ], + cronSchedules: [], + eventSubscriptions: [], + }); + const parentId = await db.client.invoke({ task_key: "parent", queue: "parent-retention" }); + if (!parentId) throw new Error("expected parent execution"); + const parent = ( + await db.client.getExecutions({ + orchestratorId: crypto.randomUUID(), + queueName: "parent-retention", + batchSize: 1, + filterTaskKeys: [], + }) + )[0]; + if (!parent) throw new Error("expected parent claim"); + + await db.client.returnExecutions({ + count: 1, + orchestratorId: parent.locked_by, + completed: [], + failed: [], + released: [], + invokeChild: [ + { + execution_id: parent.id, + queue: parent.queue, + orchestrator_id: parent.locked_by, + claim_token: parent.claim_token, + task_key: parent.task_key, + status: "invoke_child", + timeout_ms: 5000, + step_key: "child-step", + child_task_name: "child", + child_task_queue: "parent-retention", + child_payload: null, + }, + ], + taskKeys: new Set([parent.task_key]), + }); + + const childId = ( + await db.sql<{ id: string }[]>` + select id from pgconductor._private_executions + where parent_execution_id = ${parentId}::uuid + ` + )[0]?.id; + if (!childId) throw new Error("expected child execution"); + const child = ( + await db.client.getExecutions({ + orchestratorId: crypto.randomUUID(), + queueName: "parent-retention", + batchSize: 1, + filterTaskKeys: [], + }) + )[0]; + if (!child) throw new Error("expected child claim"); + + await db.client.returnExecutions({ + count: 1, + orchestratorId: child.locked_by, + completed: [], + failed: [ + { + execution_id: child.id, + queue: child.queue, + orchestrator_id: child.locked_by, + claim_token: child.claim_token, + task_key: child.task_key, + status: "permanently_failed", + error: "child failed", + }, + ], + released: [], + invokeChild: [], + taskKeys: new Set([child.task_key]), + }); + + const retained = await db.sql< + { id: string; failed_at: Date | null; last_error: string | null }[] + >` + select id, failed_at, last_error from pgconductor._private_executions + where id = ${parentId}::uuid + `; + expect(retained[0]?.id).toBe(parentId); + expect(retained[0]?.failed_at).not.toBeNull(); + expect(retained[0]?.last_error).toContain("Child execution failed"); + expect( + await db.sql`select 1 from pgconductor._private_executions where id = ${childId}::uuid`, + ).toHaveLength(0); + }); + + test("propagates a permanently failed child across queues to its parent", async () => { + const db = await database(); + await db.client.registerWorker({ + queueName: "parent-queue", + taskSpecs: [{ key: "parent", queue: "parent-queue", removeOnFailDays: 1 }], + cronSchedules: [], + eventSubscriptions: [], + }); + await db.client.registerWorker({ + queueName: "child-queue", + taskSpecs: [{ key: "child", queue: "child-queue", maxAttempts: 1, removeOnFailDays: 1 }], + cronSchedules: [], + eventSubscriptions: [], + }); + + const parentId = await db.client.invoke({ task_key: "parent", queue: "parent-queue" }); + if (!parentId) throw new Error("expected parent execution"); + const parent = ( + await db.client.getExecutions({ + orchestratorId: crypto.randomUUID(), + queueName: "parent-queue", + batchSize: 1, + filterTaskKeys: [], + }) + )[0]; + if (!parent) throw new Error("expected parent claim"); + + await db.client.returnExecutions({ + count: 1, + orchestratorId: parent.locked_by, + completed: [], + failed: [], + released: [], + invokeChild: [ + { + execution_id: parent.id, + queue: parent.queue, + orchestrator_id: parent.locked_by, + claim_token: parent.claim_token, + task_key: parent.task_key, + status: "invoke_child", + timeout_ms: "infinity", + step_key: "child-step", + child_task_name: "child", + child_task_queue: "child-queue", + child_payload: null, + }, + ], + taskKeys: new Set([parent.task_key]), + }); + + const child = ( + await db.client.getExecutions({ + orchestratorId: crypto.randomUUID(), + queueName: "child-queue", + batchSize: 1, + filterTaskKeys: [], + }) + )[0]; + if (!child) throw new Error("expected child claim"); + await db.client.returnExecutions({ + count: 1, + orchestratorId: child.locked_by, + completed: [], + failed: [ + { + execution_id: child.id, + queue: child.queue, + orchestrator_id: child.locked_by, + claim_token: child.claim_token, + task_key: child.task_key, + status: "permanently_failed", + error: "child failed", + }, + ], + released: [], + invokeChild: [], + taskKeys: new Set([child.task_key]), + }); + + const outcome = await db.sql<{ failed_at: Date | null; last_error: string | null }[]>` + select failed_at, last_error + from pgconductor._private_executions + where id = ${parentId}::uuid + `; + expect(outcome[0]?.failed_at).not.toBeNull(); + expect(outcome[0]?.last_error).toBe("Child execution failed: child failed"); + }); + + test("cancellation fences a buffered completion and does not retry it", async () => { + const db = await database(); + await db.client.registerWorker({ + queueName: "cancel-buffered", + taskSpecs: [{ key: "cancelled", queue: "cancel-buffered", maxAttempts: 5 }], + cronSchedules: [], + eventSubscriptions: [], + }); + const orchestratorId = crypto.randomUUID(); + await db.client.orchestratorHeartbeat({ orchestratorId, version: "test", migrationNumber: 1 }); + const executionId = await db.client.invoke({ task_key: "cancelled", queue: "cancel-buffered" }); + if (!executionId) throw new Error("expected execution"); + const claimed = ( + await db.client.getExecutions({ + orchestratorId, + queueName: "cancel-buffered", + batchSize: 1, + filterTaskKeys: [], + }) + )[0]; + if (!claimed) throw new Error("expected claim"); + await db.client.cancelExecution(executionId, { reason: "cancelled before flush" }); + await db.client.returnExecutions( + grouped({ + execution_id: claimed.id, + queue: claimed.queue, + task_key: claimed.task_key, + orchestrator_id: claimed.locked_by, + claim_token: claimed.claim_token, + status: "completed", + }), + ); + + const outcome = await db.sql< + { + failed_at: Date | null; + completed_at: Date | null; + locked_by: string | null; + attempts: number; + last_error: string | null; + }[] + >` + select failed_at, completed_at, locked_by, attempts, last_error + from pgconductor._private_executions + where id = ${executionId}::uuid + `; + expect(outcome[0]?.failed_at).not.toBeNull(); + expect(outcome[0]?.completed_at).toBeNull(); + expect(outcome[0]?.locked_by).toBeNull(); + expect(outcome[0]?.attempts).toBe(1); + expect(outcome[0]?.last_error).toBe("cancelled before flush"); + }); + + test("cancelling a waiting parent permanently fails its pending child", async () => { + const db = await database(); + await db.client.registerWorker({ + queueName: "cascade-parent", + taskSpecs: [{ key: "parent", queue: "cascade-parent" }], + cronSchedules: [], + eventSubscriptions: [], + }); + await db.client.registerWorker({ + queueName: "cascade-child", + taskSpecs: [{ key: "child", queue: "cascade-child" }], + cronSchedules: [], + eventSubscriptions: [], + }); + const parentId = await db.client.invoke({ task_key: "parent", queue: "cascade-parent" }); + if (!parentId) throw new Error("expected parent execution"); + const parent = ( + await db.client.getExecutions({ + orchestratorId: crypto.randomUUID(), + queueName: "cascade-parent", + batchSize: 1, + filterTaskKeys: [], + }) + )[0]; + if (!parent) throw new Error("expected parent claim"); + await db.client.returnExecutions({ + count: 1, + orchestratorId: parent.locked_by, + completed: [], + failed: [], + released: [], + invokeChild: [ + { + execution_id: parent.id, + queue: parent.queue, + orchestrator_id: parent.locked_by, + claim_token: parent.claim_token, + task_key: parent.task_key, + status: "invoke_child", + timeout_ms: "infinity", + step_key: "child-step", + child_task_name: "child", + child_task_queue: "cascade-child", + child_payload: null, + }, + ], + taskKeys: new Set([parent.task_key]), + }); + const childId = ( + await db.sql<{ id: string }[]>` + select id from pgconductor._private_executions where parent_execution_id = ${parentId}::uuid + ` + )[0]?.id; + if (!childId) throw new Error("expected child execution"); + await db.client.cancelExecution(parentId); + const outcome = await db.sql< + { + id: string; + failed_at: Date | null; + waiting_on_execution_id: string | null; + }[] + >` + select id, failed_at, waiting_on_execution_id + from pgconductor._private_executions + where id in (${parentId}::uuid, ${childId}::uuid) + order by id + `; + expect(outcome).toHaveLength(2); + expect(outcome.every((execution) => execution.failed_at !== null)).toBe(true); + expect( + outcome.find((execution) => execution.id === parentId)?.waiting_on_execution_id, + ).toBeNull(); + }); + + test("uses enqueue position to break priority and run-at ties", async () => { + const db = await database(); + await db.client.registerWorker({ + queueName: "enqueue-order", + taskSpecs: [{ key: "ordered", queue: "enqueue-order" }], + cronSchedules: [], + eventSubscriptions: [], + }); + const ids = await db.client.invokeBatch([ + { task_key: "ordered", queue: "enqueue-order", priority: 0 }, + { task_key: "ordered", queue: "enqueue-order", priority: 0 }, + { task_key: "ordered", queue: "enqueue-order", priority: 0 }, + ]); + const claimed = await db.client.getExecutions({ + orchestratorId: crypto.randomUUID(), + queueName: "enqueue-order", + batchSize: 3, + filterTaskKeys: [], + }); + expect(claimed.map((execution) => execution.id)).toEqual(ids); + }); + + test("fences stale completion, failure, and release results after recovery and re-claim", async () => { + const db = await database(); + await db.client.registerWorker({ + queueName: "fenced", + taskSpecs: [{ key: "fenced-task", queue: "fenced", maxAttempts: 3 }], + cronSchedules: [], + eventSubscriptions: [], + }); + const executionId = await db.client.invoke({ task_key: "fenced-task", queue: "fenced" }); + if (!executionId) throw new Error("expected execution id"); + + const oldOrchestrator = crypto.randomUUID(); + await db.client.orchestratorHeartbeat({ + orchestratorId: oldOrchestrator, + version: "test", + migrationNumber: 1, + }); + const oldClaim = ( + await db.client.getExecutions({ + orchestratorId: oldOrchestrator, + queueName: "fenced", + batchSize: 1, + filterTaskKeys: [], + }) + )[0]; + if (!oldClaim) throw new Error("expected old claim"); + + await db.sql` + update pgconductor._private_orchestrators + set last_heartbeat_at = now() - interval '1 hour' + where id = ${oldOrchestrator}::uuid + `; + await db.client.recoverStaleOrchestrators({ maxAge: "1 second" }); + + const newOrchestrator = crypto.randomUUID(); + const currentClaim = ( + await db.client.getExecutions({ + orchestratorId: newOrchestrator, + queueName: "fenced", + batchSize: 1, + filterTaskKeys: [], + }) + )[0]; + if (!currentClaim) throw new Error("expected recovered execution to be re-claimed"); + expect(currentClaim.claim_token).not.toBe(oldClaim.claim_token); + + const staleBase = { + execution_id: executionId, + queue: "fenced", + task_key: "fenced-task", + orchestrator_id: oldClaim.locked_by, + claim_token: oldClaim.claim_token, + }; + await db.client.returnExecutions({ + count: 3, + orchestratorId: oldOrchestrator, + completed: [{ ...staleBase, status: "completed" }], + failed: [{ ...staleBase, status: "failed", error: "stale" }], + released: [{ ...staleBase, status: "released", reschedule_in_ms: 0 }], + invokeChild: [], + taskKeys: new Set(["fenced-task"]), + }); + + const untouched = await db.sql< + { + completed_at: Date | null; + failed_at: Date | null; + locked_by: string; + claim_token: string; + }[] + >` + select completed_at, failed_at, locked_by, claim_token + from pgconductor._private_executions + where id = ${executionId}::uuid + `; + expect(untouched[0]?.completed_at).toBeNull(); + expect(untouched[0]?.failed_at).toBeNull(); + expect(untouched[0]?.locked_by).toBe(newOrchestrator); + expect(untouched[0]?.claim_token).toBe(currentClaim.claim_token); + + await db.client.returnExecutions( + grouped({ + execution_id: currentClaim.id, + queue: currentClaim.queue, + task_key: currentClaim.task_key, + orchestrator_id: currentClaim.locked_by, + claim_token: currentClaim.claim_token, + status: "completed", + }), + ); + const settled = await db.sql< + { completed_at: Date | null; locked_by: string | null; claim_token: string | null }[] + >` + select completed_at, locked_by, claim_token + from pgconductor._private_executions + where id = ${executionId}::uuid + `; + expect(settled[0]?.completed_at).not.toBeNull(); + expect(settled[0]?.locked_by).toBeNull(); + expect(settled[0]?.claim_token).toBeNull(); + }); +}); diff --git a/packages/pgconductor-js/tests/integration/field-selection-types.test.ts b/packages/pgconductor-js/tests/integration/field-selection-types.test.ts index 717e994..709bd42 100644 --- a/packages/pgconductor-js/tests/integration/field-selection-types.test.ts +++ b/packages/pgconductor-js/tests/integration/field-selection-types.test.ts @@ -7,6 +7,7 @@ import { defineEvent } from "../../src/event-definition"; import { TaskSchemas, EventSchemas } from "../../src/schemas"; import { TestDatabasePool } from "../fixtures/test-database"; import type { TestDatabase } from "../fixtures/test-database"; +import { Deferred } from "../../src/lib/deferred"; describe("Field Selection - Type Safety & Runtime", () => { let pool: TestDatabasePool; @@ -48,6 +49,7 @@ describe("Field Selection - Type Safety & Runtime", () => { payload: z.object({}), }); + const handled = new Deferred(); const taskFn = mock(async (event) => { // Type-level: TypeScript should know these fields exist with correct types const str: string = event.payload.stringField; @@ -70,6 +72,7 @@ describe("Field Selection - Type Safety & Runtime", () => { // Non-selected fields should not exist at runtime expect(event.payload.extraString).toBeUndefined(); expect(event.payload.extraNumber).toBeUndefined(); + handled.resolve(); }); const conductor = Conductor.create({ @@ -107,9 +110,7 @@ describe("Field Selection - Type Safety & Runtime", () => { extraNumber: 999, }); - // Wait for task to execute - await new Promise((r) => setTimeout(r, 300)); - + await handled.promise; expect(taskFn).toHaveBeenCalledTimes(1); await orchestrator.stop(); @@ -135,6 +136,7 @@ describe("Field Selection - Type Safety & Runtime", () => { payload: z.object({}), }); + const handled = new Deferred(); const taskFn = mock(async (event) => { // Type-level: Selected fields with correct camelCase const userId: string = event.payload.userId; @@ -146,6 +148,7 @@ describe("Field Selection - Type Safety & Runtime", () => { expect(event.payload.firstName).toBeUndefined(); expect(event.payload.lastName).toBeUndefined(); expect(event.payload.accountType).toBeUndefined(); + handled.resolve(); }); const conductor = Conductor.create({ @@ -177,8 +180,7 @@ describe("Field Selection - Type Safety & Runtime", () => { accountType: "premium", }); - await new Promise((r) => setTimeout(r, 300)); - + await handled.promise; expect(taskFn).toHaveBeenCalledTimes(1); await orchestrator.stop(); @@ -202,6 +204,7 @@ describe("Field Selection - Type Safety & Runtime", () => { payload: z.object({}), }); + const handled = new Deferred(); const taskFn = mock(async (event) => { // Type-level: All fields should be available const f1: string = event.payload.field1; @@ -212,6 +215,7 @@ describe("Field Selection - Type Safety & Runtime", () => { expect(event.payload.field1).toBe("value1"); expect(event.payload.field2).toBe(123); expect(event.payload.field3).toBe(true); + handled.resolve(); }); const conductor = Conductor.create({ @@ -238,8 +242,7 @@ describe("Field Selection - Type Safety & Runtime", () => { field3: true, }); - await new Promise((r) => setTimeout(r, 300)); - + await handled.promise; expect(taskFn).toHaveBeenCalledTimes(1); await orchestrator.stop(); diff --git a/packages/pgconductor-js/tests/integration/fifo.test.ts b/packages/pgconductor-js/tests/integration/fifo.test.ts new file mode 100644 index 0000000..3ae8929 --- /dev/null +++ b/packages/pgconductor-js/tests/integration/fifo.test.ts @@ -0,0 +1,613 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import { z } from "zod"; +import { Conductor } from "../../src/conductor"; +import { Orchestrator } from "../../src/orchestrator"; +import { TaskSchemas } from "../../src/schemas"; +import { defineTask } from "../../src/task-definition"; +import { Deferred } from "../../src/lib/deferred"; +import { TestDatabasePool, type TestDatabase } from "../fixtures/test-database"; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function eventually(check: () => Promise, timeoutMs = 10_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await check()) return; + await sleep(10); + } + throw new Error("condition was not met before the test timeout"); +} + +async function owner(db: TestDatabase, taskKey: string, queue = "default") { + const rows = await db.sql<{ execution_id: string }[]>` + select execution_id + from pgconductor._private_fifo_owners + where task_key = ${taskKey} and queue = ${queue} + `; + return rows[0]?.execution_id; +} + +const payloadDefinition = (name: Name) => + defineTask({ name, payload: z.object({ id: z.number() }) }); + +describe("FIFO task execution", () => { + let pool: TestDatabasePool; + const databases: TestDatabase[] = []; + + beforeAll(async () => { + pool = await TestDatabasePool.create(); + }, 60_000); + + afterEach(async () => { + await Promise.all(databases.map((db) => db.destroy())); + databases.length = 0; + }); + + afterAll(async () => { + await pool?.destroy(); + }); + + test("executes runnable work strictly in enqueue order with concurrent workers", async () => { + const db = await pool.child(); + databases.push(db); + const definition = payloadDefinition("fifo-order"); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const order: number[] = []; + const starts = Array.from({ length: 5 }, () => new Deferred()); + const releases = Array.from({ length: 5 }, () => new Deferred()); + const task = conductor.createTask( + { name: "fifo-order", fifo: true }, + { invocable: true }, + async (event) => { + const id = event.payload.id; + order.push(id); + starts[id]?.resolve(); + await releases[id]!.promise; + }, + ); + await conductor.ensureInstalled(); + for (let id = 0; id < 5; id++) await conductor.invoke({ name: "fifo-order" }, { id }); + + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: { concurrency: 5, fetchBatchSize: 2, pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await orchestrator.start(); + await starts[0]!.promise; + expect(order).toEqual([0]); + for (let id = 0; id < 5; id++) { + releases[id]!.resolve(); + if (id < 4) { + await starts[id + 1]!.promise; + expect(order).toEqual(Array.from({ length: id + 2 }, (_, i) => i)); + } + } + await orchestrator.stop(); + }, 30_000); + + test("does not let a future, never-started execution block ready work", async () => { + const db = await pool.child(); + databases.push(db); + const definition = payloadDefinition("fifo-future"); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const readyStarted = new Deferred(); + const readyRelease = new Deferred(); + const order: number[] = []; + const task = conductor.createTask( + { name: "fifo-future", fifo: true }, + { invocable: true }, + async (event) => { + order.push(event.payload.id); + if (event.payload.id === 2) { + readyStarted.resolve(); + await readyRelease.promise; + } + }, + ); + await conductor.ensureInstalled(); + const future = await conductor.invoke( + { name: "fifo-future" }, + { id: 1 }, + { run_at: new Date(Date.now() + 3_600_000) }, + ); + await conductor.invoke({ name: "fifo-future" }, { id: 2 }); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: { concurrency: 3, pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await orchestrator.start(); + await readyStarted.promise; + expect(order).toEqual([2]); + expect(await owner(db, "fifo-future")).not.toBe(future); + readyRelease.resolve(); + await orchestrator.stop(); + }, 30_000); + + test("ignores priority within a FIFO lane", async () => { + const db = await pool.child(); + databases.push(db); + const definition = payloadDefinition("fifo-priority"); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const starts = [new Deferred(), new Deferred()]; + const releases = [new Deferred(), new Deferred()]; + const order: number[] = []; + const task = conductor.createTask( + { name: "fifo-priority", fifo: true }, + { invocable: true }, + async (event) => { + const id = event.payload.id; + order.push(id); + starts[id]!.resolve(); + await releases[id]!.promise; + }, + ); + await conductor.ensureInstalled(); + await conductor.invoke({ name: "fifo-priority" }, { id: 0 }, { priority: 100 }); + await conductor.invoke({ name: "fifo-priority" }, { id: 1 }, { priority: -100 }); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: { concurrency: 2, pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await orchestrator.start(); + await starts[0]!.promise; + expect(order).toEqual([0]); + releases[0]!.resolve(); + await starts[1]!.promise; + expect(order).toEqual([0, 1]); + releases[1]!.resolve(); + await orchestrator.stop(); + }, 30_000); + + test("retains its owner over retry backoff", async () => { + const db = await pool.child(); + databases.push(db); + const definition = payloadDefinition("fifo-retry"); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const firstFailed = new Deferred(); + const retryStarted = new Deferred(); + const retryRelease = new Deferred(); + const successorStarted = new Deferred(); + let calls = 0; + const task = conductor.createTask( + { name: "fifo-retry", fifo: true }, + { invocable: true }, + async (event) => { + if (event.payload.id === 0) { + if (++calls === 1) { + firstFailed.resolve(); + throw new Error("retry me"); + } + retryStarted.resolve(); + await retryRelease.promise; + } else successorStarted.resolve(); + }, + ); + await conductor.ensureInstalled(); + const now = new Date("2024-01-01T00:00:00Z"); + await db.client.setFakeTime({ date: now }); + const first = await conductor.invoke({ name: "fifo-retry" }, { id: 0 }); + const successor = await conductor.invoke({ name: "fifo-retry" }, { id: 1 }); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: { concurrency: 2, pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await orchestrator.start(); + await firstFailed.promise; + await eventually( + async () => + (await owner(db, "fifo-retry")) === first && + ( + await db.sql< + { locked_at: Date | null }[] + >`select locked_at from pgconductor._private_executions where id = ${first}::uuid` + )[0]?.locked_at === null, + ); + expect(successorStarted.isSettled).toBe(false); + const retryAt = ( + await db.sql< + { run_at: Date }[] + >`select run_at from pgconductor._private_executions where id = ${first}::uuid` + )[0]!.run_at; + await db.client.setFakeTime({ date: new Date(retryAt.getTime() + 1) }); + await retryStarted.promise; + expect(await owner(db, "fifo-retry")).toBe(first); + retryRelease.resolve(); + await successorStarted.promise; + expect(await owner(db, "fifo-retry")).toBe(successor); + await orchestrator.stop(); + }, 30_000); + + test("retains its owner over ctx.sleep", async () => { + const db = await pool.child(); + databases.push(db); + const definition = payloadDefinition("fifo-sleep"); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const sleeping = new Deferred(); + const successorStarted = new Deferred(); + const task = conductor.createTask( + { name: "fifo-sleep", fifo: true }, + { invocable: true }, + async (event, ctx) => { + if (event.payload.id === 0) { + sleeping.resolve(); + await ctx.sleep("wait", 3_600_000); + } else successorStarted.resolve(); + }, + ); + await conductor.ensureInstalled(); + await db.client.setFakeTime({ date: new Date("2024-01-01T00:00:00Z") }); + const first = await conductor.invoke({ name: "fifo-sleep" }, { id: 0 }); + const successor = await conductor.invoke({ name: "fifo-sleep" }, { id: 1 }); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: { concurrency: 2, pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await orchestrator.start(); + await sleeping.promise; + await eventually(async () => (await owner(db, "fifo-sleep")) === first); + expect(successorStarted.isSettled).toBe(false); + expect(await owner(db, "fifo-sleep")).toBe(first); + await orchestrator.stop(); + }, 30_000); + + test("retains its owner while waiting on a child", async () => { + const db = await pool.child(); + databases.push(db); + const parentDefinition = defineTask({ name: "fifo-parent" }); + const childDefinition = defineTask({ name: "fifo-child" }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([parentDefinition, childDefinition]), + context: {}, + }); + const childStarted = new Deferred(); + const childRelease = new Deferred(); + const parentResumed = new Deferred(); + const successorStarted = new Deferred(); + const child = conductor.createTask({ name: "fifo-child" }, { invocable: true }, async () => { + childStarted.resolve(); + await childRelease.promise; + }); + const parent = conductor.createTask( + { name: "fifo-parent", fifo: true }, + { invocable: true }, + async (_event, ctx) => { + await ctx.invoke("child", { name: "fifo-child" }, {}); + parentResumed.resolve(); + }, + ); + await conductor.ensureInstalled(); + const first = await conductor.invoke({ name: "fifo-parent" }, {}); + const successor = await conductor.invoke({ name: "fifo-parent" }, {}); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [parent, child], + defaultWorker: { concurrency: 3, pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await orchestrator.start(); + await childStarted.promise; + await eventually( + async () => + (await owner(db, "fifo-parent")) === first && + ( + await db.sql< + { waiting_on_execution_id: string | null }[] + >`select waiting_on_execution_id from pgconductor._private_executions where id = ${first}::uuid` + )[0]?.waiting_on_execution_id !== null, + ); + expect(successorStarted.isSettled).toBe(false); + childRelease.resolve(); + await parentResumed.promise; + await eventually(async () => (await owner(db, "fifo-parent")) === successor); + await orchestrator.stop(); + }, 30_000); + + test("releases its owner on cancellation and permanent failure", async () => { + const db = await pool.child(); + databases.push(db); + const definition = payloadDefinition("fifo-release"); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const sleeping = new Deferred(); + const cancelledSuccessor = new Deferred(); + const task = conductor.createTask( + { name: "fifo-release", fifo: true, maxAttempts: 1 }, + { invocable: true }, + async (event, ctx) => { + if (event.payload.id === 0) { + sleeping.resolve(); + await ctx.sleep("cancel-me", 3_600_000); + } else if (event.payload.id === 1) cancelledSuccessor.resolve(); + else throw new Error("permanent failure"); + }, + ); + await conductor.ensureInstalled(); + await db.client.setFakeTime({ date: new Date("2024-01-01T00:00:00Z") }); + const first = await conductor.invoke({ name: "fifo-release" }, { id: 0 }); + const second = await conductor.invoke({ name: "fifo-release" }, { id: 1 }); + const third = await conductor.invoke({ name: "fifo-release" }, { id: 2 }); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: { concurrency: 3, pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await orchestrator.start(); + await sleeping.promise; + await eventually(async () => (await owner(db, "fifo-release")) === first); + await db.client.cancelExecution(first); + await cancelledSuccessor.promise; + expect(await owner(db, "fifo-release")).toBe(second); + await eventually( + async () => + ( + await db.sql< + { failed_at: Date | null }[] + >`select failed_at from pgconductor._private_executions where id = ${first}::uuid` + )[0]?.failed_at !== null, + ); + await eventually( + async () => + ( + await db.sql< + { failed_at: Date | null }[] + >`select failed_at from pgconductor._private_executions where id = ${third}::uuid` + )[0]?.failed_at !== null, + ); + await eventually(async () => (await owner(db, "fifo-release")) === undefined); + await orchestrator.stop(); + }, 30_000); + + test("stale recovery leaves the owner to be resumed before its successor", async () => { + const db = await pool.child(); + databases.push(db); + const definition = payloadDefinition("fifo-stale"); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const firstResumed = new Deferred(); + const firstRelease = new Deferred(); + const successorStarted = new Deferred(); + const task = conductor.createTask( + { name: "fifo-stale", fifo: true }, + { invocable: true }, + async (event) => { + if (event.payload.id === 0) { + firstResumed.resolve(); + await firstRelease.promise; + } else successorStarted.resolve(); + }, + ); + await conductor.ensureInstalled(); + await db.client.registerWorker({ + queueName: "default", + taskSpecs: [{ key: "fifo-stale", queue: "default", fifo: true }], + cronSchedules: [], + eventSubscriptions: [], + }); + const first = await conductor.invoke({ name: "fifo-stale" }, { id: 0 }); + const successor = await conductor.invoke({ name: "fifo-stale" }, { id: 1 }); + const oldOrchestrator = crypto.randomUUID(); + await db.client.orchestratorHeartbeat({ + orchestratorId: oldOrchestrator, + version: "test", + migrationNumber: 1, + }); + const oldClaim = ( + await db.client.getExecutions({ + orchestratorId: oldOrchestrator, + queueName: "default", + batchSize: 2, + filterTaskKeys: [], + }) + )[0]; + expect(oldClaim?.id).toBe(first); + await db.sql` + update pgconductor._private_orchestrators + set last_heartbeat_at = now() - interval '1 hour' + where id = ${oldOrchestrator}::uuid + `; + await db.client.recoverStaleOrchestrators({ maxAge: "1 second" }); + expect(await owner(db, "fifo-stale")).toBe(first); + expect( + ( + await db.sql< + { locked_by: string | null }[] + >`select locked_by from pgconductor._private_executions where id = ${first}::uuid` + )[0]?.locked_by, + ).toBeNull(); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: { concurrency: 2, pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await orchestrator.start(); + await firstResumed.promise; + expect(await owner(db, "fifo-stale")).toBe(first); + expect(successorStarted.isSettled).toBe(false); + firstRelease.resolve(); + await successorStarted.promise; + expect(await owner(db, "fifo-stale")).toBe(successor); + await orchestrator.stop(); + }, 30_000); + + test("allows the same task key in different queues to proceed independently", async () => { + const db = await pool.child(); + databases.push(db); + const a = defineTask({ name: "same-key", queue: "queue-a" }); + const b = defineTask({ name: "same-key", queue: "queue-b" }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([a, b]), + context: {}, + }); + const startedA = new Deferred(); + const startedB = new Deferred(); + const release = new Deferred(); + const taskA = conductor.createTask( + { name: "same-key", queue: "queue-a", fifo: true }, + { invocable: true }, + async () => { + startedA.resolve(); + await release.promise; + }, + ); + const taskB = conductor.createTask( + { name: "same-key", queue: "queue-b", fifo: true }, + { invocable: true }, + async () => { + startedB.resolve(); + await release.promise; + }, + ); + const workerA = conductor.createWorker({ + queue: "queue-a", + tasks: [taskA], + config: { concurrency: 1, pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + const workerB = conductor.createWorker({ + queue: "queue-b", + tasks: [taskB], + config: { concurrency: 1, pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await conductor.ensureInstalled(); + await conductor.invoke({ name: "same-key", queue: "queue-a" }, {}); + await conductor.invoke({ name: "same-key", queue: "queue-b" }, {}); + const orchestrator = Orchestrator.create({ conductor, workers: [workerA, workerB] }); + await orchestrator.start(); + await Promise.all([startedA.promise, startedB.promise]); + release.resolve(); + await orchestrator.stop(); + }, 30_000); + + test("allows different FIFO task keys to proceed independently", async () => { + const db = await pool.child(); + databases.push(db); + const a = defineTask({ name: "fifo-a" }); + const b = defineTask({ name: "fifo-b" }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([a, b]), + context: {}, + }); + const startedA = new Deferred(); + const startedB = new Deferred(); + const release = new Deferred(); + const taskA = conductor.createTask( + { name: "fifo-a", fifo: true }, + { invocable: true }, + async () => { + startedA.resolve(); + await release.promise; + }, + ); + const taskB = conductor.createTask( + { name: "fifo-b", fifo: true }, + { invocable: true }, + async () => { + startedB.resolve(); + await release.promise; + }, + ); + await conductor.ensureInstalled(); + await conductor.invoke({ name: "fifo-a" }, {}); + await conductor.invoke({ name: "fifo-b" }, {}); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [taskA, taskB], + defaultWorker: { concurrency: 2, pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await orchestrator.start(); + await Promise.all([startedA.promise, startedB.promise]); + release.resolve(); + await orchestrator.stop(); + }, 30_000); + + test("dedupe supersession transfers the lane instead of stranding its owner", async () => { + const db = await pool.child(); + databases.push(db); + const definition = payloadDefinition("fifo-dedupe"); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const firstStarted = new Deferred(); + const replacementStarted = new Deferred(); + const releaseFirst = new Deferred(); + const releaseReplacement = new Deferred(); + const task = conductor.createTask( + { name: "fifo-dedupe", fifo: true }, + { invocable: true }, + async (event) => { + if (event.payload.id === 0) { + firstStarted.resolve(); + await releaseFirst.promise; + } else { + replacementStarted.resolve(); + await releaseReplacement.promise; + } + }, + ); + await conductor.ensureInstalled(); + const first = await conductor.invoke( + { name: "fifo-dedupe" }, + { id: 0 }, + { dedupe_key: "same" }, + ); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: { concurrency: 2, pollIntervalMs: 10, flushIntervalMs: 10 }, + }); + await orchestrator.start(); + await firstStarted.promise; + const replacement = await conductor.invoke( + { name: "fifo-dedupe" }, + { id: 1 }, + { dedupe_key: "same" }, + ); + await replacementStarted.promise; + expect(await owner(db, "fifo-dedupe")).toBe(replacement); + expect( + ( + await db.sql< + { failed_at: Date | null }[] + >`select failed_at from pgconductor._private_executions where id = ${first}::uuid` + )[0]?.failed_at, + ).not.toBeNull(); + releaseReplacement.resolve(); + releaseFirst.resolve(); + await eventually(async () => (await owner(db, "fifo-dedupe")) === undefined); + await orchestrator.stop(); + }, 30_000); +}); diff --git a/packages/pgconductor-js/tests/integration/group-concurrency.test.ts b/packages/pgconductor-js/tests/integration/group-concurrency.test.ts new file mode 100644 index 0000000..4dd376f --- /dev/null +++ b/packages/pgconductor-js/tests/integration/group-concurrency.test.ts @@ -0,0 +1,379 @@ +import { test, expect, describe, beforeAll, afterAll, afterEach } from "bun:test"; +import { z } from "zod"; +import { Conductor } from "../../src/conductor"; +import { Orchestrator } from "../../src/orchestrator"; +import { defineTask } from "../../src/task-definition"; +import { TaskSchemas } from "../../src/schemas"; +import { Deferred } from "../../src/lib/deferred"; +import { TestDatabasePool } from "../fixtures/test-database"; +import type { TestDatabase } from "../fixtures/test-database"; + +const workerConfig = { + concurrency: 10, + fetchBatchSize: 10, + flushBatchSize: 10, + pollIntervalMs: 10, + flushIntervalMs: 10, +}; + +// Group limits are intentionally soft: concurrent claim transactions may race. +// These tests use blockers and avoid asserting exact global bounds across workers. + +async function waitUntil(predicate: () => boolean, timeoutMs = 5000): Promise { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() >= deadline) { + throw new Error("Timed out waiting for condition"); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +describe("Group concurrency", () => { + let pool: TestDatabasePool; + const databases: TestDatabase[] = []; + + beforeAll(async () => { + pool = await TestDatabasePool.create(); + }, 60000); + + afterEach(async () => { + await Promise.all(databases.map((db) => db.destroy())); + databases.length = 0; + }); + + afterAll(async () => { + await pool?.destroy(); + }); + + test("serializes executions in the same group", async () => { + const db = await pool.child(); + databases.push(db); + const definition = defineTask({ + name: "same-group", + payload: z.object({ id: z.number() }), + }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const started: number[] = []; + const blockers = new Map>([ + [1, new Deferred()], + [2, new Deferred()], + ]); + const task = conductor.createTask( + { name: "same-group", groupConcurrency: 1 }, + { invocable: true }, + async (event) => { + started.push(event.payload.id); + const blocker = blockers.get(event.payload.id); + if (blocker) await blocker.promise; + }, + ); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: workerConfig, + }); + + await orchestrator.start(); + try { + await conductor.invoke({ name: "same-group" }, { id: 1 }, { group: "tenant-a" }); + await conductor.invoke({ name: "same-group" }, { id: 2 }, { group: "tenant-a" }); + await waitUntil(() => started.length === 1); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(started).toEqual([1]); + + blockers.get(1)?.resolve(); + await waitUntil(() => started.length === 2); + expect(started).toEqual([1, 2]); + blockers.get(2)?.resolve(); + } finally { + blockers.forEach((blocker) => blocker.resolve()); + await orchestrator.stop(); + } + }, 30000); + + test("allows different groups to run in parallel", async () => { + const db = await pool.child(); + databases.push(db); + const definition = defineTask({ + name: "different-groups", + payload: z.object({ id: z.number() }), + }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const started: number[] = []; + const blocker = new Deferred(); + const task = conductor.createTask( + { name: "different-groups", groupConcurrency: 1 }, + { invocable: true }, + async (event) => { + started.push(event.payload.id); + await blocker.promise; + }, + ); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: workerConfig, + }); + + await orchestrator.start(); + try { + await conductor.invoke({ name: "different-groups" }, { id: 1 }, { group: "tenant-a" }); + await conductor.invoke({ name: "different-groups" }, { id: 2 }, { group: "tenant-b" }); + await waitUntil(() => started.length === 2); + expect(new Set(started)).toEqual(new Set([1, 2])); + } finally { + blocker.resolve(); + await orchestrator.stop(); + } + }, 30000); + + test("does not apply the group limit to ungrouped executions", async () => { + const db = await pool.child(); + databases.push(db); + const definition = defineTask({ + name: "ungrouped", + payload: z.object({ id: z.number() }), + }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const started: number[] = []; + const blocker = new Deferred(); + const task = conductor.createTask( + { name: "ungrouped", groupConcurrency: 1 }, + { invocable: true }, + async (event) => { + started.push(event.payload.id); + await blocker.promise; + }, + ); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: workerConfig, + }); + + await orchestrator.start(); + try { + await conductor.invoke({ name: "ungrouped" }, { id: 1 }, { group: "tenant-a" }); + await conductor.invoke({ name: "ungrouped" }, { id: 2 }); + await waitUntil(() => started.length === 2); + expect(new Set(started)).toEqual(new Set([1, 2])); + } finally { + blocker.resolve(); + await orchestrator.stop(); + } + }, 30000); + + test("composes task and group concurrency limits", async () => { + const db = await pool.child(); + databases.push(db); + const definition = defineTask({ + name: "composed-limits", + payload: z.object({ id: z.number() }), + }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const started: number[] = []; + const blockers = new Map>([ + [1, new Deferred()], + [2, new Deferred()], + [3, new Deferred()], + ]); + const task = conductor.createTask( + { name: "composed-limits", concurrency: 2, groupConcurrency: 1 }, + { invocable: true }, + async (event) => { + started.push(event.payload.id); + const blocker = blockers.get(event.payload.id); + if (blocker) await blocker.promise; + }, + ); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: workerConfig, + }); + + await orchestrator.start(); + try { + await conductor.invoke({ name: "composed-limits" }, { id: 1 }, { group: "tenant-a" }); + await conductor.invoke({ name: "composed-limits" }, { id: 3 }, { group: "tenant-b" }); + await conductor.invoke({ name: "composed-limits" }, { id: 2 }, { group: "tenant-a" }); + await waitUntil(() => started.includes(1) && started.includes(3)); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(started).toContain(1); + expect(started).toContain(3); + expect(started).not.toContain(2); + + blockers.forEach((blocker) => blocker.resolve()); + await waitUntil(() => started.length === 3); + } finally { + blockers.forEach((blocker) => blocker.resolve()); + await orchestrator.stop(); + } + }, 30000); + + test("does not share a group across tasks or queues", async () => { + const db = await pool.child(); + databases.push(db); + const taskADefinition = defineTask({ name: "scope-a" }); + const taskBDefinition = defineTask({ name: "scope-b" }); + const queueTaskDefinition = defineTask({ name: "scope-queue", queue: "other" }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([taskADefinition, taskBDefinition, queueTaskDefinition]), + context: {}, + }); + const started = new Set(); + const blocker = new Deferred(); + const taskA = conductor.createTask( + { name: "scope-a", groupConcurrency: 1 }, + { invocable: true }, + async () => { + started.add("task-a"); + await blocker.promise; + }, + ); + const taskB = conductor.createTask( + { name: "scope-b", groupConcurrency: 1 }, + { invocable: true }, + async () => { + started.add("task-b"); + await blocker.promise; + }, + ); + const queueTask = conductor.createTask( + { name: "scope-queue", queue: "other", groupConcurrency: 1 }, + { invocable: true }, + async () => { + started.add("queue"); + await blocker.promise; + }, + ); + const otherWorker = conductor.createWorker({ + queue: "other", + tasks: [queueTask], + config: workerConfig, + }); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [taskA, taskB], + workers: [otherWorker], + defaultWorker: workerConfig, + }); + + await orchestrator.start(); + try { + await conductor.invoke({ name: "scope-a" }, {}, { group: "shared" }); + await conductor.invoke({ name: "scope-b" }, {}, { group: "shared" }); + await conductor.invoke({ name: "scope-queue", queue: "other" }, {}, { group: "shared" }); + await waitUntil(() => started.size === 3); + expect(started).toEqual(new Set(["task-a", "task-b", "queue"])); + } finally { + blocker.resolve(); + await orchestrator.stop(); + } + }, 30000); + + test("propagates group metadata through batch invocation", async () => { + const db = await pool.child(); + databases.push(db); + const definition = defineTask({ + name: "batch-group", + payload: z.object({ id: z.number() }), + }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const started: number[] = []; + const blockers = new Map>([ + [1, new Deferred()], + [2, new Deferred()], + ]); + const task = conductor.createTask( + { name: "batch-group", groupConcurrency: 1 }, + { invocable: true }, + async (event) => { + started.push(event.payload.id); + const blocker = blockers.get(event.payload.id); + if (blocker) await blocker.promise; + }, + ); + const orchestrator = Orchestrator.create({ + conductor, + tasks: [task], + defaultWorker: workerConfig, + }); + + await orchestrator.start(); + try { + await conductor.invoke({ name: "batch-group" }, [ + { payload: { id: 1 }, group: "batch-tenant" }, + { payload: { id: 2 }, group: "batch-tenant" }, + ]); + const rows = await db.sql<{ group: string | null }[]>` + select "group" + from pgconductor._private_executions + where task_key = 'batch-group' + order by enqueue_position asc + `; + expect(rows.map((row) => row.group)).toEqual(["batch-tenant", "batch-tenant"]); + + await waitUntil(() => started.length === 1); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(started).toEqual([1]); + blockers.get(1)?.resolve(); + await waitUntil(() => started.length === 2); + blockers.get(2)?.resolve(); + } finally { + blockers.forEach((blocker) => blocker.resolve()); + await orchestrator.stop(); + } + }, 30000); + + test("rejects invalid task and group concurrency values", async () => { + const db = await pool.child(); + databases.push(db); + const definition = defineTask({ name: "validation" }); + const conductor = Conductor.create({ + sql: db.sql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + const invalidValues = [0, -1, 1.5]; + + for (const value of invalidValues) { + expect(() => + conductor.createTask( + { name: "validation", concurrency: value }, + { invocable: true }, + async () => {}, + ), + ).toThrow("concurrency must be a positive integer"); + expect(() => + conductor.createTask( + { name: "validation", groupConcurrency: value }, + { invocable: true }, + async () => {}, + ), + ).toThrow("groupConcurrency must be a positive integer"); + } + }, 30000); +}); diff --git a/packages/pgconductor-js/tests/integration/invoke-support.test.ts b/packages/pgconductor-js/tests/integration/invoke-support.test.ts index 47d9b3f..45add61 100644 --- a/packages/pgconductor-js/tests/integration/invoke-support.test.ts +++ b/packages/pgconductor-js/tests/integration/invoke-support.test.ts @@ -6,6 +6,16 @@ import { defineTask } from "../../src/task-definition"; import { TestDatabasePool } from "../fixtures/test-database"; import type { TestDatabase } from "../fixtures/test-database"; import { TaskSchemas } from "../../src/schemas"; +import { Deferred } from "../../src/lib/deferred"; + +async function waitForCondition(check: () => Promise, timeoutMs = 20_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await check()) return; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + throw new Error(`condition was not met within ${timeoutMs}ms`); +} describe("Invoke Support", () => { let pool: TestDatabasePool; @@ -41,6 +51,7 @@ describe("Invoke Support", () => { }); const childFn = mock((n: number) => n * 2); + const childCompleted = new Deferred(); const conductor = Conductor.create({ sql: db.sql, @@ -54,6 +65,7 @@ describe("Invoke Support", () => { async (event, _ctx) => { if (event.name === "pgconductor.invoke") { const result = childFn(event.payload.input); + childCompleted.resolve(); return { output: result }; } throw new Error("Unexpected event type"); @@ -85,9 +97,7 @@ describe("Invoke Support", () => { await orchestrator.start(); await conductor.invoke({ name: "parent-task" }, { value: 5 }); - - await new Promise((r) => setTimeout(r, 300)); - + await childCompleted.promise; await orchestrator.stop(); expect(childFn).toHaveBeenCalledTimes(1); @@ -156,15 +166,24 @@ describe("Invoke Support", () => { await orchestrator.start(); await conductor.invoke({ name: "timeout-parent" }, {}); - - await new Promise((r) => setTimeout(r, 300)); + await waitForCondition(async () => { + const [child] = await db.sql<{ released: boolean; sleeping: boolean }[]>` + select + locked_by is null as released, + exists ( + select 1 from pgconductor._private_steps s + where s.execution_id = e.id and s.key = 'long-sleep' + ) as sleeping + from pgconductor._private_executions e + where task_key = 'slow-child' + `; + return child?.released === true && child.sleeping; + }); // Advance time past timeout (1 second) but before sleep completes (5 seconds) const afterTimeout = new Date(startTime.getTime() + 1500); await db.client.setFakeTime({ date: afterTimeout }); - - await new Promise((r) => setTimeout(r, 300)); - + await waitForCondition(async () => parentError.mock.calls.length === 1); await orchestrator.stop(); await db.client.clearFakeTime(); @@ -172,7 +191,7 @@ describe("Invoke Support", () => { expect(parentError).toHaveBeenCalledTimes(1); const errorMsg = parentError.mock.results[0]?.value; expect(errorMsg).toContain("timed out after 1000ms"); - }, 5000); + }, 30000); test("invoke() caches child result on retry", async () => { const db = await pool.child(); @@ -191,6 +210,7 @@ describe("Invoke Support", () => { }); const childFn = mock((n: number) => n * 3); + const parentCompleted = new Deferred(); const conductor = Conductor.create({ sql: db.sql, @@ -228,6 +248,7 @@ describe("Invoke Support", () => { throw new Error("First attempt fails"); } + parentCompleted.resolve(); return { result: childResult.output }; } throw new Error("Unexpected event type"); @@ -243,9 +264,7 @@ describe("Invoke Support", () => { await orchestrator.start(); await conductor.invoke({ name: "retry-parent" }, { value: 7 }); - - await new Promise((r) => setTimeout(r, 500)); - + await parentCompleted.promise; await orchestrator.stop(); expect(childFn).toHaveBeenCalledTimes(1); @@ -269,6 +288,7 @@ describe("Invoke Support", () => { }); const childFn = mock(() => "completed"); + const childCompleted = new Deferred(); const conductor = Conductor.create({ sql: db.sql, @@ -282,6 +302,7 @@ describe("Invoke Support", () => { async (_event, ctx) => { await ctx.sleep("moderate-sleep", 2000); const result = childFn(); + childCompleted.resolve(); return { result }; }, ); @@ -304,9 +325,7 @@ describe("Invoke Support", () => { await orchestrator.start(); await conductor.invoke({ name: "patient-parent" }, {}); - - await new Promise((r) => setTimeout(r, 2500)); - + await childCompleted.promise; await orchestrator.stop(); expect(childFn).toHaveBeenCalledTimes(1); @@ -372,17 +391,31 @@ describe("Invoke Support", () => { await conductor.invoke({ name: "cascade-parent" }, {}); - // Wait for first execution cycle: parent runs → invokes child → child fails - await new Promise((r) => setTimeout(r, 500)); + await waitForCondition(async () => { + const [child] = await db.sql<{ attempts: number; released: boolean }[]>` + select attempts, locked_by is null as released + from pgconductor._private_executions + where task_key = 'failing-child' + `; + return child?.attempts === 1 && child.released; + }); expect(childFn).toHaveBeenCalledTimes(1); - // Advance fake time past first backoff (15 seconds) - const afterFirstBackoff = new Date(startTime.getTime() + 15000); - await db.client.setFakeTime({ date: afterFirstBackoff }); - - // Wait for second execution cycle: child retries and fails permanently - await new Promise((r) => setTimeout(r, 500)); + const [retry] = await db.sql<{ run_at: Date }[]>` + select run_at from pgconductor._private_executions + where task_key = 'failing-child' + `; + if (!retry) throw new Error("expected persisted child retry"); + await db.client.setFakeTime({ date: new Date(retry.run_at.getTime() + 1) }); + await waitForCondition(async () => { + const [parent] = await db.sql<{ failed: boolean }[]>` + select failed_at is not null as failed + from pgconductor._private_executions + where task_key = 'cascade-parent' + `; + return parent?.failed === true; + }); await orchestrator.stop(); await db.client.clearFakeTime(); @@ -418,6 +451,7 @@ describe("Invoke Support", () => { }); const childFn = mock((n: number) => n * 2); + const childCalled = new Deferred(); const conductor = Conductor.create({ sql: db.sql, @@ -452,6 +486,7 @@ describe("Invoke Support", () => { async (event, _ctx) => { if (event.name === "pgconductor.invoke") { const result = childFn(event.payload.input); + childCalled.resolve(); return { output: result }; } throw new Error("Unexpected event type"); @@ -472,9 +507,7 @@ describe("Invoke Support", () => { await orchestrator.start(); await conductor.invoke({ queue: "parent-queue", name: "parent-task" }, { value: 5 }); - - await new Promise((r) => setTimeout(r, 4000)); - + await childCalled.promise; await orchestrator.stop(); expect(childFn).toHaveBeenCalledTimes(1); @@ -493,6 +526,7 @@ describe("Invoke Support", () => { const childDefinition = defineTask({ name: "slow-child-2", + queue: "pending-child-queue", payload: z.object({}), returns: z.object({ completed: z.boolean() }), }); @@ -503,32 +537,34 @@ describe("Invoke Support", () => { context: {}, }); - // Child that takes 5 seconds - const slowChildTask = conductor.createTask( - { name: "slow-child-2" }, - { invocable: true }, - async (_event, ctx) => { - await ctx.sleep("long-sleep", 5000); - return { completed: true }; - }, - ); - // Parent with 1 second timeout - let error throw const timeoutParentTask = conductor.createTask( { name: "timeout-parent-2" }, { invocable: true }, async (_event, ctx) => { - await ctx.invoke("invoke-slow", { name: "slow-child-2" }, {}, 1000); + await ctx.invoke( + "invoke-slow", + { name: "slow-child-2", queue: "pending-child-queue" }, + {}, + 1000, + ); return { success: true }; }, ); + await conductor.ensureInstalled(); + await db.sql` + insert into pgconductor._private_queues (name) + values ('pending-child-queue') + on conflict (name) do nothing + `; + const startTime = new Date("2024-01-01T00:00:00Z"); await db.client.setFakeTime({ date: startTime }); const orchestrator = Orchestrator.create({ conductor, - tasks: [timeoutParentTask, slowChildTask], + tasks: [timeoutParentTask], defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50, @@ -539,16 +575,29 @@ describe("Invoke Support", () => { await conductor.invoke({ name: "timeout-parent-2" }, {}); - // Wait for parent to invoke child - await new Promise((r) => setTimeout(r, 200)); + // Wait for the parent to persist the unclaimed child execution. + await waitForCondition(async () => { + const [child] = await db.sql<{ pending: boolean }[]>` + select locked_by is null as pending + from pgconductor._private_executions + where task_key = 'slow-child-2' + and queue = 'pending-child-queue' + `; + return child?.pending === true; + }); - // Advance time past timeout (child still pending due to 50ms intervals) + // Advance time past timeout while the child is still pending. const afterTimeout = new Date(startTime.getTime() + 1500); await db.client.setFakeTime({ date: afterTimeout }); - - // Wait for timeout to be processed - await new Promise((r) => setTimeout(r, 500)); - + await waitForCondition(async () => { + const [child] = await db.sql<{ failed: boolean }[]>` + select failed_at is not null as failed + from pgconductor._private_executions + where task_key = 'slow-child-2' + and queue = 'pending-child-queue' + `; + return child?.failed === true; + }); await orchestrator.stop(); await db.client.clearFakeTime(); @@ -563,6 +612,7 @@ describe("Invoke Support", () => { select cancelled, failed_at, last_error from pgconductor._private_executions where task_key = 'slow-child-2' + and queue = 'pending-child-queue' `; expect(children.length).toBe(1); @@ -668,8 +718,14 @@ describe("Invoke Support", () => { { dedupe_key: "locked-key" }, ); - // Wait for it to be locked - await new Promise((r) => setTimeout(r, 100)); + await waitForCondition(async () => { + const [execution] = await db.sql<{ locked: boolean }[]>` + select locked_by is not null as locked + from pgconductor._private_executions + where id = ${id1} + `; + return execution?.locked === true; + }); // Second invocation while first is locked - should create NEW execution const id2 = await conductor.invoke( @@ -681,9 +737,14 @@ describe("Invoke Support", () => { // Should be different IDs expect(id2).not.toBe(id1); - // Wait for both executions to complete and flush (100ms + 500ms + 500ms + 50ms + margin) - await new Promise((r) => setTimeout(r, 1200)); - + await waitForCondition(async () => { + const [execution] = await db.sql<{ completed: boolean }[]>` + select completed_at is not null as completed + from pgconductor._private_executions + where id = ${id2} + `; + return execution?.completed === true; + }); await orchestrator.stop(); // First execution should be marked as failed (superseded) @@ -755,6 +816,9 @@ describe("Invoke Support", () => { }, ); + const startTime = new Date("2024-01-01T00:00:00Z"); + await db.client.setFakeTime({ date: startTime }); + const orchestrator = Orchestrator.create({ defaultWorker: { pollIntervalMs: 50, flushIntervalMs: 50 }, conductor, @@ -764,7 +828,7 @@ describe("Invoke Support", () => { await orchestrator.start(); // Rapid invocations with same dedupe_key and delayed run_at - const futureTime = new Date(Date.now() + 1000); + const futureTime = new Date(startTime.getTime() + 1000); const id1 = await conductor.invoke( { name: "debounce-task" }, @@ -772,16 +836,12 @@ describe("Invoke Support", () => { { dedupe_key: "debounce-1", run_at: futureTime }, ); - await new Promise((r) => setTimeout(r, 50)); - const id2 = await conductor.invoke( { name: "debounce-task" }, { value: 2 }, { dedupe_key: "debounce-1", run_at: futureTime }, ); - await new Promise((r) => setTimeout(r, 50)); - const id3 = await conductor.invoke( { name: "debounce-task" }, { value: 3 }, @@ -800,10 +860,10 @@ describe("Invoke Support", () => { `; expect(executions[0]?.count).toBe(1); - // Wait for execution - await new Promise((r) => setTimeout(r, 1500)); - + await db.client.setFakeTime({ date: futureTime }); + await waitForCondition(async () => executionCount === 1); await orchestrator.stop(); + await db.client.clearFakeTime(); // Should have executed only once with the last value expect(executionCount).toBe(1); diff --git a/packages/pgconductor-js/tests/integration/step-support.test.ts b/packages/pgconductor-js/tests/integration/step-support.test.ts index 32e1638..b43fcb6 100644 --- a/packages/pgconductor-js/tests/integration/step-support.test.ts +++ b/packages/pgconductor-js/tests/integration/step-support.test.ts @@ -6,6 +6,7 @@ import { defineTask } from "../../src/task-definition"; import { TestDatabasePool } from "../fixtures/test-database"; import type { TestDatabase } from "../fixtures/test-database"; import { TaskSchemas } from "../../src/schemas"; +import { Deferred } from "../../src/lib/deferred"; describe("Step Support", () => { let pool: TestDatabasePool; @@ -35,6 +36,7 @@ describe("Step Support", () => { }); const expensiveFn = mock((n: number) => n * 2); + const taskCompleted = new Deferred(); const conductor = Conductor.create({ sql: db.sql, @@ -51,6 +53,7 @@ describe("Step Support", () => { return expensiveFn(event.payload.value); }); + taskCompleted.resolve(); return { result }; } throw new Error("Unexpected event type"); @@ -66,9 +69,7 @@ describe("Step Support", () => { await orchestrator.start(); await conductor.invoke({ name: "step-task" }, { value: 5 }); - - await new Promise((r) => setTimeout(r, 200)); - + await taskCompleted.promise; await orchestrator.stop(); // Expensive function should only be called once @@ -87,6 +88,8 @@ describe("Step Support", () => { }); const executionSteps = mock((step: string) => step); + const sleepStarted = new Deferred(); + const sleepFinished = new Deferred(); const conductor = Conductor.create({ sql: db.sql, @@ -100,8 +103,10 @@ describe("Step Support", () => { async (event, ctx) => { if (event.name === "pgconductor.invoke") { executionSteps("before-sleep"); + sleepStarted.resolve(); await ctx.sleep("wait", event.payload.delay); executionSteps("after-sleep"); + sleepFinished.resolve(); return { completed: true }; } throw new Error("Unexpected event type"); @@ -117,18 +122,12 @@ describe("Step Support", () => { await orchestrator.start(); await conductor.invoke({ name: "sleep-task" }, { delay: 2000 }); + await sleepStarted.promise; - // Wait for initial execution (should hit sleep and release) - await new Promise((r) => setTimeout(r, 200)); - - // At this point, should have seen "before-sleep" but not "after-sleep" expect(executionSteps).toHaveBeenCalledWith("before-sleep"); expect(executionSteps).not.toHaveBeenCalledWith("after-sleep"); - // Wait for sleep to complete and task to resume (2s sleep + buffer) - await new Promise((r) => setTimeout(r, 2200)); - - // Now should see "after-sleep" + await sleepFinished.promise; expect(executionSteps).toHaveBeenCalledWith("after-sleep"); await orchestrator.stop(); @@ -145,6 +144,7 @@ describe("Step Support", () => { }); const processedItems = mock((item: number) => item); + const checkpointCompleted = new Deferred(); const conductor = Conductor.create({ sql: db.sql, @@ -163,6 +163,7 @@ describe("Step Support", () => { // Simulate some work await new Promise((r) => setTimeout(r, 50)); } + checkpointCompleted.resolve(); return { processed: event.payload.items }; } throw new Error("Unexpected event type"); @@ -178,10 +179,7 @@ describe("Step Support", () => { await orchestrator.start(); await conductor.invoke({ name: "checkpoint-task" }, { items: 5 }); - - // Wait for task to complete - await new Promise((r) => setTimeout(r, 300)); - + await checkpointCompleted.promise; await orchestrator.stop(); // Should have processed all items @@ -201,6 +199,7 @@ describe("Step Support", () => { const step1Fn = mock((n: number) => n + 1); const step2Fn = mock((n: number) => n * 2); const step3Fn = mock((n: number) => n - 3); + const stepsCompleted = new Deferred(); const conductor = Conductor.create({ sql: db.sql, @@ -216,6 +215,7 @@ describe("Step Support", () => { const a = await ctx.step("step1", () => step1Fn(event.payload.x)); const b = await ctx.step("step2", () => step2Fn(a)); const c = await ctx.step("step3", () => step3Fn(b)); + stepsCompleted.resolve(); return { result: c }; } throw new Error("Unexpected event type"); @@ -231,9 +231,7 @@ describe("Step Support", () => { await orchestrator.start(); await conductor.invoke({ name: "multi-step-task" }, { x: 5 }); - - await new Promise((r) => setTimeout(r, 200)); - + await stepsCompleted.promise; await orchestrator.stop(); // All steps should execute once @@ -259,6 +257,7 @@ describe("Step Support", () => { let transformedValue: string[] | undefined; let countValue: number | undefined; + const completed = new Deferred(); const conductor = Conductor.create({ sql: db.sql, @@ -284,6 +283,7 @@ describe("Step Support", () => { return transformed.length; }); + completed.resolve(); return { count }; } throw new Error("Unexpected event type"); @@ -299,9 +299,7 @@ describe("Step Support", () => { await orchestrator.start(); await conductor.invoke({ name: "step-unwrap-task" }, { items: ["apple", "banana", "cherry"] }); - - await new Promise((r) => setTimeout(r, 300)); - + await completed.promise; await orchestrator.stop(); expect(transformedValue).toEqual(["APPLE", "BANANA", "CHERRY"]); diff --git a/packages/pgconductor-js/tests/mocks/in-memory-database-client.ts b/packages/pgconductor-js/tests/mocks/in-memory-database-client.ts index 7170e10..f45e41a 100644 --- a/packages/pgconductor-js/tests/mocks/in-memory-database-client.ts +++ b/packages/pgconductor-js/tests/mocks/in-memory-database-client.ts @@ -39,6 +39,7 @@ interface StoredExecution { id: string; task_key: string; queue: string; + group: string | null; payload: Payload; state: "pending" | "running" | "completed" | "failed"; run_at: Date; @@ -55,9 +56,9 @@ interface StoredExecution { cron_expression: string | null; priority: number; orchestrator_id: string | null; + claim_token: string | null; parent_execution_id: string | null; parent_step_key: string | null; - slot_group_number: number | null; created_at: Date; updated_at: Date; } @@ -78,6 +79,7 @@ interface StoredTask { window_start: string | null; window_end: string | null; concurrency: number | null; + group_concurrency: number | null; } interface StoredCronSchedule { @@ -202,10 +204,15 @@ export class InMemoryDatabaseClient implements IDatabaseClient { // Release executions claimed by stale orchestrator for (const exec of this.executions.values()) { if (exec.orchestrator_id === orchestrator.id && exec.state === "running") { - exec.state = "pending"; + exec.state = exec.cancelled ? "failed" : "pending"; + exec.last_error = exec.cancelled + ? exec.last_error || "Task was cancelled" + : exec.last_error; exec.orchestrator_id = null; + exec.claim_token = null; } } + this.orchestrators.delete(orchestrator.id); } } } @@ -241,6 +248,13 @@ export class InMemoryDatabaseClient implements IDatabaseClient { _opts?: { signal?: AbortSignal }, ): Promise { this.orchestrators.delete(args.orchestratorId); + for (const exec of this.executions.values()) { + if (exec.orchestrator_id !== args.orchestratorId || exec.state !== "running") continue; + exec.state = exec.cancelled ? "failed" : "pending"; + exec.last_error = exec.cancelled ? exec.last_error || "Task was cancelled" : exec.last_error; + exec.orchestrator_id = null; + exec.claim_token = null; + } } // ============================================================================ @@ -278,8 +292,9 @@ export class InMemoryDatabaseClient implements IDatabaseClient { window_start: taskSpec.window?.[0] || null, window_end: taskSpec.window?.[1] || null, concurrency: taskSpec.concurrency || null, + group_concurrency: taskSpec.groupConcurrency || null, }; - this.tasks.set(taskSpec.key, task); + this.tasks.set(this.taskId(taskSpec.key, task.queue), task); } // Register cron schedules (ExecutionSpec[]) @@ -307,13 +322,13 @@ export class InMemoryDatabaseClient implements IDatabaseClient { ): Promise { const results: Execution[] = []; const now = this.getInternalTime(); - const taskKeysWithConcurrency = new Set(args.taskKeysWithConcurrency || []); const filterTaskKeys = new Set(args.filterTaskKeys || []); const concurrencyCount = new Map(); - // Count running executions per task for concurrency limits + // Count running executions per task for concurrency limits. for (const exec of this.executions.values()) { - if (exec.state === "running" && taskKeysWithConcurrency.has(exec.task_key)) { + const task = this.tasks.get(this.taskId(exec.task_key, exec.queue)); + if (exec.state === "running" && task?.concurrency != null) { concurrencyCount.set(exec.task_key, (concurrencyCount.get(exec.task_key) || 0) + 1); } } @@ -338,20 +353,31 @@ export class InMemoryDatabaseClient implements IDatabaseClient { if (parent && parent.state !== "completed") continue; } - // Check concurrency limit - if (taskKeysWithConcurrency.has(exec.task_key)) { - const task = this.tasks.get(exec.task_key); - const limit = task?.concurrency || 1; + const task = this.tasks.get(this.taskId(exec.task_key, exec.queue)); + // Check concurrency limit. + if (task?.concurrency != null) { const current = concurrencyCount.get(exec.task_key) || 0; - if (current >= limit) continue; + if (current >= task.concurrency) continue; + } + if (task?.group_concurrency && exec.group) { + const activeGroup = Array.from(this.executions.values()).filter( + (other) => + other.queue === exec.queue && + other.task_key === exec.task_key && + other.group === exec.group && + other.state === "running", + ).length; + if (activeGroup >= task.group_concurrency) continue; } - // Claim execution + // Claim execution with a fresh fencing token. exec.state = "running"; + exec.attempts += 1; exec.orchestrator_id = args.orchestratorId; + exec.claim_token = crypto.randomUUID(); // Update concurrency count - if (taskKeysWithConcurrency.has(exec.task_key)) { + if (task?.concurrency != null) { concurrencyCount.set(exec.task_key, (concurrencyCount.get(exec.task_key) || 0) + 1); } @@ -366,7 +392,9 @@ export class InMemoryDatabaseClient implements IDatabaseClient { last_error: exec.last_error, dedupe_key: exec.dedupe_key || undefined, cron_expression: exec.cron_expression || undefined, - slot_group_number: exec.slot_group_number || undefined, + group: exec.group, + locked_by: exec.orchestrator_id || "", + claim_token: exec.claim_token || "", }); if (results.length >= args.batchSize) break; @@ -396,13 +424,23 @@ export class InMemoryDatabaseClient implements IDatabaseClient { for (const result of results) { const exec = this.executions.get(result.execution_id); - if (!exec) continue; + if ( + !exec || + !this.ownsClaim({ + executionId: result.execution_id, + queue: result.queue, + orchestratorId: result.orchestrator_id, + claimToken: result.claim_token, + }) + ) + continue; switch (result.status) { case "completed": { exec.state = "completed"; exec.result = result.result || null; exec.orchestrator_id = null; + exec.claim_token = null; // Wake up parent if waiting if (exec.parent_execution_id) { @@ -422,7 +460,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { } // Remove if cleanup is configured (only if task registered) - const task = this.tasks.get(exec.task_key); + const task = this.tasks.get(this.taskId(exec.task_key, exec.queue)); if (task && task.remove_on_complete_days != null) { this.executions.delete(exec.id); this.steps.delete(exec.id); @@ -431,11 +469,11 @@ export class InMemoryDatabaseClient implements IDatabaseClient { } case "failed": { - exec.attempts++; exec.last_error = result.error; exec.orchestrator_id = null; + exec.claim_token = null; - const task = this.tasks.get(exec.task_key); + const task = this.tasks.get(this.taskId(exec.task_key, exec.queue)); const maxAttempts = task?.max_attempts || 3; if (exec.attempts >= maxAttempts) { @@ -476,6 +514,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { case "released": { exec.state = "pending"; exec.orchestrator_id = null; + exec.claim_token = null; if (result.reschedule_in_ms === "infinity") { exec.run_at = new Date(8640000000000000); // Max date @@ -491,6 +530,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { exec.state = "failed"; exec.last_error = result.error; exec.orchestrator_id = null; + exec.claim_token = null; // Fail parent if waiting if (exec.parent_execution_id) { @@ -504,7 +544,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { } } - const task = this.tasks.get(exec.task_key); + const task = this.tasks.get(this.taskId(exec.task_key, exec.queue)); if (task && task.remove_on_fail_days != null) { this.executions.delete(exec.id); this.steps.delete(exec.id); @@ -518,6 +558,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { task_key: result.child_task_name, queue: result.child_task_queue, payload: result.child_payload || {}, + group: result.group, parent_execution_id: exec.id, parent_step_key: result.step_key, }); @@ -527,6 +568,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { exec.waiting_on_execution_id = childId; exec.waiting_step_key = result.step_key; exec.orchestrator_id = null; + exec.claim_token = null; if (result.timeout_ms === "infinity") { exec.waiting_timeout_at = new Date(8640000000000000); @@ -554,6 +596,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { // exec.state = "pending"; // exec.run_at = new Date(8640000000000000); // Wait indefinitely // exec.orchestrator_id = null; + // exec.claim_token = null; // break; // } @@ -578,6 +621,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { // exec.state = "pending"; // exec.run_at = new Date(8640000000000000); // Wait indefinitely // exec.orchestrator_id = null; + // exec.claim_token = null; // break; // } } @@ -625,13 +669,14 @@ export class InMemoryDatabaseClient implements IDatabaseClient { // Helper to create execution const createExecution = (singletonOnValue: Date | null): string => { - const task = this.tasks.get(spec.task_key); + const task = this.tasks.get(this.taskId(spec.task_key, spec.queue)); const id = this.generateId(); const execution: StoredExecution = { id, task_key: spec.task_key, queue: spec.queue, + group: spec.group || null, payload: spec.payload || {}, state: "pending", run_at: spec.run_at || now, @@ -648,9 +693,9 @@ export class InMemoryDatabaseClient implements IDatabaseClient { cron_expression: spec.cron_expression || null, priority: spec.priority || 0, orchestrator_id: null, + claim_token: null, parent_execution_id: spec.parent_execution_id || null, parent_step_key: spec.parent_step_key || null, - slot_group_number: null, created_at: now, updated_at: now, }; @@ -729,6 +774,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { exec.last_error = "superseded by reinvoke"; exec.dedupe_key = null; exec.orchestrator_id = null; + exec.claim_token = null; // Will create new execution below } else { // Unlocked execution - update it with new values (replace behavior) @@ -823,8 +869,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { exec.last_error = options.reason || "Execution was cancelled"; if (exec.state === "running") { - exec.state = "failed"; - exec.orchestrator_id = null; + exec.cancelled = true; } return true; @@ -915,6 +960,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { run_at: nextRun, dedupe_key: dedupeKey, cron_expression: exec.cron_expression, + group: exec.group, }); } @@ -934,21 +980,24 @@ export class InMemoryDatabaseClient implements IDatabaseClient { // Steps // ============================================================================ - async loadStep(args: LoadStepArgs, _opts?: { signal?: AbortSignal }): Promise { + async loadStep( + args: LoadStepArgs, + _opts?: { signal?: AbortSignal }, + ): Promise { + if (!this.ownsClaim(args)) return undefined; const execSteps = this.steps.get(args.executionId); - if (!execSteps) return null; - + if (!execSteps) return undefined; const step = execSteps.get(args.key); - return step ? step.result : null; + return step ? step.result : undefined; } async saveStep(args: SaveStepArgs, _opts?: { signal?: AbortSignal }): Promise { + if (!this.ownsClaim(args)) return; let execSteps = this.steps.get(args.executionId); if (!execSteps) { execSteps = new Map(); this.steps.set(args.executionId, execSteps); } - execSteps.set(args.key, { execution_id: args.executionId, step_key: args.key, @@ -962,7 +1011,7 @@ export class InMemoryDatabaseClient implements IDatabaseClient { _opts?: { signal?: AbortSignal }, ): Promise { const exec = this.executions.get(args.executionId); - if (!exec) return; + if (!exec || !this.ownsClaim(args)) return; exec.waiting_on_execution_id = null; exec.waiting_step_key = null; @@ -1020,6 +1069,25 @@ export class InMemoryDatabaseClient implements IDatabaseClient { // Helpers // ============================================================================ + private taskId(key: string, queue: string): string { + return `${queue}\u0000${key}`; + } + + private ownsClaim(args: { + executionId: string; + queue: string; + orchestratorId: string; + claimToken: string; + }): boolean { + const exec = this.executions.get(args.executionId); + return Boolean( + exec && + exec.queue === args.queue && + exec.orchestrator_id === args.orchestratorId && + exec.claim_token === args.claimToken, + ); + } + private generateId(): string { this.idCounter++; return `in-memory-${this.idCounter.toString().padStart(8, "0")}`; diff --git a/packages/pgconductor-js/tests/unit/fifo-types.test.ts b/packages/pgconductor-js/tests/unit/fifo-types.test.ts new file mode 100644 index 0000000..3ed0389 --- /dev/null +++ b/packages/pgconductor-js/tests/unit/fifo-types.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, test } from "bun:test"; +import { Conductor } from "../../src/conductor"; +import { defineTask } from "../../src/task-definition"; +import { TaskSchemas } from "../../src/schemas"; + +const mockSql = {} as any; + +describe("FIFO task configuration", () => { + test("is exposed and rejects conflicting limits at type level", () => { + const definition = defineTask({ name: "fifo-task" }); + const conductor = Conductor.create({ + sql: mockSql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + + const task = conductor.createTask( + { name: "fifo-task", fifo: true }, + { invocable: true }, + async () => {}, + ); + expect(task.fifo).toBe(true); + + if (false) + conductor.createTask( + // @ts-expect-error FIFO and task concurrency are contradictory. + { name: "fifo-task", fifo: true, concurrency: 2 }, + { invocable: true }, + async () => {}, + ); + if (false) + conductor.createTask( + // @ts-expect-error FIFO and group concurrency are contradictory. + { name: "fifo-task", fifo: true, groupConcurrency: 2 }, + { invocable: true }, + async () => {}, + ); + }); + + test("rejects conflicting limits at runtime", () => { + const definition = defineTask({ name: "fifo-runtime" }); + const conductor = Conductor.create({ + sql: mockSql, + tasks: TaskSchemas.fromSchema([definition]), + context: {}, + }); + expect(() => + conductor.createTask( + { name: "fifo-runtime", fifo: true, concurrency: 2 } as any, + { invocable: true }, + async () => {}, + ), + ).toThrow("fifo cannot be combined"); + }); +}); diff --git a/packages/pgconductor-js/tests/unit/in-memory-database-client.test.ts b/packages/pgconductor-js/tests/unit/in-memory-database-client.test.ts index 366694c..09064a1 100644 --- a/packages/pgconductor-js/tests/unit/in-memory-database-client.test.ts +++ b/packages/pgconductor-js/tests/unit/in-memory-database-client.test.ts @@ -3,6 +3,17 @@ import { test, expect, describe } from "bun:test"; import { InMemoryDatabaseClient } from "../mocks/in-memory-database-client"; describe("InMemoryDatabaseClient", () => { + function fencing(db: InMemoryDatabaseClient, executionId: string) { + const execution = db.getExecution(executionId); + if (!execution?.orchestrator_id || !execution.claim_token) { + throw new Error(`execution ${executionId} is not claimed`); + } + return { + orchestrator_id: execution.orchestrator_id, + claim_token: execution.claim_token, + }; + } + describe("Basic Execution Lifecycle", () => { test("invoke creates pending execution", async () => { const db = new InMemoryDatabaseClient(); @@ -41,7 +52,6 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); @@ -67,13 +77,13 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); await db.returnExecutions([ { execution_id: id!, + ...fencing(db, id!), queue: "default", task_key: "test-task", status: "completed", @@ -113,7 +123,6 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); @@ -122,6 +131,7 @@ describe("InMemoryDatabaseClient", () => { await db.returnExecutions([ { execution_id: id!, + ...fencing(db, id!), queue: "default", task_key: "test-task", status: "failed", @@ -165,13 +175,13 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); await db.returnExecutions([ { execution_id: id!, + ...fencing(db, id!), queue: "default", task_key: "test-task", status: "failed", @@ -188,13 +198,13 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); await db.returnExecutions([ { execution_id: id!, + ...fencing(db, id!), queue: "default", task_key: "test-task", status: "failed", @@ -243,18 +253,20 @@ describe("InMemoryDatabaseClient", () => { }, scheduleName: "test-schedule", }); + db.advanceTime(60000); - await db.getExecutions({ + const claimed = await db.getExecutions({ orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); await db.returnExecutions([ { execution_id: id!, + orchestrator_id: claimed[0]!.locked_by, + claim_token: claimed[0]!.claim_token, queue: "default", task_key: "cron-task", status: "completed", @@ -333,7 +345,6 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); @@ -350,6 +361,7 @@ describe("InMemoryDatabaseClient", () => { await db.returnExecutions([ { execution_id: id!, + ...fencing(db, id!), queue: "default", task_key: "cron-task", status: "failed", @@ -373,13 +385,13 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); await db.returnExecutions([ { execution_id: id!, + ...fencing(db, id!), queue: "default", task_key: "cron-task", status: "completed", @@ -431,7 +443,6 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); @@ -453,6 +464,7 @@ describe("InMemoryDatabaseClient", () => { await db.returnExecutions([ { execution_id: id!, + ...fencing(db, id!), queue: "default", task_key: "slow-task", status: "failed", @@ -475,7 +487,6 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); @@ -486,6 +497,7 @@ describe("InMemoryDatabaseClient", () => { await db.returnExecutions([ { execution_id: id!, + ...fencing(db, id!), queue: "default", task_key: "slow-task", status: "failed", @@ -522,13 +534,13 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); await db.returnExecutions([ { execution_id: parentId!, + ...fencing(db, parentId!), queue: "default", task_key: "parent", status: "invoke_child", @@ -565,13 +577,13 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); await db.returnExecutions([ { execution_id: parentId!, + ...fencing(db, parentId!), queue: "default", task_key: "parent", status: "invoke_child", @@ -590,13 +602,13 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); await db.returnExecutions([ { execution_id: childId, + ...fencing(db, childId), queue: "default", task_key: "child", status: "completed", @@ -636,13 +648,13 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); await db.returnExecutions([ { execution_id: parentId!, + ...fencing(db, parentId!), queue: "default", task_key: "parent", status: "invoke_child", @@ -661,13 +673,13 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); await db.returnExecutions([ { execution_id: childId, + ...fencing(db, childId), queue: "default", task_key: "child", status: "failed", @@ -691,16 +703,29 @@ describe("InMemoryDatabaseClient", () => { queue: "default", payload: {}, }); + const claimed = ( + await db.getExecutions({ + orchestratorId: "test-orch", + queueName: "default", + batchSize: 1, + filterTaskKeys: [], + }) + )[0]!; await db.saveStep({ executionId: execId!, - queue: "default", + queue: claimed.queue, + orchestratorId: claimed.locked_by, + claimToken: claimed.claim_token, key: "step1", result: { data: "value" }, }); const result = await db.loadStep({ executionId: execId!, + queue: claimed.queue, + orchestratorId: claimed.locked_by, + claimToken: claimed.claim_token, key: "step1", }); @@ -712,10 +737,13 @@ describe("InMemoryDatabaseClient", () => { const result = await db.loadStep({ executionId: "nonexistent", + queue: "default", + orchestratorId: "test-orch", + claimToken: "test-claim-token", key: "step1", }); - expect(result).toBeNull(); + expect(result).toBeUndefined(); }); }); @@ -745,7 +773,6 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); @@ -759,7 +786,6 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); @@ -797,7 +823,6 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: ["limited-task"], filterTaskKeys: [], }); @@ -808,7 +833,6 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: ["limited-task"], filterTaskKeys: [], }); @@ -818,6 +842,7 @@ describe("InMemoryDatabaseClient", () => { await db.returnExecutions([ { execution_id: batch1[0]!.id, + ...fencing(db, batch1[0]!.id), queue: "default", task_key: "limited-task", status: "completed", @@ -829,7 +854,6 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: ["limited-task"], filterTaskKeys: [], }); @@ -876,13 +900,13 @@ describe("InMemoryDatabaseClient", () => { orchestratorId: "test-orch", queueName: "default", batchSize: 10, - taskKeysWithConcurrency: [], filterTaskKeys: [], }); await db.returnExecutions([ { execution_id: id1!, + ...fencing(db, id1!), queue: "default", task_key: "test", status: "completed", diff --git a/packages/pgconductor-js/tests/unit/lib/map-concurrent.test.ts b/packages/pgconductor-js/tests/unit/lib/map-concurrent.test.ts index 4e8db55..3e17be7 100644 --- a/packages/pgconductor-js/tests/unit/lib/map-concurrent.test.ts +++ b/packages/pgconductor-js/tests/unit/lib/map-concurrent.test.ts @@ -1,5 +1,6 @@ import { test, expect, describe } from "bun:test"; import { mapConcurrent } from "../../../src/lib/map-concurrent"; +import { AsyncQueue } from "../../../src/lib/async-queue"; import type { PollableAsyncIterable } from "../../../src/lib/async-queue"; class PollableGenerator implements PollableAsyncIterable { @@ -96,6 +97,27 @@ describe("mapConcurrent", () => { expect(results).toEqual(["item-0", "item-1", "item-2"]); }); + test("continues consuming a live queue while work is active", async () => { + const source = new AsyncQueue(4); + const results: number[] = []; + const consuming = (async () => { + for await (const result of mapConcurrent(source, 2, async (n) => { + await new Promise((resolve) => setTimeout(resolve, 10)); + return n * 2; + })) { + results.push(result); + } + })(); + + await source.push(1); + await new Promise((resolve) => setTimeout(resolve, 2)); + await source.push(2); + source.close(); + await consuming; + + expect(results.sort()).toEqual([2, 4]); + }); + test("handles empty source", async () => { async function* empty() { // Yields nothing @@ -109,21 +131,40 @@ describe("mapConcurrent", () => { expect(results).toEqual([]); }); - test("propagates errors from mapper", async () => { - const source = pollable(generateNumbers(5)); + test("propagates errors from mapper and closes the source", async () => { + let returned = false; + let first = true; + const iterator: AsyncIterator = { + next: async () => { + if (first) { + first = false; + return { value: 1, done: false }; + } + return new Promise>(() => {}); + }, + return: async () => { + returned = true; + return { value: undefined, done: true }; + }, + }; + const source: PollableAsyncIterable = { + tryNext: () => undefined, + [Symbol.asyncIterator]: () => iterator, + }; try { - for await (const _ of mapConcurrent(source, 2, async (n) => { - if (n === 2) throw new Error("test error"); - return n; + for await (const _ of mapConcurrent(source, 1, async () => { + throw new Error("test error"); })) { - // Should throw before completing + expect.unreachable(); } expect.unreachable(); } catch (err) { expect(err).toBeInstanceOf(Error); expect((err as Error).message).toBe("test error"); } + + expect(returned).toBe(true); }); test("handles single concurrency", async () => { diff --git a/packages/pgconductor-js/tests/unit/throttle-debounce.test.ts b/packages/pgconductor-js/tests/unit/throttle-debounce.test.ts index 23816d5..6ae108d 100644 --- a/packages/pgconductor-js/tests/unit/throttle-debounce.test.ts +++ b/packages/pgconductor-js/tests/unit/throttle-debounce.test.ts @@ -134,7 +134,6 @@ describe("Throttle and Debounce", () => { queueName: "default", batchSize: 100, orchestratorId: "test-orch", - taskKeysWithConcurrency: [], filterTaskKeys: [], });