From b1cd1c372e0a9d32076e27df16fbe081151365cc Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 13:40:03 +0000 Subject: [PATCH 01/57] Add custom background task API Generalize Fedify's enqueue-and-process-later pattern, previously limited to outgoing activity delivery, to arbitrary application-defined background jobs. `Federation` and `FederationBuilder` gain `defineTask()` (via the new `TaskRegistry` interface), and `Context` gains `enqueueTask()`/`enqueueTaskMany()`. Each task carries a Standard Schema that infers the payload type and validates it both at enqueue time and at dequeue time, guarding against schema drift across deployments. Payloads are serialized with devalue so that `Date`, `Map`, `Set`, `URL`, `bigint`, circular references, and Activity Vocabulary objects round-trip faithfully across every message queue backend. Failed handlers retry with exponential backoff by default, configurable per task or federation-wide, and tasks can be isolated onto a dedicated queue or fall back to the outbox queue. The payload codec is implemented twice on purpose: `codec.ts` as a class (`TaskCodec`) and `codec-fn.ts` as standalone utility functions, each with its own tests. Only the class is wired into the runtime; the functional variant is kept temporarily so the team can compare the two styles and decide which reads better before one is removed. https://github.com/fedify-dev/fedify/issues/206 https://github.com/fedify-dev/fedify/issues/797 Assisted-by: Claude Code:claude-opus-4-8 --- CHANGES.md | 32 + deno.lock | 9 +- docs/.vitepress/config.mts | 1 + docs/manual/tasks.md | 247 +++++++ packages/fedify/deno.json | 2 + packages/fedify/package.json | 2 + packages/fedify/src/federation/builder.ts | 29 + packages/fedify/src/federation/context.ts | 47 ++ packages/fedify/src/federation/federation.ts | 31 +- packages/fedify/src/federation/middleware.ts | 232 ++++++- packages/fedify/src/federation/mod.ts | 7 + packages/fedify/src/federation/queue.ts | 25 +- .../src/federation/tasks/codec-fn.test.ts | 230 +++++++ .../fedify/src/federation/tasks/codec-fn.ts | 188 +++++ .../fedify/src/federation/tasks/codec.test.ts | 284 ++++++++ packages/fedify/src/federation/tasks/codec.ts | 186 +++++ packages/fedify/src/federation/tasks/mod.ts | 11 + packages/fedify/src/federation/tasks/task.ts | 169 +++++ .../fedify/src/federation/tasks/tasks.test.ts | 647 ++++++++++++++++++ packages/fedify/src/testing/context.ts | 8 + packages/testing/src/context.ts | 8 + packages/testing/src/mock.ts | 37 +- pnpm-lock.yaml | 22 +- 23 files changed, 2427 insertions(+), 27 deletions(-) create mode 100644 docs/manual/tasks.md create mode 100644 packages/fedify/src/federation/tasks/codec-fn.test.ts create mode 100644 packages/fedify/src/federation/tasks/codec-fn.ts create mode 100644 packages/fedify/src/federation/tasks/codec.test.ts create mode 100644 packages/fedify/src/federation/tasks/codec.ts create mode 100644 packages/fedify/src/federation/tasks/mod.ts create mode 100644 packages/fedify/src/federation/tasks/task.ts create mode 100644 packages/fedify/src/federation/tasks/tasks.test.ts diff --git a/CHANGES.md b/CHANGES.md index 0e9760324..3d36e287d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -343,6 +343,37 @@ Released on June 25, 2026. the dispatcher overload hot path; other contributors to `check-all` cost may remain. [[#613], [#800] by ChanHaeng Lee] + - Added a custom background task API that generalizes Fedify's + enqueue-and-process-later pattern to arbitrary application-defined jobs: + + - `Federation` and `FederationBuilder` gained a `defineTask()` method + through the new `TaskRegistry` interface, which `Federatable` now + extends. + - `Context` gained `enqueueTask()` and `enqueueTaskMany()` methods, + with `delay` and `orderingKey` options + (new `TaskEnqueueOptions` interface). + - Every task requires a [Standard Schema] + (`schema` option) from which the payload type is inferred; payloads + are validated at enqueue time (fail fast) and again at dequeue time + (protection against schema drift across deployments). + - Payloads are serialized by Fedify with devalue, so `Date`, `Map`, + `Set`, `URL`, `bigint`, circular references, and Activity Vocabulary + objects round-trip faithfully across every message queue backend. + - Failed handlers are retried with exponential backoff by default; + tasks support per-task `retryPolicy` and `onError` options, the new + `FederationOptions.taskRetryPolicy` sets the federation-wide default, + and queues with `nativeRetrial` delegate retries to the backend. + - Tasks can be isolated from activity delivery through the new + `FederationQueueOptions.task` slot or a per-task `queue` option; + without them, tasks fall back to the outbox queue unless the new + `FederationOptions.taskQueueResolution` option is set to `"strict"`. + `Federation.startQueue()` now accepts `queue: "task"` to run + a task-only worker. + + [[#206], [#797] by ChanHaeng Lee] + +[Standard Schema]: https://standardschema.dev/ +[#206]: https://github.com/fedify-dev/fedify/issues/206 [#316]: https://github.com/fedify-dev/fedify/issues/316 [#418]: https://github.com/fedify-dev/fedify/issues/418 [#613]: https://github.com/fedify-dev/fedify/issues/613 @@ -372,6 +403,7 @@ Released on June 25, 2026. [#778]: https://github.com/fedify-dev/fedify/pull/778 [#782]: https://github.com/fedify-dev/fedify/issues/782 [#787]: https://github.com/fedify-dev/fedify/pull/787 +[#797]: https://github.com/fedify-dev/fedify/issues/797 [#800]: https://github.com/fedify-dev/fedify/pull/800 ### @fedify/cli diff --git a/deno.lock b/deno.lock index 8d3d97c57..b24477b71 100644 --- a/deno.lock +++ b/deno.lock @@ -124,6 +124,7 @@ "npm:cli-highlight@^2.1.11": "2.1.11", "npm:cli-table3@~0.6.5": "0.6.5", "npm:dax@~0.46.1": "0.46.1", + "npm:devalue@^5.8.1": "5.8.1", "npm:enquirer@^2.4.1": "2.4.1", "npm:es-toolkit@^1.46.1": "1.46.1", "npm:esbuild-wasm@~0.25.11": "0.25.12", @@ -4763,8 +4764,8 @@ "base-64" ] }, - "devalue@5.7.1": { - "integrity": "sha512-MUbZ586EgQqdRnC4yDrlod3BEdyvE4TapGYHMW2CiaW+KkkFmWEFqBUaLltEZCGi0iFXCEjRF0OjF0DV2QHjOA==" + "devalue@5.8.1": { + "integrity": "sha512-4CXDYRBGqN+57wVJkuXBYmpAVUSg3L6JAQa/DFqm238G73E1wuyc/JhGQJzN7vUf/CMphYau2zXbfWzDR5aTEw==" }, "devlop@1.1.0": { "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", @@ -8421,7 +8422,6 @@ "aria-query@5.3.1", "axobject-query", "clsx", - "devalue", "esm-env", "esrap", "is-reference@3.0.3", @@ -9627,9 +9627,11 @@ }, "packages/fedify": { "dependencies": [ + "jsr:@standard-schema/spec@^1.1.0", "jsr:@std/assert@0.226", "jsr:@std/url@~0.225.1", "npm:@multiformats/base-x@^4.0.1", + "npm:devalue@^5.8.1", "npm:fast-check@^3.22.0", "npm:fetch-mock@^12.5.2", "npm:json-canon@^1.0.1", @@ -9642,6 +9644,7 @@ "npm:@js-temporal/polyfill@~0.5.1", "npm:@jsr/std__assert@0.226", "npm:@types/node@^24.2.1", + "npm:devalue@^5.8.1", "npm:json-canon@^1.0.1", "npm:jsonld@9", "npm:miniflare@^4.20250523.0", diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 08ed094cd..e15c0a0dc 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -151,6 +151,7 @@ const MANUAL = { { text: "Pragmatics", link: "/manual/pragmatics.md" }, { text: "Key–value store", link: "/manual/kv.md" }, { text: "Message queue", link: "/manual/mq.md" }, + { text: "Background tasks", link: "/manual/tasks.md" }, { text: "Circuit breaker", link: "/manual/circuit-breaker.md" }, { text: "Integration", link: "/manual/integration.md" }, { text: "Migration", link: "/manual/migrate.md" }, diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md new file mode 100644 index 000000000..dd52e1ef4 --- /dev/null +++ b/docs/manual/tasks.md @@ -0,0 +1,247 @@ +Background tasks +================ + +*This API is available since Fedify 2.3.0.* + +Fedify already processes outgoing and incoming activities on background +workers through its [message queue](./mq.md). The custom background task API +generalizes the same pattern—enqueue work, return immediately, process the +payload on a separate worker—to arbitrary application-defined jobs: sending +digest e-mails, rebuilding timelines, fetching link previews, and so on. + +A task is *defined* once on the `Federation` (or `FederationBuilder`) object +with `~TaskRegistry.defineTask()`, and *dispatched* from any `Context` with +`~Context.enqueueTask()`. The payload is validated, serialized by Fedify, +delivered through a message queue, and handed—decoded and re-validated—to the +task's handler on a worker. + + +Defining a task +--------------- + +`~TaskRegistry.defineTask()` registers a named task and returns a handle that +`~Context.enqueueTask()` consumes. Every task requires a `schema`, a +[Standard Schema] (implemented by [Zod], [Valibot], [ArkType], and friends) +that validates the payload; the payload type is inferred from it, so handlers +and call sites are fully typed without manual type annotations: + +~~~~ typescript +import { z } from "zod"; + +const sendDigest = federation.defineTask("sendDigest", { + schema: z.object({ + userId: z.string(), + since: z.date(), + }), + handler: async (ctx, data) => { + // data is typed as { userId: string; since: Date } + const digest = await buildDigest(data.userId, data.since); + await sendEmail(data.userId, digest); + }, +}); +~~~~ + +Task names must be unique within a federation; defining the same name twice +throws a `TypeError`. + +[Standard Schema]: https://standardschema.dev/ +[Zod]: https://zod.dev/ +[Valibot]: https://valibot.dev/ +[ArkType]: https://arktype.io/ + + +Payload handling +---------------- + +Task payloads cross a message queue, so they are serialized on enqueue and +deserialized on dispatch. Fedify owns this codec—applications never encode +payloads themselves. The codec is built on [devalue], which means payloads +are not limited to JSON: `Date`, `Map`, `Set`, `URL`, `RegExp`, `bigint`, +typed arrays, circular references, and repeated references all round-trip +faithfully. + +Activity Vocabulary objects (`Note`, `Create`, `Person`, `Link`, and so on) +are also supported as payload values. Each vocabulary object is bridged +through expanded JSON-LD on the wire and comes back as a real instance, so +the handler can call its methods and getters as usual: + +~~~~ typescript +import { Note } from "@fedify/fedify/vocab"; +import { z } from "zod"; + +const indexNote = federation.defineTask("indexNote", { + schema: z.object({ + note: z.instanceof(Note), // an opaque instanceof leaf + indexedAt: z.date(), + }), + handler: async (ctx, data) => { + await searchIndex.add(data.note.id?.href, data.note.content?.toString()); + }, +}); +~~~~ + +The schema validates the *envelope* of the payload; vocabulary objects are +opaque `instanceof` leaves (e.g., `z.instanceof(Note)`), so no enormous +schema is needed. + +Validation runs twice: once at enqueue time, so a caller passing a wrong +shape fails fast at the call site, and once at dequeue time, which protects +against *schema drift*—a durable queue can hand a new deployment a payload +that an older deployment enqueued. A payload that fails dequeue-time +validation (or cannot be decoded at all) is dropped with an error log rather +than retried, because retrying cannot make it valid. + +[devalue]: https://github.com/sveltejs/devalue + + +Dispatching tasks +----------------- + +`~Context.enqueueTask()` validates the payload, serializes it, and enqueues +it. It returns as soon as the message is accepted by the queue: + +~~~~ typescript +await ctx.enqueueTask(sendDigest, { + userId: "alice", + since: new Date("2026-06-01T00:00:00Z"), +}); +~~~~ + +Passing a payload that does not match the task's schema is a compile-time +type error, and—for shapes the type system cannot catch—a runtime +`TypeError` at the call site. + +`~Context.enqueueTaskMany()` enqueues multiple payloads at once, using the +queue's bulk `~MessageQueue.enqueueMany()` operation when the backend +supports it and falling back to parallel single enqueues otherwise: + +~~~~ typescript +await ctx.enqueueTaskMany(sendDigest, users.map((u) => ({ + userId: u.id, + since: u.lastDigestAt, +}))); +~~~~ + +Both methods accept options: + +`delay` +: A `Temporal.DurationLike` to postpone processing, e.g., + `{ minutes: 30 }`. + +`orderingKey` +: Tasks with the same ordering key are processed sequentially (one at + a time), like the same option on the message queue layer. + +~~~~ typescript +await ctx.enqueueTask(sendDigest, payload, { + delay: { minutes: 30 }, + orderingKey: `digest:${payload.userId}`, +}); +~~~~ + + +Retry and error handling +------------------------ + +When a handler throws, Fedify consults the retry policy and re-enqueues the +message with an incremented attempt counter. The policy is resolved in this +order: + +1. The task's own `retryPolicy` passed to `~TaskRegistry.defineTask()`. +2. The federation-wide `~FederationOptions.taskRetryPolicy`. +3. The default: exponential backoff with a maximum of 10 attempts. + +When the queue backend reports `~MessageQueue.nativeRetrial`, Fedify rethrows +the error instead and lets the backend drive retries. + +A task can also register an `onError` callback, which is invoked with the +error and the decoded payload before a retry is scheduled—useful for +reporting or compensating actions: + +~~~~ typescript +const sendDigest = federation.defineTask("sendDigest", { + schema: digestSchema, + handler: async (ctx, data) => { + await sendEmail(data.userId, await buildDigest(data.userId, data.since)); + }, + retryPolicy: createExponentialBackoffPolicy({ maxAttempts: 3 }), + onError: async (ctx, error, data) => { + await reportFailure("sendDigest", data.userId, error); + }, +}); +~~~~ + +Two failure cases are *dropped without retry*, because retrying cannot help: +a message whose `taskName` has no registered handler (logged as a warning), +and a payload that cannot be decoded or fails dequeue-time validation +(logged as an error). + + +Queue routing and isolation +--------------------------- + +By default tasks share the outbox queue, so no extra configuration is needed +beyond a `queue` on `createFederation()`. For heavier workloads, tasks can +be isolated at two levels. + +A dedicated task queue, separate from activity delivery, is configured with +the `~FederationQueueOptions.task` slot: + +~~~~ typescript +const federation = createFederation({ + // ... + queue: { + inbox: new PostgresMessageQueue(sql, { channel: "inbox" }), + outbox: new PostgresMessageQueue(sql, { channel: "outbox" }), + task: new PostgresMessageQueue(sql, { channel: "task" }), // [!code highlight] + }, +}); +~~~~ + +A single task can also carry its own queue, which takes precedence over +everything else: + +~~~~ typescript +const transcodeVideo = federation.defineTask("transcodeVideo", { + schema: transcodeSchema, + handler: transcodeHandler, + queue: new PostgresMessageQueue(sql, { channel: "transcode" }), +}); +~~~~ + +The queue for a task is resolved in order: the per-task `queue`, then the +federation's `task` queue, then the outbox queue. Deployments that must +*not* silently share the outbox queue can opt out of the last step with +`~FederationOptions.taskQueueResolution`: + +~~~~ typescript +const federation = createFederation({ + // ... + taskQueueResolution: "strict", // no outbox fallback // [!code highlight] +}); +~~~~ + +Under `"strict"` resolution, enqueuing a task that has no queue throws +a `TypeError` instead of falling back. + +On the worker side, `~Federation.startQueue()` accepts `"task"` in its +`queue` option, so a dedicated worker process can consume only tasks: + +~~~~ typescript +await federation.startQueue(contextData, { queue: "task" }); +~~~~ + +A task that falls back to the outbox queue needs no dedicated worker; the +outbox worker dispatches every message by its type regardless of which queue +delivered it. + + +Limitations +----------- + +The current API intentionally ships without deduplication, task-specific +OpenTelemetry spans and metrics, cron-style periodic scheduling, result +backends, and per-task priority. Some of these are planned as follow-ups; +see the [tracking issue]. + +[tracking issue]: https://github.com/fedify-dev/fedify/issues/206 diff --git a/packages/fedify/deno.json b/packages/fedify/deno.json index 825c42cf9..910c4cd2d 100644 --- a/packages/fedify/deno.json +++ b/packages/fedify/deno.json @@ -15,8 +15,10 @@ }, "imports": { "@multiformats/base-x": "npm:@multiformats/base-x@^4.0.1", + "@standard-schema/spec": "jsr:@standard-schema/spec@^1.1.0", "@std/assert": "jsr:@std/assert@^0.226.0", "@std/url": "jsr:@std/url@^0.225.1", + "devalue": "npm:devalue@^5.8.1", "fast-check": "npm:fast-check@^3.22.0", "fetch-mock": "npm:fetch-mock@^12.5.2", "json-canon": "npm:json-canon@^1.0.1", diff --git a/packages/fedify/package.json b/packages/fedify/package.json index f10d75ccf..9549e3f68 100644 --- a/packages/fedify/package.json +++ b/packages/fedify/package.json @@ -147,7 +147,9 @@ "@opentelemetry/sdk-metrics": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:", "@opentelemetry/semantic-conventions": "catalog:", + "@standard-schema/spec": "catalog:", "byte-encodings": "catalog:", + "devalue": "^5.8.1", "es-toolkit": "catalog:", "json-canon": "^1.0.1", "jsonld": "^9.0.0", diff --git a/packages/fedify/src/federation/builder.ts b/packages/fedify/src/federation/builder.ts index 1503ccf7b..e237d9c2e 100644 --- a/packages/fedify/src/federation/builder.ts +++ b/packages/fedify/src/federation/builder.ts @@ -67,6 +67,13 @@ import type { CollectionCallbacks, CustomCollectionCallbacks, } from "./handler.ts"; +import type { StandardSchemaV1 } from "@standard-schema/spec"; +import type { + TaskDefinition, + TaskDefinitionInternal, + TaskDefinitionOptions, + TaskHandler, +} from "./tasks/mod.ts"; export const ACTOR_ALIAS_PREFIX = "actorAlias:"; @@ -181,6 +188,7 @@ export class FederationBuilderImpl TContextData > >; + taskDefinitions: Record>; /** * Symbol registry for unique identification of unnamed symbols. @@ -193,6 +201,7 @@ export class FederationBuilderImpl this.objectTypeIds = {}; this.collectionCallbacks = {}; this.collectionTypeIds = {}; + this.taskDefinitions = {}; } /** @@ -258,6 +267,7 @@ export class FederationBuilderImpl f.unverifiedActivityHandler = this.unverifiedActivityHandler; f.outboxPermanentFailureHandler = this.outboxPermanentFailureHandler; f.idempotencyStrategy = this.idempotencyStrategy; + f.taskDefinitions = { ...this.taskDefinitions }; return f; } @@ -593,6 +603,25 @@ export class FederationBuilderImpl this.webFingerLinksDispatcher = dispatcher; } + defineTask( + name: string, + options: TaskDefinitionOptions, + ): TaskDefinition> { + if (name in this.taskDefinitions) { + throw new TypeError(`Task ${JSON.stringify(name)} is already defined.`); + } + this.taskDefinitions[name] = { + name, + schema: options.schema, + handler: options.handler as TaskHandler, + onError: options + .onError as TaskDefinitionInternal["onError"], + retryPolicy: options.retryPolicy, + queue: options.queue, + }; + return { name, schema: options.schema }; + } + /** * The RFC 6570 template-literal `path` overloads were removed for * type-checking efficiency, so the URI variable types can no longer be diff --git a/packages/fedify/src/federation/context.ts b/packages/fedify/src/federation/context.ts index 3a95bfdde..8375e6a76 100644 --- a/packages/fedify/src/federation/context.ts +++ b/packages/fedify/src/federation/context.ts @@ -22,6 +22,7 @@ import type { JsonValue, NodeInfo } from "../nodeinfo/types.ts"; import type { GetKeyOwnerOptions } from "../sig/owner.ts"; import type { ConstructorWithTypeId, Federation } from "./federation.ts"; import type { SenderKeyPair } from "./send.ts"; +import type { TaskDefinition, TaskEnqueueOptions } from "./tasks/mod.ts"; /** * A context. @@ -429,6 +430,52 @@ export interface Context { options?: RouteActivityOptions, ): Promise; + /** + * Enqueues a custom background task. The payload is validated against + * the task's schema, serialized, and processed by the task's handler on + * a background worker. + * + * @example + * ``` typescript + * await ctx.enqueueTask(sendDigest, { userId: "alice" }); + * ``` + * + * @template TData The type of the task payload, inferred from the task's + * schema. + * @param task The handle returned by {@link TaskRegistry.defineTask}. + * @param data The task payload. It is validated against the task's + * schema before being enqueued. + * @param options Options for enqueuing the task. + * @throws {TypeError} If no message queue is configured for tasks, or if + * the payload fails schema validation. + * @since 2.3.0 + */ + enqueueTask( + task: TaskDefinition, + data: TData, + options?: TaskEnqueueOptions, + ): Promise; + + /** + * Enqueues multiple payloads for a custom background task at once. + * Uses the queue's bulk enqueue operation when available, falling back + * to parallel single enqueues. + * @template TData The type of the task payload, inferred from the task's + * schema. + * @param task The handle returned by {@link TaskRegistry.defineTask}. + * @param payloads The task payloads. Each is validated against the + * task's schema before being enqueued. + * @param options Options for enqueuing the tasks. + * @throws {TypeError} If no message queue is configured for tasks, or if + * a payload fails schema validation. + * @since 2.3.0 + */ + enqueueTaskMany( + task: TaskDefinition, + payloads: readonly TData[], + options?: TaskEnqueueOptions, + ): Promise; + /** * Builds the URI of a collection of objects with the given name and values. * @param name The name of the collection, which can be a string or a symbol. diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index e1d5e9d40..dbb5231f1 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -49,6 +49,7 @@ import type { import type { MessageQueue } from "./mq.ts"; import type { Message } from "./queue.ts"; import type { RetryPolicy } from "./retry.ts"; +import type { TaskRegistry } from "./tasks/mod.ts"; /** * Options for {@link Federation.startQueue} method. @@ -62,11 +63,11 @@ export interface FederationStartQueueOptions { /** * Starts the task worker only for the specified queue. If unspecified, - * which is the default, the task worker starts for all three queues: - * inbox, outbox, and fanout. + * which is the default, the task worker starts for all four queues: + * inbox, outbox, fanout, and task. * @since 1.3.0 */ - queue?: "inbox" | "outbox" | "fanout"; + queue?: "inbox" | "outbox" | "fanout" | "task"; } /** @@ -74,7 +75,7 @@ export interface FederationStartQueueOptions { * @template TContextData The context data to pass to the {@link Context}. * @since 1.6.0 */ -export interface Federatable { +export interface Federatable extends TaskRegistry { /** * Registers a NodeInfo dispatcher. * @param path The URI path pattern for the NodeInfo dispatcher. The syntax @@ -1080,6 +1081,28 @@ export interface FederationOptions { */ inboxRetryPolicy?: RetryPolicy; + /** + * The retry policy for processing custom background tasks. By default, + * this uses an exponential backoff strategy with a maximum of 10 attempts + * and a maximum delay of 12 hours. A per-task retry policy + * ({@link TaskDefinitionOptions.retryPolicy}) overrides this. + * @since 2.3.0 + */ + taskRetryPolicy?: RetryPolicy; + + /** + * How a queue is resolved for a custom background task when neither + * a per-task queue ({@link TaskDefinitionOptions.queue}) nor a dedicated + * task queue ({@link FederationQueueOptions.task}) is configured. + * + * - `"fallback"` (the default): the task is routed to the outbox queue. + * - `"strict"`: no fallback; enqueuing the task throws instead of + * silently sharing the outbox queue. + * @default `"fallback"` + * @since 2.3.0 + */ + taskQueueResolution?: "fallback" | "strict"; + /** * Activity transformers that are applied to outgoing activities. It is * useful for adjusting outgoing activities to satisfy some ActivityPub diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 5bcbfc3c9..ee96cd8df 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -57,13 +57,6 @@ import type { ActivityTransformer } from "../compat/types.ts"; import { getNodeInfo, type GetNodeInfoOptions } from "../nodeinfo/client.ts"; import { handleNodeInfo, handleNodeInfoJrd } from "../nodeinfo/handler.ts"; import type { JsonValue, NodeInfo } from "../nodeinfo/types.ts"; -import { - type BenchmarkMetricReader, - type BenchmarkTriggerOptions, - createBenchmarkMeterProvider, - handleBenchmarkStats, - handleBenchmarkTrigger, -} from "./bench.ts"; import { type HttpMessageSignaturesSpec, type HttpMessageSignaturesSpecDeterminer, @@ -87,6 +80,13 @@ import { getKeyOwner, type GetKeyOwnerOptions } from "../sig/owner.ts"; import { hasProofLike, signObject, verifyObject } from "../sig/proof.ts"; import { getAuthenticatedDocumentLoader } from "../utils/docloader.ts"; import { kvCache } from "../utils/kv-cache.ts"; +import { + type BenchmarkMetricReader, + type BenchmarkTriggerOptions, + createBenchmarkMeterProvider, + handleBenchmarkStats, + handleBenchmarkTrigger, +} from "./bench.ts"; import { ACTOR_ALIAS_PREFIX, FederationBuilderImpl } from "./builder.ts"; import type { OutboxErrorHandler } from "./callback.ts"; import { @@ -158,6 +158,7 @@ import type { Message, OutboxMessage, SenderKeyJwkPair, + TaskMessage, } from "./queue.ts"; import { createExponentialBackoffPolicy, type RetryPolicy } from "./retry.ts"; import { @@ -166,8 +167,13 @@ import { SendActivityError, type SenderKeyPair, } from "./send.ts"; -import { handleWebFinger } from "./webfinger.ts"; +import { + TaskCodec, + type TaskDefinition, + type TaskEnqueueOptions, +} from "./tasks/mod.ts"; import { hasMalformedKnownTemporalLiteral } from "./temporal.ts"; +import { handleWebFinger } from "./webfinger.ts"; const circuitBreakerCasWarningKvStores = new WeakSet(); let nextQueueDepthGaugeSourceId = 0; @@ -484,6 +490,14 @@ export interface FederationQueueOptions { * {@link Context.sendActivity} calls. */ readonly fanout?: MessageQueue; + + /** + * The message queue for custom background tasks. If not provided, + * tasks are routed to the outbox queue (unless + * {@link FederationOptions.taskQueueResolution} is `"strict"`). + * @since 2.3.0 + */ + readonly task?: MessageQueue; } /** @@ -577,9 +591,11 @@ export class FederationImpl inboxQueue?: MessageQueue; outboxQueue?: MessageQueue; fanoutQueue?: MessageQueue; + taskQueue?: MessageQueue; inboxQueueStarted: boolean; outboxQueueStarted: boolean; fanoutQueueStarted: boolean; + taskQueueStarted: boolean; manuallyStartQueue: boolean; origin?: FederationOrigin; documentLoaderFactory: DocumentLoaderFactory; @@ -593,6 +609,8 @@ export class FederationImpl skipSignatureVerification: boolean; outboxRetryPolicy: RetryPolicy; inboxRetryPolicy: RetryPolicy; + taskRetryPolicy: RetryPolicy; + taskQueueResolution: "fallback" | "strict"; circuitBreaker?: CircuitBreaker; activityTransformers: readonly ActivityTransformer[]; _tracerProvider: TracerProvider | undefined; @@ -661,14 +679,19 @@ export class FederationImpl this.inboxQueue = undefined; this.outboxQueue = undefined; this.fanoutQueue = undefined; + this.taskQueue = undefined; } else if ("enqueue" in options.queue && "listen" in options.queue) { this.inboxQueue = options.queue; this.outboxQueue = options.queue; this.fanoutQueue = options.queue; + // A bare queue leaves taskQueue undefined; tasks are served through + // the outboxQueue fallback. + this.taskQueue = undefined; } else { this.inboxQueue = options.queue.inbox; this.outboxQueue = options.queue.outbox; this.fanoutQueue = options.queue.fanout; + this.taskQueue = options.queue.task; } if (options.circuitBreaker !== false && this.outboxQueue != null) { this.circuitBreaker = new CircuitBreaker({ @@ -699,6 +722,7 @@ export class FederationImpl this.inboxQueueStarted = false; this.outboxQueueStarted = false; this.fanoutQueueStarted = false; + this.taskQueueStarted = false; this.manuallyStartQueue = options.manuallyStartQueue ?? false; if (options.origin != null) { if (typeof options.origin === "string") { @@ -877,6 +901,9 @@ export class FederationImpl createExponentialBackoffPolicy(); this.inboxRetryPolicy = options.inboxRetryPolicy ?? createExponentialBackoffPolicy(); + this.taskRetryPolicy = options.taskRetryPolicy ?? + createExponentialBackoffPolicy(); + this.taskQueueResolution = options.taskQueueResolution ?? "fallback"; this.activityTransformers = options.activityTransformers ?? getDefaultActivityTransformers(); this._tracerProvider = options.tracerProvider; @@ -929,12 +956,24 @@ export class FederationImpl return this.tracerProvider.getTracer(metadata.name, metadata.version); } + resolveTaskQueue(taskName: string): MessageQueue | undefined { + const def = this.taskDefinitions[taskName]; + const resolved = def?.queue ?? this.taskQueue; + if (resolved != null) return resolved; + return this.taskQueueResolution === "strict" ? undefined : this.outboxQueue; + } + async _startQueueInternal( ctxData: TContextData, signal?: AbortSignal, queue?: keyof FederationQueueOptions, ): Promise { - if (this.inboxQueue == null && this.outboxQueue == null) return; + if ( + this.inboxQueue == null && this.outboxQueue == null && + this.fanoutQueue == null && this.taskQueue == null + ) { + return; + } const logger = getLogger(["fedify", "federation", "queue"]); const promises: Promise[] = []; if ( @@ -981,6 +1020,23 @@ export class FederationImpl ), ); } + if ( + this.taskQueue != null && + this.taskQueue !== this.inboxQueue && + this.taskQueue !== this.outboxQueue && + this.taskQueue !== this.fanoutQueue && + (queue == null || queue === "task") && + !this.taskQueueStarted + ) { + logger.debug("Starting a task worker."); + this.taskQueueStarted = true; + promises.push( + this.taskQueue.listen( + (msg) => this.processQueuedTask(ctxData, msg), + { signal }, + ), + ); + } await Promise.all(promises); } @@ -1161,6 +1217,8 @@ export class FederationImpl ); }, ); + } else if (message.type === "task") { + await this.#listenTaskMessage(contextData, message); } }); } @@ -2056,6 +2114,82 @@ export class FederationImpl ); } + async #listenTaskMessage( + contextData: TContextData, + message: TaskMessage, + ): Promise { + const logger = getLogger(["fedify", "federation", "task"]); + const def = this.taskDefinitions[message.taskName]; + if (def == null) { + // Unknown task: a handler won't appear by retrying. Drop and log. + logger.warn( + "Received a custom task {taskName} with no registered handler; " + + "dropping.", + { taskName: message.taskName }, + ); + return; + } + const context = this.#createContext(new URL(message.baseUrl), contextData); + let data: unknown; + try { + // decode() deserializes then re-validates at the dequeue boundary + // (drift protection): a durable queue can hand a new deploy a payload + // an old deploy enqueued. + data = await context.codec.decode(def.schema, message.data); + } catch (error) { + // A malformed or incompatible payload won't succeed by retrying. + logger.error( + "Custom task {taskName} payload could not be decoded or validated; " + + "dropping:\n{error}", + { taskName: message.taskName, error }, + ); + return; + } + try { + await def.handler(context, data); + } catch (error) { + if (def.onError != null) { + try { + await def.onError(context, error, data); + } catch (onErrorError) { + logger.error( + "onError for custom task {taskName} threw:\n{error}", + { taskName: message.taskName, error: onErrorError }, + ); + } + } + const queue = this.resolveTaskQueue(def.name); + if (queue?.nativeRetrial) throw error; // the backend owns retries + const retryPolicy = def.retryPolicy ?? this.taskRetryPolicy; + const delay = retryPolicy({ + elapsedTime: Temporal.Instant.from(message.started) + .until(Temporal.Now.instant()), + attempts: message.attempt, + }); + if (delay != null && queue != null) { + logger.error( + "Custom task {taskName} failed (attempt #{attempt}); retry...:" + + "\n{error}", + { taskName: message.taskName, attempt: message.attempt, error }, + ); + const retryMessage = { + ...message, + attempt: message.attempt + 1, + } satisfies TaskMessage; + await queue.enqueue(retryMessage, { + delay: clampNegativeDelay(delay), + orderingKey: message.orderingKey, + }); + } else { + logger.error( + "Custom task {taskName} failed after {attempt} attempts; giving " + + "up:\n{error}", + { taskName: message.taskName, attempt: message.attempt, error }, + ); + } + } + } + startQueue( contextData: TContextData, options: FederationStartQueueOptions = {}, @@ -2935,6 +3069,7 @@ export class ContextImpl implements Context { readonly documentLoader: DocumentLoader; readonly contextLoader: DocumentLoader; readonly invokedFromActorKeyPairsDispatcher?: { identifier: string }; + #codec?: TaskCodec; constructor( { @@ -2955,6 +3090,16 @@ export class ContextImpl implements Context { invokedFromActorKeyPairsDispatcher; } + /** + * A {@link TaskCodec} bound to this context's loaders, used to encode + * and decode custom task payloads. Lazily created and cached so a context + * that never enqueues or dispatches a task pays nothing. + * @internal + */ + get codec(): TaskCodec { + return this.#codec ??= new TaskCodec(this); + } + clone(data: TContextData): Context { return new ContextImpl({ url: this.url, @@ -3491,6 +3636,75 @@ export class ContextImpl implements Context { }); } + async enqueueTask( + task: TaskDefinition, + data: TData, + options: TaskEnqueueOptions = {}, + ): Promise { + await this.#enqueueTasks(task, [data], options); + } + + async enqueueTaskMany( + task: TaskDefinition, + payloads: readonly TData[], + options: TaskEnqueueOptions = {}, + ): Promise { + await this.#enqueueTasks(task, payloads, options); + } + + async #enqueueTasks( + task: TaskDefinition, + items: readonly TData[], + options: TaskEnqueueOptions, + ): Promise { + const queue = this.federation.resolveTaskQueue(task.name); + if (queue == null) { + throw new TypeError( + "No message queue is configured for tasks; pass `queue` to " + + "createFederation() or to defineTask().", + ); + } + const delay = options.delay == null + ? undefined + : Temporal.Duration.from(options.delay); + // Encode in parallel: `enqueueTaskMany` is the bulk path, and the enqueue + // below already parallelizes, so serial encoding would be the bottleneck. + // `map` preserves order, and a rejected encode (validation failure) rejects + // the whole batch before anything is enqueued, keeping fail-fast intact. + const messages: TaskMessage[] = await Promise.all( + items.map(this.#enqueueSingular(task, options)), + ); + const enqueueOptions = { delay, orderingKey: options.orderingKey }; + if (messages.length === 1) { + await queue.enqueue(messages[0], enqueueOptions); + } else if (queue.enqueueMany != null) { + await queue.enqueueMany(messages, enqueueOptions); + } else { + await Promise.all(messages.map((m) => queue.enqueue(m, enqueueOptions))); + } + } + + #enqueueSingular = ( + task: TaskDefinition, + options: TaskEnqueueOptions, + ) => + async (data: TData): Promise => { + const encoded = await this.codec.encode(task.schema, data); + const carrier: Record = {}; + propagation.inject(context.active(), carrier); + return { + type: "task", + id: crypto.randomUUID(), + baseUrl: this.origin, + taskName: task.name, + data: encoded, + started: Temporal.Now.instant().toString(), + attempt: 0, + orderingKey: options.orderingKey, + traceContext: carrier, + }; + }; + sendActivity( sender: | SenderKeyPair diff --git a/packages/fedify/src/federation/mod.ts b/packages/fedify/src/federation/mod.ts index 5a87fdf0f..6a3bb1f6e 100644 --- a/packages/fedify/src/federation/mod.ts +++ b/packages/fedify/src/federation/mod.ts @@ -25,6 +25,13 @@ export * from "./mq.ts"; export type { Message } from "./queue.ts"; export * from "./retry.ts"; export * from "./router.ts"; +export type { + TaskDefinition, + TaskDefinitionOptions, + TaskEnqueueOptions, + TaskHandler, + TaskRegistry, +} from "./tasks/mod.ts"; export { SendActivityError, type SenderKeyPair } from "./send.ts"; export { handleWebFinger, diff --git a/packages/fedify/src/federation/queue.ts b/packages/fedify/src/federation/queue.ts index 36f35ad02..5bef2abfc 100644 --- a/packages/fedify/src/federation/queue.ts +++ b/packages/fedify/src/federation/queue.ts @@ -12,7 +12,11 @@ export interface SenderKeyJwkPair { * type. * @since 1.6.0 */ -export type Message = FanoutMessage | OutboxMessage | InboxMessage; +export type Message = + | FanoutMessage + | OutboxMessage + | InboxMessage + | TaskMessage; export interface FanoutMessage { readonly type: "fanout"; @@ -71,6 +75,25 @@ export interface OutboxMessage { readonly traceContext: Readonly>; } +/** + * A message that carries a custom background task. Every field is + * a string, number, or plain record so that the message survives both + * JSON serialization and structured clone on every queue backend. + * @since 2.3.0 + */ +export interface TaskMessage { + readonly type: "task"; + readonly id: ReturnType; + readonly baseUrl: string; + readonly taskName: string; + /** devalue-encoded task data; vocab objects bridged to expanded JSON-LD. */ + readonly data: string; + readonly started: string; + readonly attempt: number; + readonly orderingKey?: string; + readonly traceContext: Readonly>; +} + export interface InboxMessage { readonly type: "inbox"; readonly id: ReturnType; diff --git a/packages/fedify/src/federation/tasks/codec-fn.test.ts b/packages/fedify/src/federation/tasks/codec-fn.test.ts new file mode 100644 index 000000000..9cc4b9279 --- /dev/null +++ b/packages/fedify/src/federation/tasks/codec-fn.test.ts @@ -0,0 +1,230 @@ +import { mockDocumentLoader, test } from "@fedify/fixture"; +import { Create, Link, Note, Person } from "@fedify/vocab"; +import type { StandardSchemaV1 } from "@standard-schema/spec"; +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; +import { + deserializeTaskData, + serializeTaskData, + validateTaskData, +} from "./codec-fn.ts"; + +const loaders = { + contextLoader: mockDocumentLoader, + documentLoader: mockDocumentLoader, +}; + +function makeSchema( + check: (data: unknown) => data is T, +): StandardSchemaV1 { + return { + "~standard": { + version: 1, + vendor: "fedify-test", + validate(value: unknown) { + return check(value) + ? { value } + : { issues: [{ message: "Invalid task data." }] }; + }, + }, + }; +} + +test("serializeTaskData() / deserializeTaskData()", async (t) => { + const note = new Note({ + id: new URL("https://example.com/notes/1"), + content: "Hello, world!", + }); + const person = new Person({ + id: new URL("https://example.com/users/alice"), + name: "Alice", + }); + const link = new Link({ + href: new URL("https://example.com/"), + mediaType: "text/html", + }); + const create = new Create({ + id: new URL("https://example.com/activities/1"), + actor: new URL("https://example.com/users/alice"), + object: note, + }); + + await t.step("round-trips a mixed payload", async () => { + const payload = { + note, + when: new Date("2026-01-02T03:04:05Z"), + big: 1234567890123456789n, + url: new URL("https://example.com/some/path"), + list: [person, link], + map: new Map([["create", create], ["n", 42]]), + set: new Set([1, 2, 3]), + nested: { create }, + }; + const encoded = await serializeTaskData(payload, mockDocumentLoader); + strictEqual(typeof encoded, "string"); + const decoded = await deserializeTaskData(encoded, loaders) as Record< + string, + unknown + >; + ok(decoded.note instanceof Note); + strictEqual(decoded.note.content?.toString(), "Hello, world!"); + strictEqual(decoded.note.id?.href, "https://example.com/notes/1"); + ok(decoded.when instanceof Date); + strictEqual(decoded.when.toISOString(), "2026-01-02T03:04:05.000Z"); + strictEqual(decoded.big, 1234567890123456789n); + ok(decoded.url instanceof URL); + strictEqual(decoded.url.href, "https://example.com/some/path"); + const list = decoded.list as unknown[]; + ok(list[0] instanceof Person); + strictEqual(list[0].name?.toString(), "Alice"); + ok(list[1] instanceof Link); + strictEqual(list[1].href?.href, "https://example.com/"); + const map = decoded.map as Map; + ok(map.get("create") instanceof Create); + strictEqual(map.get("n"), 42); + deepStrictEqual(decoded.set, new Set([1, 2, 3])); + const nested = decoded.nested as Record; + ok(nested.create instanceof Create); + const nestedObject = await nested.create.getObject({ + documentLoader: mockDocumentLoader, + contextLoader: mockDocumentLoader, + }); + ok(nestedObject instanceof Note); + strictEqual(nestedObject.content?.toString(), "Hello, world!"); + }); + + await t.step( + "encodes vocab objects in expand form (no @context)", + async () => { + const encoded = await serializeTaskData({ note }, mockDocumentLoader); + ok(!encoded.includes("@context")); + }, + ); + + await t.step("leaves a non-vocab payload untouched", async () => { + const payload = { + text: "plain", + n: 1, + flag: true, + nothing: null, + when: new Date("2026-06-10T00:00:00Z"), + list: [1, "two", 3n], + }; + const encoded = await serializeTaskData(payload, mockDocumentLoader); + const decoded = await deserializeTaskData(encoded, loaders); + deepStrictEqual(decoded, payload); + }); + + await t.step("throws on a malformed wire string", async () => { + // deserializeTaskData() throws synchronously on a malformed wire string + // (devalue's parse() runs before the first await); the async wrapper + // funnels both sync throws and rejections into one assertion. + await rejects(async () => await deserializeTaskData("garbage", loaders)); + }); + + await t.step("preserves circular and repeated references", async () => { + const shared = new Note({ content: "shared" }); + interface Cyclic { + name: string; + self?: Cyclic; + notes: Note[]; + } + const payload: Cyclic = { name: "root", notes: [shared, shared] }; + payload.self = payload; + const encoded = await serializeTaskData(payload, mockDocumentLoader); + const decoded = await deserializeTaskData(encoded, loaders) as Cyclic; + strictEqual(decoded.self, decoded); + ok(decoded.notes[0] instanceof Note); + strictEqual(decoded.notes[0], decoded.notes[1]); + strictEqual(decoded.notes[0].content?.toString(), "shared"); + }); + + await t.step("preserves a cycle through an array", async () => { + const list: unknown[] = ["head"]; + list.push(list); + const encoded = await serializeTaskData({ list }, mockDocumentLoader); + const decoded = await deserializeTaskData(encoded, loaders) as { + list: unknown[]; + }; + strictEqual(decoded.list[0], "head"); + strictEqual(decoded.list[1], decoded.list); + }); + + await t.step("preserves cycles re-entering at a Map and a Set", async () => { + // The cycle must re-enter at the Map/Set *itself* (not at a plain + // object) to exercise their pre-registration in the reviver. + const set = new Set(); + set.add({ set }); + const map = new Map(); + map.set("entry", { map }); + const encoded = await serializeTaskData({ set, map }, mockDocumentLoader); + const decoded = await deserializeTaskData(encoded, loaders) as { + set: Set<{ set: Set }>; + map: Map }>; + }; + const [member] = decoded.set; + strictEqual(member.set, decoded.set); + strictEqual(decoded.map.get("entry")?.map, decoded.map); + }); +}); + +test("validateTaskData()", async (t) => { + interface Envelope { + note: Note; + title: string; + } + const schema = makeSchema( + (data): data is Envelope => + typeof data === "object" && data != null && + (data as Envelope).note instanceof Note && + typeof (data as Envelope).title === "string", + ); + + await t.step("accepts a payload with a vocab instanceof leaf", async () => { + const payload = { + note: new Note({ content: "Hi" }), + title: "greeting", + }; + const validated = await validateTaskData(schema, payload); + deepStrictEqual(validated, payload); + }); + + await t.step("rejects a wrong-shaped payload", async () => { + await rejects( + () => validateTaskData(schema, { note: "not a Note", title: 42 }), + { name: "TypeError", message: /Task data failed schema validation/ }, + ); + }); + + await t.step("supports async validation", async () => { + const asyncSchema: StandardSchemaV1 = { + "~standard": { + version: 1, + vendor: "fedify-test", + validate: (value: unknown) => + Promise.resolve( + typeof value === "number" + ? { value } + : { issues: [{ message: "not a number" }] }, + ), + }, + }; + strictEqual(await validateTaskData(asyncSchema, 42), 42); + await rejects(() => validateTaskData(asyncSchema, "nope")); + }); + + await t.step( + "round-trip then validate (same schema on both sides)", + async () => { + const payload = { + note: new Note({ content: "Hi" }), + title: "greeting", + }; + const encoded = await serializeTaskData(payload, mockDocumentLoader); + const decoded = await deserializeTaskData(encoded, loaders); + const validated = await validateTaskData(schema, decoded); + ok(validated.note instanceof Note); + strictEqual(validated.title, "greeting"); + strictEqual(validated.note.content?.toString(), "Hi"); + }, + ); +}); diff --git a/packages/fedify/src/federation/tasks/codec-fn.ts b/packages/fedify/src/federation/tasks/codec-fn.ts new file mode 100644 index 000000000..0f181d7fc --- /dev/null +++ b/packages/fedify/src/federation/tasks/codec-fn.ts @@ -0,0 +1,188 @@ +/** + * Serializes custom-task payloads with [devalue], bridging Activity + * Vocabulary objects (`Note`, `Create`, `Person`, `Link`, and so on) through + * JSON-LD. + * + * Vocabulary objects keep their state in private fields, so devalue cannot + * serialize them directly. devalue's custom-type hook (a reducer on encode, + * a reviver on decode) carries each object as JSON-LD without writing a + * marker into the payload. Encoding uses the *expand* JSON-LD form, which + * has no `@context`, so decoding dereferences nothing and never touches the + * network. + * + * [devalue]: https://github.com/sveltejs/devalue + * + * @module + */ +import { Link, Object as APObject } from "@fedify/vocab"; +import type { DocumentLoader } from "@fedify/vocab-runtime"; +import type { TracerProvider } from "@opentelemetry/api"; +import type { StandardSchemaV1 } from "@standard-schema/spec"; +import { parse, stringifyAsync } from "devalue"; + +/** Which `fromJsonLd` entry point rebuilds a given vocabulary object. */ +type VocabKind = "object" | "link"; + +/** A vocabulary object reduced to its wire form: a kind tag plus JSON-LD. */ +interface VocabWire { + readonly kind: VocabKind; + readonly jsonLd: unknown; +} + +/** + * The loaders a worker {@link Context} already exposes; both decode passes + * use them. + * @internal + */ +export interface TaskCodecLoaders { + readonly contextLoader?: DocumentLoader; + readonly documentLoader?: DocumentLoader; + readonly tracerProvider?: TracerProvider; + readonly baseUrl?: URL; +} + +const isVocab = (value: unknown): value is APObject | Link => + value instanceof APObject || value instanceof Link; + +const isPlainObject = (value: unknown): value is Record => + value === null || typeof value !== "object" + ? false + : globalThis.Object.getPrototypeOf(value) === + globalThis.Object.prototype; + +/** Reduce a vocabulary object to expanded JSON-LD (no `@context`). */ +const vocabToJsonLd = async ( + value: APObject | Link, + contextLoader: DocumentLoader, +): Promise => ({ + kind: value instanceof Link ? "link" : "object", + jsonLd: await value.toJsonLd({ format: "expand", contextLoader }), +}); + +/** Rebuild a vocabulary object from its wire form. */ +const vocabFromJsonLd = ( + { kind, jsonLd }: VocabWire, + loaders: TaskCodecLoaders, +): Promise => + kind === "link" + ? Link.fromJsonLd(jsonLd, loaders) + : APObject.fromJsonLd(jsonLd, loaders); + +/** + * Encodes a task payload to a devalue string. + * + * The reducer is deliberately a plain function, not an `async` one: + * `stringifyAsync` treats a truthy return as a match and awaits it. An + * `async` reducer would return a promise for *every* node, which is always + * truthy, so it would "match" non-vocab values too. The plain + * `isVocab(v) && …` form returns a synchronous `false` for non-vocab nodes + * and the `toJsonLd()` promise only for vocab ones. + * + * @internal + */ +export const serializeTaskData = ( + data: unknown, + contextLoader: DocumentLoader, +): Promise => + stringifyAsync(data, { + Vocab: (value: unknown) => + isVocab(value) && vocabToJsonLd(value, contextLoader), + }); + +/** + * A vocabulary object parked by the synchronous decode reviver, held until + * the async {@link reviveVocab} pass can `fromJsonLd()` it back into an + * instance. + */ +class VocabHolder implements VocabWire { + constructor(readonly kind: VocabKind, readonly jsonLd: unknown) {} + static from = ({ kind, jsonLd }: VocabWire) => new VocabHolder(kind, jsonLd); +} + +/** + * Second decode pass: replace every parked holder with a real instance. + * + * devalue preserves circular and repeated references, so the walker keeps + * a `seen` map from each visited container to its revived counterpart. + * Containers are registered *before* their contents are walked; a cycle + * therefore resolves to the (still-filling) revived container instead of + * recursing forever, and a repeated reference revives to the same instance. + */ +function reviveVocab( + loaders: TaskCodecLoaders, +): (node: unknown) => Promise { + const seen = new Map(); + return async function inner(node: unknown): Promise { + if (node === null || typeof node !== "object") return node; + if (seen.has(node)) return seen.get(node); + if (node instanceof VocabHolder) { + const revived = await vocabFromJsonLd(node, loaders); + seen.set(node, revived); + return revived; + } + if (Array.isArray(node)) { + const out: unknown[] = []; + seen.set(node, out); + out.push(...await Array.fromAsync(node, inner)); + return out; + } + if (node instanceof Map) { + const out = new Map(); + seen.set(node, out); + for (const [k, v] of node) out.set(await inner(k), await inner(v)); + return out; + } + if (node instanceof Set) { + const out = new Set(); + seen.set(node, out); + for (const v of await Array.fromAsync(node, inner)) out.add(v); + return out; + } + if (isPlainObject(node)) { + const out: Record = {}; + seen.set(node, out); + for (const [k, v] of globalThis.Object.entries(node)) { + out[k] = await inner(v); + } + return out; + } + return node; // Date / URL / RegExp and the like — devalue handled them + }; +} + +/** + * Decodes a devalue string back to a task payload. + * + * Two passes are unavoidable: `parse` revivers are synchronous while + * `fromJsonLd()` is async. The reviver only parks each object; + * {@link reviveVocab} then walks the result and awaits `fromJsonLd()`. + * + * @internal + */ +export const deserializeTaskData = ( + raw: string, + loaders: TaskCodecLoaders, +): Promise => + reviveVocab(loaders)( + parse(raw, { + Vocab: ({ kind, jsonLd }: VocabWire) => new VocabHolder(kind, jsonLd), + }), + ); + +/** + * Validates `data` through the vendor-agnostic + * [Standard Schema](https://standardschema.dev/) interface. + * @internal + */ +export const validateTaskData = async ( + schema: S, + data: unknown, +): Promise> => { + const result = await schema["~standard"].validate(data); + if (result.issues) { + throw new TypeError( + `Task data failed schema validation: ${JSON.stringify(result.issues)}`, + ); + } + return result.value; +}; diff --git a/packages/fedify/src/federation/tasks/codec.test.ts b/packages/fedify/src/federation/tasks/codec.test.ts new file mode 100644 index 000000000..d7dd4b636 --- /dev/null +++ b/packages/fedify/src/federation/tasks/codec.test.ts @@ -0,0 +1,284 @@ +import { mockDocumentLoader, test } from "@fedify/fixture"; +import { Create, Link, Note, Person } from "@fedify/vocab"; +import type { StandardSchemaV1 } from "@standard-schema/spec"; +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; +import TaskCodec from "./codec.ts"; + +const loaders = { + contextLoader: mockDocumentLoader, + documentLoader: mockDocumentLoader, +}; + +const codec = new TaskCodec(loaders); + +function makeSchema( + check: (data: unknown) => data is T, +): StandardSchemaV1 { + return { + "~standard": { + version: 1, + vendor: "fedify-test", + validate(value: unknown) { + return check(value) + ? { value } + : { issues: [{ message: "Invalid task data." }] }; + }, + }, + }; +} + +test("TaskCodec (fresh instance per operation)", async (t) => { + const note = new Note({ + id: new URL("https://example.com/notes/1"), + content: "Hello, world!", + }); + const person = new Person({ + id: new URL("https://example.com/users/alice"), + name: "Alice", + }); + const link = new Link({ + href: new URL("https://example.com/"), + mediaType: "text/html", + }); + const create = new Create({ + id: new URL("https://example.com/activities/1"), + actor: new URL("https://example.com/users/alice"), + object: note, + }); + + await t.step("round-trips a mixed payload", async () => { + const payload = { + note, + when: new Date("2026-01-02T03:04:05Z"), + big: 1234567890123456789n, + url: new URL("https://example.com/some/path"), + list: [person, link], + map: new Map([["create", create], ["n", 42]]), + set: new Set([1, 2, 3]), + nested: { create }, + }; + const encoded = await codec.serialize(payload); + strictEqual(typeof encoded, "string"); + const decoded = await codec.deserialize(encoded) as Record< + string, + unknown + >; + ok(decoded.note instanceof Note); + strictEqual(decoded.note.content?.toString(), "Hello, world!"); + strictEqual(decoded.note.id?.href, "https://example.com/notes/1"); + ok(decoded.when instanceof Date); + strictEqual(decoded.when.toISOString(), "2026-01-02T03:04:05.000Z"); + strictEqual(decoded.big, 1234567890123456789n); + ok(decoded.url instanceof URL); + strictEqual(decoded.url.href, "https://example.com/some/path"); + const list = decoded.list as unknown[]; + ok(list[0] instanceof Person); + strictEqual(list[0].name?.toString(), "Alice"); + ok(list[1] instanceof Link); + strictEqual(list[1].href?.href, "https://example.com/"); + const map = decoded.map as Map; + ok(map.get("create") instanceof Create); + strictEqual(map.get("n"), 42); + deepStrictEqual(decoded.set, new Set([1, 2, 3])); + const nested = decoded.nested as Record; + ok(nested.create instanceof Create); + const nestedObject = await nested.create.getObject({ + documentLoader: mockDocumentLoader, + contextLoader: mockDocumentLoader, + }); + ok(nestedObject instanceof Note); + strictEqual(nestedObject.content?.toString(), "Hello, world!"); + }); + + await t.step( + "encodes vocab objects in expand form (no @context)", + async () => { + const encoded = await codec.serialize({ note }); + ok(!encoded.includes("@context")); + }, + ); + + await t.step("leaves a non-vocab payload untouched", async () => { + const payload = { + text: "plain", + n: 1, + flag: true, + nothing: null, + when: new Date("2026-06-10T00:00:00Z"), + list: [1, "two", 3n], + }; + const encoded = await codec.serialize(payload); + const decoded = await codec.deserialize(encoded); + deepStrictEqual(decoded, payload); + }); + + await t.step("throws on a malformed wire string", async () => { + await rejects(async () => await codec.deserialize("garbage")); + }); + + await t.step("preserves circular and repeated references", async () => { + const shared = new Note({ content: "shared" }); + interface Cyclic { + name: string; + self?: Cyclic; + notes: Note[]; + } + const payload: Cyclic = { name: "root", notes: [shared, shared] }; + payload.self = payload; + const encoded = await codec.serialize(payload); + const decoded = await codec.deserialize(encoded) as Cyclic; + strictEqual(decoded.self, decoded); + ok(decoded.notes[0] instanceof Note); + strictEqual(decoded.notes[0], decoded.notes[1]); + strictEqual(decoded.notes[0].content?.toString(), "shared"); + }); + + await t.step("preserves a cycle through an array", async () => { + const list: unknown[] = ["head"]; + list.push(list); + const encoded = await codec.serialize({ list }); + const decoded = await codec.deserialize(encoded) as { + list: unknown[]; + }; + strictEqual(decoded.list[0], "head"); + strictEqual(decoded.list[1], decoded.list); + }); + + await t.step("preserves cycles re-entering at a Map and a Set", async () => { + const set = new Set(); + set.add({ set }); + const map = new Map(); + map.set("entry", { map }); + const encoded = await codec.serialize({ set, map }); + const decoded = await codec.deserialize(encoded) as { + set: Set<{ set: Set }>; + map: Map }>; + }; + const [member] = decoded.set; + strictEqual(member.set, decoded.set); + strictEqual(decoded.map.get("entry")?.map, decoded.map); + }); +}); + +test("TaskCodec (one instance reused across decodes)", async (t) => { + // The instance carries the cycle-tracking `#seen` map across decodes, but + // each decode parses a fresh object graph with distinct identities, so a + // reused instance still decodes every payload independently. + await t.step("two sequential decodes stay independent", async () => { + const codec = new TaskCodec(loaders); + const first = await codec.serialize({ + note: new Note({ content: "A" }), + }); + const second = await codec.serialize({ + note: new Note({ content: "B" }), + }); + const a = await codec.deserialize(first) as { note: Note }; + const b = await codec.deserialize(second) as { note: Note }; + ok(a.note instanceof Note); + ok(b.note instanceof Note); + strictEqual(a.note.content?.toString(), "A"); + strictEqual(b.note.content?.toString(), "B"); + }); +}); + +test("TaskCodec.validate()", async (t) => { + interface Envelope { + note: Note; + title: string; + } + const schema = makeSchema( + (data): data is Envelope => + typeof data === "object" && data != null && + (data as Envelope).note instanceof Note && + typeof (data as Envelope).title === "string", + ); + + await t.step("accepts a payload with a vocab instanceof leaf", async () => { + const payload = { note: new Note({ content: "Hi" }), title: "greeting" }; + const validated = await TaskCodec.validate(schema, payload); + deepStrictEqual(validated, payload); + }); + + await t.step("rejects a wrong-shaped payload", async () => { + await rejects( + () => TaskCodec.validate(schema, { note: "not a Note", title: 42 }), + { name: "TypeError", message: /Task data failed schema validation/ }, + ); + }); + + await t.step("supports async validation", async () => { + const asyncSchema: StandardSchemaV1 = { + "~standard": { + version: 1, + vendor: "fedify-test", + validate: (value: unknown) => + Promise.resolve( + typeof value === "number" + ? { value } + : { issues: [{ message: "not a number" }] }, + ), + }, + }; + strictEqual(await TaskCodec.validate(asyncSchema, 42), 42); + await rejects(() => TaskCodec.validate(asyncSchema, "nope")); + }); + + await t.step( + "round-trip then validate (same schema on both sides)", + async () => { + const payload = { note: new Note({ content: "Hi" }), title: "greeting" }; + const encoded = await codec.serialize(payload); + const decoded = await codec.deserialize(encoded); + const validated = await TaskCodec.validate(schema, decoded); + ok(validated.note instanceof Note); + strictEqual(validated.title, "greeting"); + strictEqual(validated.note.content?.toString(), "Hi"); + }, + ); +}); + +test("TaskCodec.encode() / decode()", async (t) => { + interface Envelope { + note: Note; + title: string; + } + const schema = makeSchema( + (data): data is Envelope => + typeof data === "object" && data != null && + (data as Envelope).note instanceof Note && + typeof (data as Envelope).title === "string", + ); + + await t.step( + "encode() validates then serializes; decode() round-trips", + async () => { + const payload = { note: new Note({ content: "Hi" }), title: "greeting" }; + const wire = await codec.encode(schema, payload); + strictEqual(typeof wire, "string"); + const back = await codec.decode(schema, wire); + ok(back.note instanceof Note); + strictEqual(back.note.content?.toString(), "Hi"); + strictEqual(back.title, "greeting"); + }, + ); + + await t.step("encode() rejects a wrong-shaped payload", async () => { + await rejects( + () => codec.encode(schema, { note: "nope", title: 42 }), + { name: "TypeError", message: /Task data failed schema validation/ }, + ); + }); + + await t.step( + "decode() re-validates and rejects a drifted payload", + async () => { + // Encode under a permissive schema, decode under the strict one. + const loose = makeSchema((_data): _data is unknown => true); + const wire = await codec.encode(loose, { note: "not a note" }); + await rejects( + () => codec.decode(schema, wire), + { name: "TypeError", message: /Task data failed schema validation/ }, + ); + }, + ); +}); diff --git a/packages/fedify/src/federation/tasks/codec.ts b/packages/fedify/src/federation/tasks/codec.ts new file mode 100644 index 000000000..c073c413a --- /dev/null +++ b/packages/fedify/src/federation/tasks/codec.ts @@ -0,0 +1,186 @@ +import { Link, Object as APObject } from "@fedify/vocab"; +import type { StandardSchemaV1 } from "@standard-schema/spec"; +import { parse, stringifyAsync } from "devalue"; +import type { TaskCodecLoaders } from "./codec-fn.ts"; + +export default class TaskCodec { + constructor(readonly options: TaskCodecLoaders) {} + + serialize = (data: unknown): Promise => + stringifyAsync(data, { Vocab: this.#stringifyVocab }); + + deserialize = (raw: string): Promise => + this.#revive(new Map())(parse(raw, { Vocab: VocabHolder.from })); + + /** Validates `data` against `schema`, then serializes it. */ + encode = async ( + schema: S, + data: StandardSchemaV1.InferInput, + ): Promise => this.serialize(await TaskCodec.validate(schema, data)); + + /** Deserializes `raw`, then validates the result against `schema`. */ + decode = async ( + schema: S, + raw: string, + ): Promise> => + TaskCodec.validate(schema, await this.deserialize(raw)); + + static validate = async ( + schema: S, + data: unknown, + ): Promise> => + getValueIfSchema(await schema["~standard"].validate(data)); + + #stringifyVocab = (value: unknown) => isVocab(value) && this.#toWire(value); + + #toWire = async (value: APObject | Link): Promise => ({ + kind: value instanceof Link ? "link" : "object", + jsonLd: await value.toJsonLd({ format: "expand", ...this.options }), + }); + + // The explicit return type breaks the inference cycle between #revive and + // #classRevivers (whose `set` callbacks call back into #revive). + #revive = (seen: Seen): Revive => async (node: unknown): Promise => + node === null || typeof node !== "object" + ? node + : seen.has(node) + ? seen.get(node) + : (await Array.fromAsync( + this.#classRevivers.map(this.#reviveByClass(seen)), + (f) => f(node), + )).find(Boolean) ?? + node; // Date / URL / RegExp and the like — devalue handled them + + #reviveByClass = + (seen: Seen) => + ([filter, init, set]: ClassReviver) => + async (node: object): Promise => { + if (!filter(node)) return; + // @ts-ignore tsc faults + const out: Revived = await init(node); + seen.set(node, out); + // @ts-ignore tsc faults + await set(this.#revive(seen), node, out); + return out; + }; + + #classRevivers = [ + [ + isInstanceOf(VocabHolder), + ({ kind, jsonLd }: VocabWire): Promise => + kind === "link" + ? Link.fromJsonLd(jsonLd, this.options) + : APObject.fromJsonLd(jsonLd, this.options), + () => Promise.resolve(), + ], + [ + isInstanceOf(Array), + (): unknown[] => [], + async (revive: Revive, node: unknown[], arr: typeof node) => { + arr.push(...await Array.fromAsync(node, revive)); + }, + ], + [ + isInstanceOf(Map), + () => new Map(), + async (revive: Revive, node: Map, map: typeof node) => { + for (const [k, v] of node) map.set(await revive(k), await revive(v)); + }, + ], + [ + isInstanceOf(Set), + () => new Set(), + async (revive: Revive, node: Set, set: typeof node) => { + for (const v of await Array.fromAsync(node, revive)) set.add(v); + }, + ], + [ + isPlainObject, + () => ({}), + async ( + revive: Revive, + node: Record, + obj: typeof node, + ) => { + for (const [k, v] of globalThis.Object.entries(node)) { + obj[k] = await revive(v); + } + }, + ], + ] as const; +} + +const isVocab = (value: unknown): value is APObject | Link => + value instanceof APObject || value instanceof Link; + +const isPlainObject = (value: unknown): value is Record => + value === null || typeof value !== "object" + ? false + : globalThis.Object.getPrototypeOf(value) === globalThis.Object.prototype; + +const isInstanceOf = (cls: Constructor) => (v: unknown): v is T => + v instanceof cls; + +function getValueIfSchema(result: StandardSchemaV1.Result) { + assertSchema(result); + return result.value; +} + +function assertSchema( + result: StandardSchemaV1.Result, +): asserts result is StandardSchemaV1.SuccessResult { + if (result.issues && result.issues.length > 0) { + throw new TypeError( + `Task data failed schema validation: ${JSON.stringify(result.issues)}`, + ); + } +} + +/** Which `fromJsonLd` entry point rebuilds a given vocabulary object. */ +type VocabKind = "object" | "link"; + +/** A vocabulary object reduced to its wire form: a kind tag plus JSON-LD. */ +interface VocabWire { + readonly kind: VocabKind; + readonly jsonLd: unknown; +} + +/** + * A vocabulary object parked by the synchronous decode reviver, held until + * the async revive pass can `fromJsonLd()` it back into an instance. + */ +class VocabHolder implements VocabWire { + constructor(readonly kind: VocabKind, readonly jsonLd: unknown) {} + static from = ({ kind, jsonLd }: VocabWire) => new VocabHolder(kind, jsonLd); +} + +/** Per-decode map from each visited container to its revived counterpart. */ +type Seen = Map; + +/** Revives one node, sharing the per-decode {@link Seen} map via closure. */ +type Revive = (node: unknown) => Promise; + +/** + * One row of {@link TaskCodec.#classRevivers}: a type guard, a factory + * that makes the empty revived container, and a filler that walks the source + * into it using the supplied per-node {@link Revive}. `#reviveByClass` + * cannot annotate its parameter as `typeof this.#classRevivers[number]` + * because a `typeof` query on a private field does not parse, so this loose + * structural shape stands in; the `init` and `set` calls are reconciled with + * `@ts-ignore` at the call site. + */ +type ClassReviver = readonly [ + (value: unknown) => boolean, + (node: never) => unknown, + (revive: Revive, node: never, out: never) => Promise, +]; + +type Container = + | VocabHolder + | Map + | Set + | Array + | Record; +type Revived = Exclude | APObject | Link; +// deno-lint-ignore no-explicit-any +type Constructor = new (...arg: any[]) => T; diff --git a/packages/fedify/src/federation/tasks/mod.ts b/packages/fedify/src/federation/tasks/mod.ts new file mode 100644 index 000000000..a40b57bd3 --- /dev/null +++ b/packages/fedify/src/federation/tasks/mod.ts @@ -0,0 +1,11 @@ +/** + * The internal barrel for the custom background task API. Cross-directory + * consumers (*federation.ts*, *builder.ts*, *context.ts*, *middleware.ts*) + * import from this module, not the individual files. Only the public subset + * is re-exported from *federation/mod.ts*. + * + * @module + */ +export * from "./codec-fn.ts"; +export { default as TaskCodec } from "./codec.ts"; +export * from "./task.ts"; diff --git a/packages/fedify/src/federation/tasks/task.ts b/packages/fedify/src/federation/tasks/task.ts new file mode 100644 index 000000000..1318d647b --- /dev/null +++ b/packages/fedify/src/federation/tasks/task.ts @@ -0,0 +1,169 @@ +import type { StandardSchemaV1 } from "@standard-schema/spec"; +import type { Context } from "../context.ts"; +import type { MessageQueue } from "../mq.ts"; +import type { RetryPolicy } from "../retry.ts"; + +/** + * A callback that processes a custom background task. + * @template TContextData The context data to pass to the {@link Context}. + * @template TData The type of the task payload, inferred from the task's + * schema. + * @param ctx The context for the worker processing the task. + * @param data The decoded and validated task payload. + * @since 2.3.0 + */ +export type TaskHandler = ( + ctx: Context, + data: TData, +) => Promise | void; + +/** + * Options for {@link TaskRegistry.defineTask}. + * @template TContextData The context data to pass to the {@link Context}. + * @template TSchema The [Standard Schema](https://standardschema.dev/) that + * validates the task payload. + * @since 2.3.0 + */ +export interface TaskDefinitionOptions< + TContextData, + TSchema extends StandardSchemaV1, +> { + /** + * The [Standard Schema](https://standardschema.dev/) that validates the + * task payload. The payload type is inferred from this schema. + * + * The payload is validated twice: once at enqueue time (fail fast) and + * once at dequeue time (drift protection against payloads enqueued by an + * older deployment). + */ + readonly schema: TSchema; + + /** + * The callback that processes the task on a background worker. + */ + readonly handler: TaskHandler< + TContextData, + StandardSchemaV1.InferOutput + >; + + /** + * The retry policy for this task. If omitted, the federation-wide + * task retry policy is used, which defaults to an exponential backoff + * policy. + */ + readonly retryPolicy?: RetryPolicy; + + /** + * A callback invoked when the {@link handler} throws an error, before + * a retry is scheduled. + * @param ctx The context for the worker processing the task. + * @param error The error thrown by the handler. + * @param data The decoded and validated task payload. + */ + readonly onError?: ( + ctx: Context, + error: unknown, + data: StandardSchemaV1.InferOutput, + ) => Promise | void; + + /** + * The message queue dedicated to this task. If omitted, the task is + * routed to the federation-wide task queue, falling back to the outbox + * queue (unless `taskQueueResolution: "strict"` is configured). + */ + readonly queue?: MessageQueue; +} + +/** + * The handle returned by {@link TaskRegistry.defineTask}. It carries the + * task name and schema so that {@link Context.enqueueTask} can validate the + * payload and infer its type at every call site. + * @template TContextData The context data to pass to the {@link Context}. + * @template TData The type of the task payload, inferred from the task's + * schema. + * @since 2.3.0 + */ +export interface TaskDefinition { + /** + * The unique name of the task. + */ + readonly name: string; + + /** + * The [Standard Schema](https://standardschema.dev/) that validates the + * task payload. + */ + readonly schema: StandardSchemaV1; + + /** + * @internal Phantom marker binding the handle to its federation. + */ + readonly __contextData?: TContextData; +} + +/** + * Registration of custom background tasks. Both {@link Federation} and + * {@link FederationBuilder} implement this interface. + * @template TContextData The context data to pass to the {@link Context}. + * @since 2.3.0 + */ +export interface TaskRegistry { + /** + * Defines a custom background task. The returned handle is passed to + * {@link Context.enqueueTask} to enqueue the task. + * + * @example + * ``` typescript + * const sendDigest = federation.defineTask("sendDigest", { + * schema: digestSchema, + * handler: async (ctx, data) => { + * // …process the payload on a background worker… + * }, + * }); + * ``` + * + * @param name The unique name of the task. + * @param options The task definition options. The payload type is + * inferred from `options.schema`. + * @returns The handle to pass to {@link Context.enqueueTask}. + * @throws {TypeError} If a task with the same name is already defined. + */ + defineTask( + name: string, + options: TaskDefinitionOptions, + ): TaskDefinition>; +} + +/** + * Options for {@link Context.enqueueTask} and {@link Context.enqueueTaskMany}. + * @since 2.3.0 + */ +export interface TaskEnqueueOptions { + /** + * The delay before the task is processed. No delay by default. + */ + readonly delay?: Temporal.DurationLike; + + /** + * An optional key that ensures tasks with the same ordering key are + * processed sequentially (one at a time). + */ + readonly orderingKey?: string; +} + +/** + * The stored shape of a task definition, read at dispatch time. + * @internal + */ +export interface TaskDefinitionInternal { + readonly name: string; + readonly schema: StandardSchemaV1; + readonly handler: TaskHandler; + readonly retryPolicy?: RetryPolicy; + readonly onError?: ( + ctx: Context, + error: unknown, + data: unknown, + ) => Promise | void; + readonly queue?: MessageQueue; +} diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts new file mode 100644 index 000000000..bf4e862f8 --- /dev/null +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -0,0 +1,647 @@ +import { mockDocumentLoader, test } from "@fedify/fixture"; +import { Note } from "@fedify/vocab"; +import type { StandardSchemaV1 } from "@standard-schema/spec"; +import { delay } from "es-toolkit"; +import { + deepStrictEqual, + ok, + rejects, + strictEqual, + throws, +} from "node:assert/strict"; +import { createFederationBuilder } from "../builder.ts"; +import type { Context } from "../context.ts"; +import type { Federatable, FederationOptions } from "../federation.ts"; +import { MemoryKvStore } from "../kv.ts"; +import { createFederation, type FederationImpl } from "../middleware.ts"; +import { + InProcessMessageQueue, + type MessageQueue, + type MessageQueueEnqueueOptions, + type MessageQueueListenOptions, +} from "../mq.ts"; +import type { TaskMessage } from "../queue.ts"; +import TaskCodec from "./codec.ts"; +import type { TaskDefinition, TaskRegistry } from "./task.ts"; + +type Assert = T; + +const makeSchema = ( + check: (data: unknown) => data is T, +): StandardSchemaV1 => ({ + "~standard": { + version: 1, + vendor: "fedify-test", + validate: (value: unknown) => + check(value) + ? { value } + : { issues: [{ message: "Invalid task data." }] }, + }, +}); + +interface Envelope { + note: Note; + title: string; +} + +const envelopeSchema = makeSchema( + (data): data is Envelope => + typeof data === "object" && data != null && + (data as Envelope).note instanceof Note && + typeof (data as Envelope).title === "string", +); + +const stringSchema = makeSchema((d): d is string => typeof d === "string"); +const numberSchema = makeSchema((d): d is number => typeof d === "number"); + +class MockQueue implements MessageQueue { + readonly nativeRetrial: boolean; + readonly enqueued: { + message: TaskMessage; + options?: MessageQueueEnqueueOptions; + }[] = []; + readonly enqueuedMany: { + messages: readonly TaskMessage[]; + options?: MessageQueueEnqueueOptions; + }[] = []; + listenCount = 0; + enqueueMany?: ( + messages: readonly TaskMessage[], + options?: MessageQueueEnqueueOptions, + ) => Promise; + + constructor( + options: { nativeRetrial?: boolean; supportsEnqueueMany?: boolean } = {}, + ) { + this.nativeRetrial = options.nativeRetrial ?? false; + if (options.supportsEnqueueMany) { + this.enqueueMany = (messages, opts) => { + this.enqueuedMany.push({ messages, options: opts }); + return Promise.resolve(); + }; + } + } + + enqueue( + message: TaskMessage, + options?: MessageQueueEnqueueOptions, + ): Promise { + this.enqueued.push({ message, options }); + return Promise.resolve(); + } + + listen( + _handler: (message: TaskMessage) => Promise | void, + options?: MessageQueueListenOptions, + ): Promise { + this.listenCount++; + return new Promise((resolve) => { + options?.signal?.addEventListener("abort", () => resolve()); + }); + } +} + +const baseOptions: Omit, "queue"> = { + kv: new MemoryKvStore(), + documentLoaderFactory: () => mockDocumentLoader, + contextLoaderFactory: () => mockDocumentLoader, + manuallyStartQueue: true, +}; + +const makeTaskMessage = async ( + taskName: string, + data: unknown, + overrides: Partial = {}, +): Promise => ({ + type: "task", + id: crypto.randomUUID(), + baseUrl: "https://example.com/", + taskName, + data: await new TaskCodec({ contextLoader: mockDocumentLoader }) + .serialize(data), + started: Temporal.Now.instant().toString(), + attempt: 0, + traceContext: {}, + ...overrides, +}); + +async function waitFor( + predicate: () => boolean, + timeoutMs: number, +): Promise { + const started = Date.now(); + while (!predicate()) { + await delay(50); + if (Date.now() - started > timeoutMs) throw new Error("Timeout"); + } +} + +test("defineTask()", async (t) => { + await t.step("returns a handle carrying name and schema", () => { + const federation = createFederation({ + ...baseOptions, + queue: { task: new MockQueue() }, + }); + const task = federation.defineTask("greet", { + schema: stringSchema, + handler: () => {}, + }); + strictEqual(task.name, "greet"); + strictEqual(task.schema, stringSchema); + }); + + await t.step("throws on a duplicate name", () => { + const federation = createFederation({ + ...baseOptions, + queue: { task: new MockQueue() }, + }); + federation.defineTask("dup", { + schema: stringSchema, + handler: () => {}, + }); + throws( + () => + federation.defineTask("dup", { + schema: stringSchema, + handler: () => {}, + }), + { name: "TypeError", message: /already defined/ }, + ); + }); + + await t.step("build() clones the task registry", async () => { + const builder = createFederationBuilder(); + builder.defineTask("first", { + schema: stringSchema, + handler: () => {}, + }); + const f1 = await builder.build({ + ...baseOptions, + queue: { task: new MockQueue() }, + }) as FederationImpl; + builder.defineTask("second", { + schema: stringSchema, + handler: () => {}, + }); + const f2 = await builder.build({ + ...baseOptions, + queue: { task: new MockQueue() }, + }) as FederationImpl; + deepStrictEqual(Object.keys(f1.taskDefinitions), ["first"]); + deepStrictEqual(Object.keys(f2.taskDefinitions), ["first", "second"]); + // Defining on a built federation does not leak back into the builder: + f1.defineTask("third", { schema: stringSchema, handler: () => {} }); + deepStrictEqual(Object.keys(f2.taskDefinitions), ["first", "second"]); + }); +}); + +test("task type-level guards", () => { + // Forward-compat seam: Federatable must remain assignable to TaskRegistry, + // so a future Worker can implement TaskRegistry directly. + type _ForwardCompat = Assert< + Federatable extends TaskRegistry ? true : false + >; + const _wrongPayloadIsACompileError = ( + ctx: Context, + task: TaskDefinition, + ) => { + // @ts-expect-error: a wrong-shaped payload must not type-check. + return ctx.enqueueTask(task, { n: "not a number" }); + }; +}); + +test("Context.enqueueTask() end-to-end", async (t) => { + await t.step("round-trips a typed payload to the handler", async () => { + const queue = new InProcessMessageQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const received: { ctx: Context; data: Envelope }[] = []; + const task = federation.defineTask("notify", { + schema: envelopeSchema, + handler: (ctx, data) => { + received.push({ ctx, data }); + }, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, { + note: new Note({ content: "Hello, world!" }), + title: "greeting", + }); + const controller = new AbortController(); + const listening = federation.startQueue(undefined, { + signal: controller.signal, + queue: "task", + }); + try { + await waitFor(() => received.length > 0, 15_000); + } finally { + controller.abort(); + await listening; + } + strictEqual(received.length, 1); + const { ctx: handlerCtx, data } = received[0]; + ok(data.note instanceof Note); + strictEqual(data.note.content?.toString(), "Hello, world!"); + strictEqual(data.title, "greeting"); + strictEqual(handlerCtx.origin, "https://example.com"); + }); + + await t.step("rejects an invalid payload at enqueue", async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("strictly-typed", { + schema: numberSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + // deno-lint-ignore no-explicit-any + () => ctx.enqueueTask(task, "not a number" as any), + { name: "TypeError", message: /Task data failed schema validation/ }, + ); + strictEqual(queue.enqueued.length, 0); + }); + + await t.step("passes delay and orderingKey through", async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("delayed", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { + delay: { seconds: 30 }, + orderingKey: "user:alice", + }); + strictEqual(queue.enqueued.length, 1); + const { message, options } = queue.enqueued[0]; + strictEqual(message.taskName, "delayed"); + strictEqual(message.orderingKey, "user:alice"); + strictEqual(message.attempt, 0); + ok(options?.delay instanceof Temporal.Duration); + strictEqual(options.delay.total("second"), 30); + strictEqual(options.orderingKey, "user:alice"); + }); + + await t.step( + "enqueueTaskMany() uses enqueueMany when available", + async () => { + const queue = new MockQueue({ supportsEnqueueMany: true }); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("bulk", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTaskMany(task, ["a", "b", "c"]); + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 1); + strictEqual(queue.enqueuedMany[0].messages.length, 3); + }, + ); + + await t.step( + "enqueueTaskMany() falls back to parallel enqueues", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("bulk-fallback", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTaskMany(task, ["a", "b"]); + strictEqual(queue.enqueued.length, 2); + }, + ); +}); + +test("task queue routing", async (t) => { + await t.step("prefers the per-task queue", async () => { + const taskQueue = new MockQueue(); + const perTaskQueue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: taskQueue }, + }); + const task = federation.defineTask("isolated", { + schema: stringSchema, + handler: () => {}, + queue: perTaskQueue, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "data"); + strictEqual(perTaskQueue.enqueued.length, 1); + strictEqual(taskQueue.enqueued.length, 0); + }); + + await t.step("falls back to the outbox queue by default", async () => { + const outboxQueue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { outbox: outboxQueue }, + }); + const task = federation.defineTask("fallback", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "data"); + strictEqual(outboxQueue.enqueued.length, 1); + }); + + await t.step( + 'taskQueueResolution: "strict" throws at enqueue instead', + async () => { + const outboxQueue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { outbox: outboxQueue }, + taskQueueResolution: "strict", + }); + const task = federation.defineTask("strict", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + () => ctx.enqueueTask(task, "data"), + { name: "TypeError", message: /No message queue is configured/ }, + ); + strictEqual(outboxQueue.enqueued.length, 0); + }, + ); + + await t.step("throws when no queue is configured at all", async () => { + const federation = createFederation({ ...baseOptions }); + const task = federation.defineTask("queueless", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + () => ctx.enqueueTask(task, "data"), + { name: "TypeError", message: /No message queue is configured/ }, + ); + }); +}); + +test("startQueue() task worker", async (t) => { + await t.step('starts only the task worker for queue: "task"', async () => { + const inbox = new MockQueue(); + const outbox = new MockQueue(); + const fanout = new MockQueue(); + const taskQueue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { inbox, outbox, fanout, task: taskQueue }, + }); + const controller = new AbortController(); + const listening = federation.startQueue(undefined, { + signal: controller.signal, + queue: "task", + }); + strictEqual(taskQueue.listenCount, 1); + strictEqual(inbox.listenCount, 0); + strictEqual(outbox.listenCount, 0); + strictEqual(fanout.listenCount, 0); + controller.abort(); + await listening; + }); + + await t.step("starts the worker for a task-only deployment", async () => { + const taskQueue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: taskQueue }, + }); + const controller = new AbortController(); + const listening = federation.startQueue(undefined, { + signal: controller.signal, + }); + strictEqual(taskQueue.listenCount, 1); + controller.abort(); + await listening; + }); + + await t.step("does not double-listen on a shared queue", async () => { + const shared = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { outbox: shared, task: shared }, + }); + const controller = new AbortController(); + const listening = federation.startQueue(undefined, { + signal: controller.signal, + }); + strictEqual(shared.listenCount, 1); + controller.abort(); + await listening; + }); +}); + +test("processQueuedTask() task dispatch", async (t) => { + await t.step("drops an unknown task with a warning", async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }) as FederationImpl; + const message = await makeTaskMessage("never-defined", "data"); + await federation.processQueuedTask(undefined, message); + strictEqual(queue.enqueued.length, 0); + }); + + await t.step("drops an undecodable payload without retry", async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }) as FederationImpl; + let called = 0; + federation.defineTask("broken-wire", { + schema: stringSchema, + handler: () => { + called++; + }, + }); + const message = await makeTaskMessage("broken-wire", "data"); + await federation.processQueuedTask(undefined, { + ...message, + data: "garbage that is not devalue", + }); + strictEqual(called, 0); + strictEqual(queue.enqueued.length, 0); + }); + + await t.step("drops a drifted payload without retry", async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }) as FederationImpl; + let called = 0; + federation.defineTask("drifted", { + schema: numberSchema, // the "new deploy" expects a number… + handler: () => { + called++; + }, + }); + // …but the payload was enqueued by an "old deploy" as a string: + const message = await makeTaskMessage("drifted", "stringly-typed"); + await federation.processQueuedTask(undefined, message); + strictEqual(called, 0); + strictEqual(queue.enqueued.length, 0); + }); + + await t.step( + "re-enqueues with attempt + 1 when the handler throws", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }) as FederationImpl; + const errors: unknown[] = []; + federation.defineTask("flaky", { + schema: stringSchema, + handler: () => { + throw new Error("boom"); + }, + retryPolicy: () => Temporal.Duration.from({ milliseconds: 1 }), + onError: (_ctx, error, data) => { + errors.push([error, data]); + }, + }); + const message = await makeTaskMessage("flaky", "data", { + orderingKey: "k", + }); + await federation.processQueuedTask(undefined, message); + strictEqual(queue.enqueued.length, 1); + const retry = queue.enqueued[0]; + strictEqual(retry.message.attempt, 1); + strictEqual(retry.message.taskName, "flaky"); + strictEqual(retry.message.orderingKey, "k"); + strictEqual(retry.options?.orderingKey, "k"); + ok(retry.options?.delay instanceof Temporal.Duration); + strictEqual(errors.length, 1); + deepStrictEqual(errors[0], [new Error("boom"), "data"]); + }, + ); + + await t.step("gives up when the retry policy returns null", async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }) as FederationImpl; + federation.defineTask("hopeless", { + schema: stringSchema, + handler: () => { + throw new Error("boom"); + }, + retryPolicy: () => null, + }); + const message = await makeTaskMessage("hopeless", "data"); + await federation.processQueuedTask(undefined, message); + strictEqual(queue.enqueued.length, 0); + }); + + await t.step( + "per-task retryPolicy overrides the federation default", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + taskRetryPolicy: () => null, // the federation default gives up… + }) as FederationImpl; + federation.defineTask("override", { + schema: stringSchema, + handler: () => { + throw new Error("boom"); + }, + // …but the per-task policy retries: + retryPolicy: () => Temporal.Duration.from({ milliseconds: 1 }), + }); + federation.defineTask("default", { + schema: stringSchema, + handler: () => { + throw new Error("boom"); + }, + }); + await federation.processQueuedTask( + undefined, + await makeTaskMessage("override", "data"), + ); + strictEqual(queue.enqueued.length, 1); + await federation.processQueuedTask( + undefined, + await makeTaskMessage("default", "data"), + ); + strictEqual(queue.enqueued.length, 1); // unchanged: gave up + }, + ); + + await t.step("rethrows on a nativeRetrial queue", async () => { + const queue = new MockQueue({ nativeRetrial: true }); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }) as FederationImpl; + federation.defineTask("native", { + schema: stringSchema, + handler: () => { + throw new Error("boom"); + }, + }); + const message = await makeTaskMessage("native", "data"); + await rejects( + () => federation.processQueuedTask(undefined, message), + { message: /boom/ }, + ); + strictEqual(queue.enqueued.length, 0); + }); +}); diff --git a/packages/fedify/src/testing/context.ts b/packages/fedify/src/testing/context.ts index 0f9781d14..c613c93c4 100644 --- a/packages/fedify/src/testing/context.ts +++ b/packages/fedify/src/testing/context.ts @@ -51,6 +51,8 @@ export function createContext( lookupWebFinger, sendActivity, routeActivity, + enqueueTask, + enqueueTaskMany, } = values; function throwRouterError(): URL { throw new RouterError("Not implemented"); @@ -113,6 +115,12 @@ export function createContext( routeActivity: routeActivity ?? ((_params) => { throw new Error("Not implemented"); }), + enqueueTask: enqueueTask ?? ((_task, _data, _options) => { + throw new Error("Not implemented"); + }), + enqueueTaskMany: enqueueTaskMany ?? ((_task, _payloads, _options) => { + throw new Error("Not implemented"); + }), }; } diff --git a/packages/testing/src/context.ts b/packages/testing/src/context.ts index a4771cf14..c8e91a2af 100644 --- a/packages/testing/src/context.ts +++ b/packages/testing/src/context.ts @@ -100,6 +100,8 @@ function createContext( lookupWebFinger, sendActivity, routeActivity, + enqueueTask, + enqueueTaskMany, } = values; function throwRouterError(): URL { throw new RouterError("Not implemented"); @@ -165,6 +167,12 @@ function createContext( routeActivity: routeActivity ?? ((_params) => { throw new Error("Not implemented"); }), + enqueueTask: enqueueTask ?? ((_task, _data, _options) => { + throw new Error("Not implemented"); + }), + enqueueTaskMany: enqueueTaskMany ?? ((_task, _payloads, _options) => { + throw new Error("Not implemented"); + }), }; } diff --git a/packages/testing/src/mock.ts b/packages/testing/src/mock.ts index 271387c25..74cfe1a0d 100644 --- a/packages/testing/src/mock.ts +++ b/packages/testing/src/mock.ts @@ -263,7 +263,7 @@ type ActivityConstructor = new (...args: any[]) => Activity; class MockFederation implements Federation { public sentActivities: SentActivity[] = []; public queueStarted = false; - private activeQueues: Set<"inbox" | "outbox" | "fanout"> = new Set(); + private activeQueues: Set<"inbox" | "outbox" | "fanout" | "task"> = new Set(); public sentCounter = 0; private nodeInfoDispatcher?: any; // Note: Using `any` instead of WebFingerLinksDispatcher to avoid JSR hang. @@ -285,6 +285,7 @@ class MockFederation implements Federation { public sharedInboxPath?: string; public objectPaths: Map = new Map(); public objectDispatchers: Map = new Map(); + public taskDefinitions: Map = new Map(); private inboxDispatcher?: any; private outboxDispatcher?: any; private outboxAuthorizePredicate?: any; @@ -317,6 +318,16 @@ class MockFederation implements Federation { this.nodeInfoPath = path; } + // Note: Parameter and return types are `any` like the other mock methods; + // the structural shape follows TaskRegistry.defineTask(). + defineTask(name: string, options: any): any { + if (this.taskDefinitions.has(name)) { + throw new TypeError(`Task ${JSON.stringify(name)} is already defined.`); + } + this.taskDefinitions.set(name, { name, ...options }); + return { name, schema: options.schema }; + } + // Note: Parameter type is `any` instead of WebFingerLinksDispatcher to avoid // JSR type analyzer hang (issue #468). See comment on webFingerDispatcher field. setWebFingerLinksDispatcher( @@ -504,10 +515,11 @@ class MockFederation implements Federation { if (options?.queue) { this.activeQueues.add(options.queue); } else { - // If no specific queue, activate all three + // If no specific queue, activate all four this.activeQueues.add("inbox"); this.activeQueues.add("outbox"); this.activeQueues.add("fanout"); + this.activeQueues.add("task"); } } @@ -955,6 +967,27 @@ class MockContext implements Context { return Promise.resolve(null); } + // No queue in mock type: the task handler is invoked immediately, + // mirroring how processQueuedTask() processes immediately. + async enqueueTask(task: any, data: any, _options?: any): Promise { + if (!(this.federation instanceof MockFederation)) { + throw new TypeError("No task definitions are available."); + } + const def = this.federation.taskDefinitions.get(task.name); + if (def == null) { + throw new TypeError(`Task ${JSON.stringify(task.name)} is not defined.`); + } + await def.handler(this, data); + } + + async enqueueTaskMany( + task: any, + payloads: readonly any[], + options?: any, + ): Promise { + for (const data of payloads) await this.enqueueTask(task, data, options); + } + clone(data: TContextData): TestContext { return new MockContext({ url: this.url, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b2279217a..63cedfc77 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1241,9 +1241,15 @@ importers: '@opentelemetry/semantic-conventions': specifier: 'catalog:' version: 1.40.0 + '@standard-schema/spec': + specifier: 'catalog:' + version: 1.1.0 byte-encodings: specifier: 'catalog:' version: 1.0.11 + devalue: + specifier: ^5.8.1 + version: 5.8.1 es-toolkit: specifier: 'catalog:' version: 1.46.1 @@ -7986,15 +7992,15 @@ packages: resolution: {integrity: sha512-KxektNH63SrbfUyDiwXqRb1rLwKt33AmMv+5Nhsw1kqZ13SJBRTgZHtGbE+hH3a1mVW1cz+4pqSWVPAtLVXTzQ==} engines: {node: '>=18'} - devalue@5.1.1: - resolution: {integrity: sha512-maua5KUiapvEwiEAe+XnlZ3Rh0GD+qI1J/nb9vrJc3muPXvcF/8gXYTWF76+5DAqHyDUtOIImEuo0YKE9mshVw==} - devalue@5.6.1: resolution: {integrity: sha512-jDwizj+IlEZBunHcOuuFVBnIMPAEHvTsJj0BcIp94xYguLRVBcXO853px/MyIJvbVzWdsGvrRweIUWJw8hBP7A==} devalue@5.7.1: resolution: {integrity: sha512-MUbZ586EgQqdRnC4yDrlod3BEdyvE4TapGYHMW2CiaW+KkkFmWEFqBUaLltEZCGi0iFXCEjRF0OjF0DV2QHjOA==} + devalue@5.8.1: + resolution: {integrity: sha512-4CXDYRBGqN+57wVJkuXBYmpAVUSg3L6JAQa/DFqm238G73E1wuyc/JhGQJzN7vUf/CMphYau2zXbfWzDR5aTEw==} + devlop@1.1.0: resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} @@ -15135,7 +15141,7 @@ snapshots: consola: 3.4.2 defu: 6.1.7 destr: 2.0.5 - devalue: 5.7.1 + devalue: 5.8.1 errx: 0.1.0 escape-string-regexp: 5.0.0 exsolve: 1.0.8 @@ -16758,7 +16764,7 @@ snapshots: '@types/cookie': 0.6.0 acorn: 8.15.0 cookie: 0.6.0 - devalue: 5.1.1 + devalue: 5.8.1 esm-env: 1.2.2 kleur: 4.1.5 magic-string: 0.30.17 @@ -16779,7 +16785,7 @@ snapshots: '@types/cookie': 0.6.0 acorn: 8.15.0 cookie: 0.6.0 - devalue: 5.1.1 + devalue: 5.8.1 esm-env: 1.2.2 kleur: 4.1.5 magic-string: 0.30.17 @@ -19507,12 +19513,12 @@ snapshots: dependencies: base-64: 1.0.0 - devalue@5.1.1: {} - devalue@5.6.1: {} devalue@5.7.1: {} + devalue@5.8.1: {} + devlop@1.1.0: dependencies: dequal: 2.0.3 From a096f2c1822a30eebb0d1944807e920a86927b88 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 13:57:26 +0000 Subject: [PATCH 02/57] Restructure mise install tasks and update docs Split the monolithic `install` task into `install:deno` and `install:pnpm`, with `codegen` as an explicit dependency, so each runtime's setup can be run on its own. `test:deno` now depends on `install:deno` instead of `prepare`, since Deno runs the TypeScript sources directly and does not need the build step. Update AGENTS.md to match: document `mise run prepare`/`prepare-each` for building, `check-each` and `test-each` for scoping work to specific packages, and add a section directing agents to consult `mise tasks`. Assisted-by: Claude Code:claude-opus-4-8 --- AGENTS.md | 35 ++++++++++++++++++++++++++++++----- mise.toml | 2 +- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4544f7895..458069844 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -145,11 +145,29 @@ Development workflow the editor/LSP after editing YAML. - **Building Packages**: All packages are built automatically as part of setup. Run `mise run build` to rebuild everything, or - `mise run prepare-each ` to rebuild just one (without the `@fedify/` - prefix). - - **Checking Code**: Run `mise run check` before committing. - - **Running Tests**: Use `mise run test:deno` for Deno tests or - `mise run test` for all environments. + `mise run prepare-each ` to rebuild specific packages (without the + `@fedify/` prefix). + - **Checking Code**: Run `mise run check` before committing, or run + `mise run check-each ` to check specific packages. If any issues from + `check:fmt`, `check:lint` or `check:md`, are found, refers + **Formatting and Linting** section. + - **Formatting and Linting**: Run `mise run fmt` to format all code and docs. + - **Running Tests**: + While testing is certainly important, blindly running every test suite every + time is inefficient. Since Deno executes TS source code directly, it doesn't + waste resources on builds. Therefore, during development, run + `mise run test:deno {TEST_PATH} --filter ` for most tests that + are independent of the runtime. If the test is dependent on a specific + runtime other than Deno, replace `test:deno` with `test:node` or `test:bun`. + Once development is complete, run `mise run test-each ` to test the + modified packages (without the `@fedify/` prefix). + Finally, when ready for deployment, run `mise run test` to execute the + whole codebase-wide tests. + - `mise run test`: Executes all the tests in every runtime. + - `mise run test:`: + Executes all the tests by the runtime. + - `mise run test-each `: Executes tests in packages that include + `pkgs` in every runtime (without the `@fedify/` prefix). For detailed contribution guidelines, see *CONTRIBUTING.md*. @@ -169,6 +187,13 @@ When working with federation code: Common tasks ------------ +### **BE WELL-ACQUAINTED WITH `mise tasks`** + +*mise.toml* has useful tasks. **Acquaint all of them** and use them in the right +place at the right time. If it has too many information, use `mise tasks`. This +command show the summary of the tasks and descriptions. If `mise tasks` didn't +not make it sure, use `mise tasks ` to check the details for the task. + ### Adding ActivityPub vocabulary types 1. Create a new YAML file in *packages/vocab/vocab/* following existing diff --git a/mise.toml b/mise.toml index 7ae57df3b..1fbf82a53 100644 --- a/mise.toml +++ b/mise.toml @@ -246,9 +246,9 @@ for pkg in ($env.usage_packages | split row " " | where ($it | is-not-empty)) { ''' # Testing + [tasks."test:deno"] description = "Run the test suite using Deno" -depends = ["build"] run = "deno test --check --doc --allow-all --unstable-kv --trace-leaks --parallel" [tasks."test:node"] From bc71b3faa623e8a78e9f1a8ecf1cf88b8fa27355 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 15:46:48 +0000 Subject: [PATCH 03/57] Add PR #803 link in changelog --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3d36e287d..d858acb37 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -370,7 +370,7 @@ Released on June 25, 2026. `Federation.startQueue()` now accepts `queue: "task"` to run a task-only worker. - [[#206], [#797] by ChanHaeng Lee] + [[#206], [#797], [#803] by ChanHaeng Lee] [Standard Schema]: https://standardschema.dev/ [#206]: https://github.com/fedify-dev/fedify/issues/206 @@ -405,6 +405,7 @@ Released on June 25, 2026. [#787]: https://github.com/fedify-dev/fedify/pull/787 [#797]: https://github.com/fedify-dev/fedify/issues/797 [#800]: https://github.com/fedify-dev/fedify/pull/800 +[#803]: https://github.com/fedify-dev/fedify/pull/803 ### @fedify/cli From 6bc8bdf6f08732d57a2c3d2707195ede5cbc6b78 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 16:41:08 +0000 Subject: [PATCH 04/57] Revive vocab nested in null-prototype objects `isPlainObject` in the task codec only accepted objects whose prototype is exactly `Object.prototype`, so an object made with `Object.create(null)` was treated as a non-plain leaf. Any vocab object nested inside such an object was therefore left as its parked holder instead of being revived, even though devalue round-trips null-prototype objects without throwing. Accept a `null` prototype as well, and add a regression test that round-trips a vocab object nested in an `Object.create(null)` object. https://github.com/fedify-dev/fedify/pull/803#discussion_r3396953879 Assisted-by: Claude Code:claude-opus-4-8 --- packages/fedify/src/federation/tasks/codec.test.ts | 14 ++++++++++++++ packages/fedify/src/federation/tasks/codec.ts | 5 ++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/fedify/src/federation/tasks/codec.test.ts b/packages/fedify/src/federation/tasks/codec.test.ts index d7dd4b636..d45d39ea1 100644 --- a/packages/fedify/src/federation/tasks/codec.test.ts +++ b/packages/fedify/src/federation/tasks/codec.test.ts @@ -158,6 +158,20 @@ test("TaskCodec (fresh instance per operation)", async (t) => { strictEqual(member.set, decoded.set); strictEqual(decoded.map.get("entry")?.map, decoded.map); }); + + await t.step( + "revives a vocab object nested in a null-prototype object", + async () => { + const nullProto = Object.create(null) as Record; + nullProto.note = note; + const encoded = await codec.serialize({ wrap: nullProto }); + const decoded = await codec.deserialize(encoded) as { + wrap: Record; + }; + ok(decoded.wrap.note instanceof Note); + strictEqual(decoded.wrap.note.content?.toString(), "Hello, world!"); + }, + ); }); test("TaskCodec (one instance reused across decodes)", async (t) => { diff --git a/packages/fedify/src/federation/tasks/codec.ts b/packages/fedify/src/federation/tasks/codec.ts index c073c413a..e83da0045 100644 --- a/packages/fedify/src/federation/tasks/codec.ts +++ b/packages/fedify/src/federation/tasks/codec.ts @@ -116,7 +116,10 @@ const isVocab = (value: unknown): value is APObject | Link => const isPlainObject = (value: unknown): value is Record => value === null || typeof value !== "object" ? false - : globalThis.Object.getPrototypeOf(value) === globalThis.Object.prototype; + : isObjectPrototype(globalThis.Object.getPrototypeOf(value)); + +const isObjectPrototype = (proto: unknown): boolean => + proto === null || proto === globalThis.Object.prototype; const isInstanceOf = (cls: Constructor) => (v: unknown): v is T => v instanceof cls; From f7c9a344fcca7bd50188631731390d484543f11a Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 16:58:44 +0000 Subject: [PATCH 05/57] Guard against an unparsable task started time When a custom task handler throws and the queue does not own retries, the error path computes the elapsed time from `message.started` to feed the retry policy. `message.started` is normally a valid ISO instant set at enqueue time, but a corrupted or drifted queue could hand back an invalid string, in which case `Temporal.Instant.from()` threw out of the error-handling block. That masked the original handler error and aborted the retry, silently dropping the task. Wrap the parse in a try-catch, fall back to a zero elapsed time, and log the offending value. A regression test drives a message with a malformed `started` through a throwing handler and asserts the retry is still enqueued. https://github.com/fedify-dev/fedify/pull/803#discussion_r3396953943 Assisted-by: Claude Code:claude-opus-4-8 --- packages/fedify/src/federation/middleware.ts | 19 +++++++++++-- .../fedify/src/federation/tasks/tasks.test.ts | 27 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index ee96cd8df..1fea1652b 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -2161,9 +2161,24 @@ export class FederationImpl const queue = this.resolveTaskQueue(def.name); if (queue?.nativeRetrial) throw error; // the backend owns retries const retryPolicy = def.retryPolicy ?? this.taskRetryPolicy; + // A corrupted `started` must not throw here and abort the retry. + let elapsedTime = Temporal.Duration.from({ seconds: 0 }); + try { + elapsedTime = Temporal.Instant.from(message.started) + .until(Temporal.Now.instant()); + } catch (parseError) { + logger.error( + "Custom task {taskName} has an unparsable started time " + + "{started}; treating elapsedTime as zero:\n{error}", + { + taskName: message.taskName, + started: message.started, + error: parseError, + }, + ); + } const delay = retryPolicy({ - elapsedTime: Temporal.Instant.from(message.started) - .until(Temporal.Now.instant()), + elapsedTime, attempts: message.attempt, }); if (delay != null && queue != null) { diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index bf4e862f8..3ef0489d6 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -625,6 +625,33 @@ test("processQueuedTask() task dispatch", async (t) => { }, ); + await t.step( + "still retries when message.started is malformed", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }) as FederationImpl; + federation.defineTask("bad-started", { + schema: stringSchema, + handler: () => { + throw new Error("boom"); + }, + retryPolicy: () => Temporal.Duration.from({ milliseconds: 1 }), + }); + // A corrupted or drifted queue can hand back an invalid `started`; + // computing elapsedTime must not throw out of the error path and abort + // the retry. + const message = await makeTaskMessage("bad-started", "data", { + started: "not-an-instant", + }); + await federation.processQueuedTask(undefined, message); + strictEqual(queue.enqueued.length, 1); + strictEqual(queue.enqueued[0].message.attempt, 1); + }, + ); + await t.step("rethrows on a nativeRetrial queue", async () => { const queue = new MockQueue({ nativeRetrial: true }); const federation = createFederation({ From 58c01caeccd91f6219439d3b17c1795fc5257a29 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 18:30:47 +0000 Subject: [PATCH 06/57] Store task definitions in a prototype-safe Map `FederationBuilderImpl.taskDefinitions` was a plain object, so the duplicate check `name in this.taskDefinitions` and the lookups `this.taskDefinitions[taskName]` consulted the prototype chain. Task names are arbitrary user-supplied strings, so a name such as "constructor", "toString", or "__proto__" was wrongly reported as already defined and resolved to an inherited method on lookup. Switch the registry to a `Map`, which is immune to prototype keys by construction and avoids the clone footgun where a later spread or `Object.assign` would silently reintroduce the prototype. Sibling registries stay plain objects since they are keyed by controlled values (type-id URLs). Add a regression test covering names that collide with `Object.prototype`. https://github.com/fedify-dev/fedify/pull/803#discussion_r3397749459 Assisted-by: Claude Code:claude-opus-4-8 --- packages/fedify/src/federation/builder.ts | 12 +++---- packages/fedify/src/federation/middleware.ts | 4 +-- .../fedify/src/federation/tasks/tasks.test.ts | 33 +++++++++++++++++-- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/packages/fedify/src/federation/builder.ts b/packages/fedify/src/federation/builder.ts index e237d9c2e..59460f3f9 100644 --- a/packages/fedify/src/federation/builder.ts +++ b/packages/fedify/src/federation/builder.ts @@ -188,7 +188,7 @@ export class FederationBuilderImpl TContextData > >; - taskDefinitions: Record>; + taskDefinitions: Map>; /** * Symbol registry for unique identification of unnamed symbols. @@ -201,7 +201,7 @@ export class FederationBuilderImpl this.objectTypeIds = {}; this.collectionCallbacks = {}; this.collectionTypeIds = {}; - this.taskDefinitions = {}; + this.taskDefinitions = new Map(); } /** @@ -267,7 +267,7 @@ export class FederationBuilderImpl f.unverifiedActivityHandler = this.unverifiedActivityHandler; f.outboxPermanentFailureHandler = this.outboxPermanentFailureHandler; f.idempotencyStrategy = this.idempotencyStrategy; - f.taskDefinitions = { ...this.taskDefinitions }; + f.taskDefinitions = new Map(this.taskDefinitions); return f; } @@ -607,10 +607,10 @@ export class FederationBuilderImpl name: string, options: TaskDefinitionOptions, ): TaskDefinition> { - if (name in this.taskDefinitions) { + if (this.taskDefinitions.has(name)) { throw new TypeError(`Task ${JSON.stringify(name)} is already defined.`); } - this.taskDefinitions[name] = { + this.taskDefinitions.set(name, { name, schema: options.schema, handler: options.handler as TaskHandler, @@ -618,7 +618,7 @@ export class FederationBuilderImpl .onError as TaskDefinitionInternal["onError"], retryPolicy: options.retryPolicy, queue: options.queue, - }; + }); return { name, schema: options.schema }; } diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 1fea1652b..ea8b8718d 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -957,7 +957,7 @@ export class FederationImpl } resolveTaskQueue(taskName: string): MessageQueue | undefined { - const def = this.taskDefinitions[taskName]; + const def = this.taskDefinitions.get(taskName); const resolved = def?.queue ?? this.taskQueue; if (resolved != null) return resolved; return this.taskQueueResolution === "strict" ? undefined : this.outboxQueue; @@ -2119,7 +2119,7 @@ export class FederationImpl message: TaskMessage, ): Promise { const logger = getLogger(["fedify", "federation", "task"]); - const def = this.taskDefinitions[message.taskName]; + const def = this.taskDefinitions.get(message.taskName); if (def == null) { // Unknown task: a handler won't appear by retrying. Drop and log. logger.warn( diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index 3ef0489d6..6b313f0a9 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -169,6 +169,33 @@ test("defineTask()", async (t) => { ); }); + await t.step("accepts names that collide with Object.prototype", () => { + const federation = createFederation({ + ...baseOptions, + queue: { task: new MockQueue() }, + }) as FederationImpl; + // These names exist on Object.prototype; a plain-object registry would + // mistake them for already-defined tasks (`name in {}`) and would return + // an inherited method on lookup. + for (const name of ["constructor", "toString", "hasOwnProperty"]) { + const task = federation.defineTask(name, { + schema: stringSchema, + handler: () => {}, + }); + strictEqual(task.name, name); + strictEqual(federation.taskDefinitions.get(name)?.name, name); + } + // A genuine duplicate still throws. + throws( + () => + federation.defineTask("toString", { + schema: stringSchema, + handler: () => {}, + }), + { name: "TypeError", message: /already defined/ }, + ); + }); + await t.step("build() clones the task registry", async () => { const builder = createFederationBuilder(); builder.defineTask("first", { @@ -187,11 +214,11 @@ test("defineTask()", async (t) => { ...baseOptions, queue: { task: new MockQueue() }, }) as FederationImpl; - deepStrictEqual(Object.keys(f1.taskDefinitions), ["first"]); - deepStrictEqual(Object.keys(f2.taskDefinitions), ["first", "second"]); + deepStrictEqual([...f1.taskDefinitions.keys()], ["first"]); + deepStrictEqual([...f2.taskDefinitions.keys()], ["first", "second"]); // Defining on a built federation does not leak back into the builder: f1.defineTask("third", { schema: stringSchema, handler: () => {} }); - deepStrictEqual(Object.keys(f2.taskDefinitions), ["first", "second"]); + deepStrictEqual([...f2.taskDefinitions.keys()], ["first", "second"]); }); }); From e36d692948700bb1c12394831fff5e5608ecd4e7 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 18:36:51 +0000 Subject: [PATCH 07/57] Simplify task payload revival to a single matching reviver `#revive` mapped every node through all five class revivers, allocating five promises per node and resolving them with `Array.fromAsync` before picking the first truthy result. The class filters are mutually exclusive, so it now finds the single matching reviver and runs only that one, cutting the per-node work to a single promise. This keeps the existing behaviour (cycles, repeated references, and Map/Set/Array/plain-object/null-prototype containers all still round-trip, as the codec tests assert) and folds the rationale for two declined suggestions into a comment: the walked tree is devalue's throwaway parse output, so there is no external identity to preserve and nothing to clone lazily; and a recursion-depth cap is moot because this pass recurses with `await` (unwinding the stack each level) while devalue's own recursive `stringify`/`parse` is the binding limit on nesting and would overflow first. https://github.com/fedify-dev/fedify/pull/803#discussion_r3397748334 https://github.com/fedify-dev/fedify/pull/803#discussion_r3397748344 Assisted-by: Claude Code:claude-opus-4-8 --- packages/fedify/src/federation/tasks/codec.ts | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/packages/fedify/src/federation/tasks/codec.ts b/packages/fedify/src/federation/tasks/codec.ts index e83da0045..3eab7f52f 100644 --- a/packages/fedify/src/federation/tasks/codec.ts +++ b/packages/fedify/src/federation/tasks/codec.ts @@ -40,29 +40,31 @@ export default class TaskCodec { // The explicit return type breaks the inference cycle between #revive and // #classRevivers (whose `set` callbacks call back into #revive). - #revive = (seen: Seen): Revive => async (node: unknown): Promise => - node === null || typeof node !== "object" - ? node - : seen.has(node) - ? seen.get(node) - : (await Array.fromAsync( - this.#classRevivers.map(this.#reviveByClass(seen)), - (f) => f(node), - )).find(Boolean) ?? - node; // Date / URL / RegExp and the like — devalue handled them - - #reviveByClass = - (seen: Seen) => - ([filter, init, set]: ClassReviver) => - async (node: object): Promise => { - if (!filter(node)) return; - // @ts-ignore tsc faults - const out: Revived = await init(node); - seen.set(node, out); - // @ts-ignore tsc faults - await set(this.#revive(seen), node, out); - return out; - }; + // + // Every node walked here belongs to the throwaway tree that devalue's + // `parse` just built from the wire string, not to any caller-shared graph, + // so the revived containers are always fresh: there is nothing to clone + // lazily and no external identity to preserve. A recursion-depth cap is + // likewise unnecessary: this pass recurses with `await`, which unwinds the + // synchronous stack at each level, and the binding limit on nesting is + // devalue's own synchronous, recursive `stringify`/`parse`, which would + // overflow long before this pass — capping depth here would add nothing. + #revive = (seen: Seen): Revive => async (node: unknown): Promise => { + if (node === null || typeof node !== "object") return node; + if (seen.has(node)) return seen.get(node); + // The class filters are mutually exclusive, so find the single matching + // reviver instead of running all of them against every node. + const reviver = this.#classRevivers.find(([filter]) => filter(node)); + // Date / URL / RegExp and the like — devalue already handled them. + if (reviver == null) return node; + const [, init, set] = reviver; + // @ts-ignore tsc faults + const out: Revived = await init(node); + seen.set(node, out); + // @ts-ignore tsc faults + await set(this.#revive(seen), node, out); + return out; + }; #classRevivers = [ [ From 29dec043203049eac9f0978763d1b5b00e4cfd49 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 18:42:10 +0000 Subject: [PATCH 08/57] Start workers for dedicated per-task queues A task may route to its own queue via `defineTask(name, { queue })`, and `resolveTaskQueue()` enqueues its messages there, but `_startQueueInternal()` only listened on the four federation-wide queues (inbox, outbox, fanout, task). A task queue that was none of those got no worker, so its messages were never processed even while `startQueue()` was running. Collect the distinct dedicated queue instances from the task registry and start a worker for each, treating them as part of the "task" selector. Dedupe against the standard queues and against task queues already started on an earlier call so no instance is listened on twice, and let a deployment whose only queues are per-task ones still start: the early return no longer bails out when a dedicated task queue exists. https://github.com/fedify-dev/fedify/pull/803#discussion_r3397749449 Assisted-by: Claude Code:claude-opus-4-8 --- packages/fedify/src/federation/middleware.ts | 41 ++++++++- .../fedify/src/federation/tasks/tasks.test.ts | 92 +++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index ea8b8718d..7ce326d02 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -596,6 +596,10 @@ export class FederationImpl outboxQueueStarted: boolean; fanoutQueueStarted: boolean; taskQueueStarted: boolean; + // Dedicated per-task queues (defineTask({ queue })) that already have a + // worker listening, so a later _startQueueInternal() call does not listen + // on the same instance twice. + startedTaskQueues: Set; manuallyStartQueue: boolean; origin?: FederationOrigin; documentLoaderFactory: DocumentLoaderFactory; @@ -723,6 +727,7 @@ export class FederationImpl this.outboxQueueStarted = false; this.fanoutQueueStarted = false; this.taskQueueStarted = false; + this.startedTaskQueues = new Set(); this.manuallyStartQueue = options.manuallyStartQueue ?? false; if (options.origin != null) { if (typeof options.origin === "string") { @@ -968,9 +973,16 @@ export class FederationImpl signal?: AbortSignal, queue?: keyof FederationQueueOptions, ): Promise { + // Tasks may route to a dedicated queue of their own (defineTask({ queue })) + // even when no federation-wide queue is configured, so a deployment with + // only per-task queues still has work to start. + const hasDedicatedTaskQueue = [...this.taskDefinitions.values()].some( + (def) => def.queue != null, + ); if ( this.inboxQueue == null && this.outboxQueue == null && - this.fanoutQueue == null && this.taskQueue == null + this.fanoutQueue == null && this.taskQueue == null && + !hasDedicatedTaskQueue ) { return; } @@ -1037,6 +1049,33 @@ export class FederationImpl ), ); } + // Dedicated per-task queues belong to the "task" selector. Each distinct + // instance needs its own worker; dedupe against the standard queues and + // against task queues already started on an earlier call so no instance is + // listened on twice. + if (queue == null || queue === "task") { + const standardQueues = new Set( + [this.inboxQueue, this.outboxQueue, this.fanoutQueue, this.taskQueue] + .filter((q): q is MessageQueue => q != null), + ); + for (const def of this.taskDefinitions.values()) { + const taskQueue = def.queue; + if ( + taskQueue == null || standardQueues.has(taskQueue) || + this.startedTaskQueues.has(taskQueue) + ) { + continue; + } + logger.debug("Starting a worker for a dedicated per-task queue."); + this.startedTaskQueues.add(taskQueue); + promises.push( + taskQueue.listen( + (msg) => this.processQueuedTask(ctxData, msg), + { signal }, + ), + ); + } + } await Promise.all(promises); } diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index 6b313f0a9..cbaca519b 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -507,6 +507,98 @@ test("startQueue() task worker", async (t) => { controller.abort(); await listening; }); + + await t.step("starts a worker for a dedicated per-task queue", async () => { + const taskQueue = new MockQueue(); + const dedicated = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: taskQueue }, + }); + federation.defineTask("dedicated", { + schema: stringSchema, + handler: () => {}, + queue: dedicated, + }); + const controller = new AbortController(); + const listening = federation.startQueue(undefined, { + signal: controller.signal, + }); + strictEqual(taskQueue.listenCount, 1); + strictEqual(dedicated.listenCount, 1); + controller.abort(); + await listening; + }); + + await t.step( + "starts a per-task queue even without a federation queue", + async () => { + const dedicated = new MockQueue(); + const federation = createFederation({ ...baseOptions }); + federation.defineTask("dedicated", { + schema: stringSchema, + handler: () => {}, + queue: dedicated, + }); + const controller = new AbortController(); + const listening = federation.startQueue(undefined, { + signal: controller.signal, + }); + strictEqual(dedicated.listenCount, 1); + controller.abort(); + await listening; + }, + ); + + await t.step( + "does not listen twice on a per-task queue shared with a standard queue", + async () => { + const shared = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: shared }, + }); + federation.defineTask("reuses-task-queue", { + schema: stringSchema, + handler: () => {}, + queue: shared, + }); + const controller = new AbortController(); + const listening = federation.startQueue(undefined, { + signal: controller.signal, + }); + strictEqual(shared.listenCount, 1); + controller.abort(); + await listening; + }, + ); + + await t.step( + "routes an enqueued task on a dedicated queue to its handler", + async () => { + const dedicated = new MockQueue(); + const federation = createFederation({ ...baseOptions }); + let received: string | undefined; + const task = federation.defineTask("dedicated-end-to-end", { + schema: stringSchema, + handler: (_ctx, data) => { + received = data; + }, + queue: dedicated, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload"); + strictEqual(dedicated.enqueued.length, 1); + await (federation as FederationImpl).processQueuedTask( + undefined, + dedicated.enqueued[0].message, + ); + strictEqual(received, "payload"); + }, + ); }); test("processQueuedTask() task dispatch", async (t) => { From 36817474af09fde4753183648d7db8596a9d8d56 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 19:11:36 +0000 Subject: [PATCH 09/57] Removed overly detailed comments and unused code --- packages/fedify/src/federation/middleware.ts | 3 -- packages/fedify/src/federation/tasks/codec.ts | 30 +------------------ 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 7ce326d02..3ecfa7047 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -596,9 +596,6 @@ export class FederationImpl outboxQueueStarted: boolean; fanoutQueueStarted: boolean; taskQueueStarted: boolean; - // Dedicated per-task queues (defineTask({ queue })) that already have a - // worker listening, so a later _startQueueInternal() call does not listen - // on the same instance twice. startedTaskQueues: Set; manuallyStartQueue: boolean; origin?: FederationOrigin; diff --git a/packages/fedify/src/federation/tasks/codec.ts b/packages/fedify/src/federation/tasks/codec.ts index 3eab7f52f..0681ce444 100644 --- a/packages/fedify/src/federation/tasks/codec.ts +++ b/packages/fedify/src/federation/tasks/codec.ts @@ -38,24 +38,11 @@ export default class TaskCodec { jsonLd: await value.toJsonLd({ format: "expand", ...this.options }), }); - // The explicit return type breaks the inference cycle between #revive and - // #classRevivers (whose `set` callbacks call back into #revive). - // - // Every node walked here belongs to the throwaway tree that devalue's - // `parse` just built from the wire string, not to any caller-shared graph, - // so the revived containers are always fresh: there is nothing to clone - // lazily and no external identity to preserve. A recursion-depth cap is - // likewise unnecessary: this pass recurses with `await`, which unwinds the - // synchronous stack at each level, and the binding limit on nesting is - // devalue's own synchronous, recursive `stringify`/`parse`, which would - // overflow long before this pass — capping depth here would add nothing. #revive = (seen: Seen): Revive => async (node: unknown): Promise => { if (node === null || typeof node !== "object") return node; if (seen.has(node)) return seen.get(node); - // The class filters are mutually exclusive, so find the single matching - // reviver instead of running all of them against every node. const reviver = this.#classRevivers.find(([filter]) => filter(node)); - // Date / URL / RegExp and the like — devalue already handled them. + // devalue can handled non-container objects. if (reviver == null) return node; const [, init, set] = reviver; // @ts-ignore tsc faults @@ -165,21 +152,6 @@ type Seen = Map; /** Revives one node, sharing the per-decode {@link Seen} map via closure. */ type Revive = (node: unknown) => Promise; -/** - * One row of {@link TaskCodec.#classRevivers}: a type guard, a factory - * that makes the empty revived container, and a filler that walks the source - * into it using the supplied per-node {@link Revive}. `#reviveByClass` - * cannot annotate its parameter as `typeof this.#classRevivers[number]` - * because a `typeof` query on a private field does not parse, so this loose - * structural shape stands in; the `init` and `set` calls are reconciled with - * `@ts-ignore` at the call site. - */ -type ClassReviver = readonly [ - (value: unknown) => boolean, - (node: never) => unknown, - (revive: Revive, node: never, out: never) => Promise, -]; - type Container = | VocabHolder | Map From 7d5c3a1d55a2ba267f47324ae8bfae903827b412 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 19:24:08 +0000 Subject: [PATCH 10/57] Remove *codec-fn.ts* @dahlia, the maintainer picks `TaskCodec` because it carries the loader state on the instance at [a comment](https://github.com/fedify-dev/fedify/pull/803#issuecomment-4684047851). Therefore remove the *codec-fn.ts*. `TaskCodecLoaders` moved to *codec.ts* because `TaskCodec` use it. --- .../src/federation/tasks/codec-fn.test.ts | 230 ------------------ .../fedify/src/federation/tasks/codec-fn.ts | 188 -------------- packages/fedify/src/federation/tasks/codec.ts | 15 +- packages/fedify/src/federation/tasks/mod.ts | 1 - 4 files changed, 14 insertions(+), 420 deletions(-) delete mode 100644 packages/fedify/src/federation/tasks/codec-fn.test.ts delete mode 100644 packages/fedify/src/federation/tasks/codec-fn.ts diff --git a/packages/fedify/src/federation/tasks/codec-fn.test.ts b/packages/fedify/src/federation/tasks/codec-fn.test.ts deleted file mode 100644 index 9cc4b9279..000000000 --- a/packages/fedify/src/federation/tasks/codec-fn.test.ts +++ /dev/null @@ -1,230 +0,0 @@ -import { mockDocumentLoader, test } from "@fedify/fixture"; -import { Create, Link, Note, Person } from "@fedify/vocab"; -import type { StandardSchemaV1 } from "@standard-schema/spec"; -import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; -import { - deserializeTaskData, - serializeTaskData, - validateTaskData, -} from "./codec-fn.ts"; - -const loaders = { - contextLoader: mockDocumentLoader, - documentLoader: mockDocumentLoader, -}; - -function makeSchema( - check: (data: unknown) => data is T, -): StandardSchemaV1 { - return { - "~standard": { - version: 1, - vendor: "fedify-test", - validate(value: unknown) { - return check(value) - ? { value } - : { issues: [{ message: "Invalid task data." }] }; - }, - }, - }; -} - -test("serializeTaskData() / deserializeTaskData()", async (t) => { - const note = new Note({ - id: new URL("https://example.com/notes/1"), - content: "Hello, world!", - }); - const person = new Person({ - id: new URL("https://example.com/users/alice"), - name: "Alice", - }); - const link = new Link({ - href: new URL("https://example.com/"), - mediaType: "text/html", - }); - const create = new Create({ - id: new URL("https://example.com/activities/1"), - actor: new URL("https://example.com/users/alice"), - object: note, - }); - - await t.step("round-trips a mixed payload", async () => { - const payload = { - note, - when: new Date("2026-01-02T03:04:05Z"), - big: 1234567890123456789n, - url: new URL("https://example.com/some/path"), - list: [person, link], - map: new Map([["create", create], ["n", 42]]), - set: new Set([1, 2, 3]), - nested: { create }, - }; - const encoded = await serializeTaskData(payload, mockDocumentLoader); - strictEqual(typeof encoded, "string"); - const decoded = await deserializeTaskData(encoded, loaders) as Record< - string, - unknown - >; - ok(decoded.note instanceof Note); - strictEqual(decoded.note.content?.toString(), "Hello, world!"); - strictEqual(decoded.note.id?.href, "https://example.com/notes/1"); - ok(decoded.when instanceof Date); - strictEqual(decoded.when.toISOString(), "2026-01-02T03:04:05.000Z"); - strictEqual(decoded.big, 1234567890123456789n); - ok(decoded.url instanceof URL); - strictEqual(decoded.url.href, "https://example.com/some/path"); - const list = decoded.list as unknown[]; - ok(list[0] instanceof Person); - strictEqual(list[0].name?.toString(), "Alice"); - ok(list[1] instanceof Link); - strictEqual(list[1].href?.href, "https://example.com/"); - const map = decoded.map as Map; - ok(map.get("create") instanceof Create); - strictEqual(map.get("n"), 42); - deepStrictEqual(decoded.set, new Set([1, 2, 3])); - const nested = decoded.nested as Record; - ok(nested.create instanceof Create); - const nestedObject = await nested.create.getObject({ - documentLoader: mockDocumentLoader, - contextLoader: mockDocumentLoader, - }); - ok(nestedObject instanceof Note); - strictEqual(nestedObject.content?.toString(), "Hello, world!"); - }); - - await t.step( - "encodes vocab objects in expand form (no @context)", - async () => { - const encoded = await serializeTaskData({ note }, mockDocumentLoader); - ok(!encoded.includes("@context")); - }, - ); - - await t.step("leaves a non-vocab payload untouched", async () => { - const payload = { - text: "plain", - n: 1, - flag: true, - nothing: null, - when: new Date("2026-06-10T00:00:00Z"), - list: [1, "two", 3n], - }; - const encoded = await serializeTaskData(payload, mockDocumentLoader); - const decoded = await deserializeTaskData(encoded, loaders); - deepStrictEqual(decoded, payload); - }); - - await t.step("throws on a malformed wire string", async () => { - // deserializeTaskData() throws synchronously on a malformed wire string - // (devalue's parse() runs before the first await); the async wrapper - // funnels both sync throws and rejections into one assertion. - await rejects(async () => await deserializeTaskData("garbage", loaders)); - }); - - await t.step("preserves circular and repeated references", async () => { - const shared = new Note({ content: "shared" }); - interface Cyclic { - name: string; - self?: Cyclic; - notes: Note[]; - } - const payload: Cyclic = { name: "root", notes: [shared, shared] }; - payload.self = payload; - const encoded = await serializeTaskData(payload, mockDocumentLoader); - const decoded = await deserializeTaskData(encoded, loaders) as Cyclic; - strictEqual(decoded.self, decoded); - ok(decoded.notes[0] instanceof Note); - strictEqual(decoded.notes[0], decoded.notes[1]); - strictEqual(decoded.notes[0].content?.toString(), "shared"); - }); - - await t.step("preserves a cycle through an array", async () => { - const list: unknown[] = ["head"]; - list.push(list); - const encoded = await serializeTaskData({ list }, mockDocumentLoader); - const decoded = await deserializeTaskData(encoded, loaders) as { - list: unknown[]; - }; - strictEqual(decoded.list[0], "head"); - strictEqual(decoded.list[1], decoded.list); - }); - - await t.step("preserves cycles re-entering at a Map and a Set", async () => { - // The cycle must re-enter at the Map/Set *itself* (not at a plain - // object) to exercise their pre-registration in the reviver. - const set = new Set(); - set.add({ set }); - const map = new Map(); - map.set("entry", { map }); - const encoded = await serializeTaskData({ set, map }, mockDocumentLoader); - const decoded = await deserializeTaskData(encoded, loaders) as { - set: Set<{ set: Set }>; - map: Map }>; - }; - const [member] = decoded.set; - strictEqual(member.set, decoded.set); - strictEqual(decoded.map.get("entry")?.map, decoded.map); - }); -}); - -test("validateTaskData()", async (t) => { - interface Envelope { - note: Note; - title: string; - } - const schema = makeSchema( - (data): data is Envelope => - typeof data === "object" && data != null && - (data as Envelope).note instanceof Note && - typeof (data as Envelope).title === "string", - ); - - await t.step("accepts a payload with a vocab instanceof leaf", async () => { - const payload = { - note: new Note({ content: "Hi" }), - title: "greeting", - }; - const validated = await validateTaskData(schema, payload); - deepStrictEqual(validated, payload); - }); - - await t.step("rejects a wrong-shaped payload", async () => { - await rejects( - () => validateTaskData(schema, { note: "not a Note", title: 42 }), - { name: "TypeError", message: /Task data failed schema validation/ }, - ); - }); - - await t.step("supports async validation", async () => { - const asyncSchema: StandardSchemaV1 = { - "~standard": { - version: 1, - vendor: "fedify-test", - validate: (value: unknown) => - Promise.resolve( - typeof value === "number" - ? { value } - : { issues: [{ message: "not a number" }] }, - ), - }, - }; - strictEqual(await validateTaskData(asyncSchema, 42), 42); - await rejects(() => validateTaskData(asyncSchema, "nope")); - }); - - await t.step( - "round-trip then validate (same schema on both sides)", - async () => { - const payload = { - note: new Note({ content: "Hi" }), - title: "greeting", - }; - const encoded = await serializeTaskData(payload, mockDocumentLoader); - const decoded = await deserializeTaskData(encoded, loaders); - const validated = await validateTaskData(schema, decoded); - ok(validated.note instanceof Note); - strictEqual(validated.title, "greeting"); - strictEqual(validated.note.content?.toString(), "Hi"); - }, - ); -}); diff --git a/packages/fedify/src/federation/tasks/codec-fn.ts b/packages/fedify/src/federation/tasks/codec-fn.ts deleted file mode 100644 index 0f181d7fc..000000000 --- a/packages/fedify/src/federation/tasks/codec-fn.ts +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Serializes custom-task payloads with [devalue], bridging Activity - * Vocabulary objects (`Note`, `Create`, `Person`, `Link`, and so on) through - * JSON-LD. - * - * Vocabulary objects keep their state in private fields, so devalue cannot - * serialize them directly. devalue's custom-type hook (a reducer on encode, - * a reviver on decode) carries each object as JSON-LD without writing a - * marker into the payload. Encoding uses the *expand* JSON-LD form, which - * has no `@context`, so decoding dereferences nothing and never touches the - * network. - * - * [devalue]: https://github.com/sveltejs/devalue - * - * @module - */ -import { Link, Object as APObject } from "@fedify/vocab"; -import type { DocumentLoader } from "@fedify/vocab-runtime"; -import type { TracerProvider } from "@opentelemetry/api"; -import type { StandardSchemaV1 } from "@standard-schema/spec"; -import { parse, stringifyAsync } from "devalue"; - -/** Which `fromJsonLd` entry point rebuilds a given vocabulary object. */ -type VocabKind = "object" | "link"; - -/** A vocabulary object reduced to its wire form: a kind tag plus JSON-LD. */ -interface VocabWire { - readonly kind: VocabKind; - readonly jsonLd: unknown; -} - -/** - * The loaders a worker {@link Context} already exposes; both decode passes - * use them. - * @internal - */ -export interface TaskCodecLoaders { - readonly contextLoader?: DocumentLoader; - readonly documentLoader?: DocumentLoader; - readonly tracerProvider?: TracerProvider; - readonly baseUrl?: URL; -} - -const isVocab = (value: unknown): value is APObject | Link => - value instanceof APObject || value instanceof Link; - -const isPlainObject = (value: unknown): value is Record => - value === null || typeof value !== "object" - ? false - : globalThis.Object.getPrototypeOf(value) === - globalThis.Object.prototype; - -/** Reduce a vocabulary object to expanded JSON-LD (no `@context`). */ -const vocabToJsonLd = async ( - value: APObject | Link, - contextLoader: DocumentLoader, -): Promise => ({ - kind: value instanceof Link ? "link" : "object", - jsonLd: await value.toJsonLd({ format: "expand", contextLoader }), -}); - -/** Rebuild a vocabulary object from its wire form. */ -const vocabFromJsonLd = ( - { kind, jsonLd }: VocabWire, - loaders: TaskCodecLoaders, -): Promise => - kind === "link" - ? Link.fromJsonLd(jsonLd, loaders) - : APObject.fromJsonLd(jsonLd, loaders); - -/** - * Encodes a task payload to a devalue string. - * - * The reducer is deliberately a plain function, not an `async` one: - * `stringifyAsync` treats a truthy return as a match and awaits it. An - * `async` reducer would return a promise for *every* node, which is always - * truthy, so it would "match" non-vocab values too. The plain - * `isVocab(v) && …` form returns a synchronous `false` for non-vocab nodes - * and the `toJsonLd()` promise only for vocab ones. - * - * @internal - */ -export const serializeTaskData = ( - data: unknown, - contextLoader: DocumentLoader, -): Promise => - stringifyAsync(data, { - Vocab: (value: unknown) => - isVocab(value) && vocabToJsonLd(value, contextLoader), - }); - -/** - * A vocabulary object parked by the synchronous decode reviver, held until - * the async {@link reviveVocab} pass can `fromJsonLd()` it back into an - * instance. - */ -class VocabHolder implements VocabWire { - constructor(readonly kind: VocabKind, readonly jsonLd: unknown) {} - static from = ({ kind, jsonLd }: VocabWire) => new VocabHolder(kind, jsonLd); -} - -/** - * Second decode pass: replace every parked holder with a real instance. - * - * devalue preserves circular and repeated references, so the walker keeps - * a `seen` map from each visited container to its revived counterpart. - * Containers are registered *before* their contents are walked; a cycle - * therefore resolves to the (still-filling) revived container instead of - * recursing forever, and a repeated reference revives to the same instance. - */ -function reviveVocab( - loaders: TaskCodecLoaders, -): (node: unknown) => Promise { - const seen = new Map(); - return async function inner(node: unknown): Promise { - if (node === null || typeof node !== "object") return node; - if (seen.has(node)) return seen.get(node); - if (node instanceof VocabHolder) { - const revived = await vocabFromJsonLd(node, loaders); - seen.set(node, revived); - return revived; - } - if (Array.isArray(node)) { - const out: unknown[] = []; - seen.set(node, out); - out.push(...await Array.fromAsync(node, inner)); - return out; - } - if (node instanceof Map) { - const out = new Map(); - seen.set(node, out); - for (const [k, v] of node) out.set(await inner(k), await inner(v)); - return out; - } - if (node instanceof Set) { - const out = new Set(); - seen.set(node, out); - for (const v of await Array.fromAsync(node, inner)) out.add(v); - return out; - } - if (isPlainObject(node)) { - const out: Record = {}; - seen.set(node, out); - for (const [k, v] of globalThis.Object.entries(node)) { - out[k] = await inner(v); - } - return out; - } - return node; // Date / URL / RegExp and the like — devalue handled them - }; -} - -/** - * Decodes a devalue string back to a task payload. - * - * Two passes are unavoidable: `parse` revivers are synchronous while - * `fromJsonLd()` is async. The reviver only parks each object; - * {@link reviveVocab} then walks the result and awaits `fromJsonLd()`. - * - * @internal - */ -export const deserializeTaskData = ( - raw: string, - loaders: TaskCodecLoaders, -): Promise => - reviveVocab(loaders)( - parse(raw, { - Vocab: ({ kind, jsonLd }: VocabWire) => new VocabHolder(kind, jsonLd), - }), - ); - -/** - * Validates `data` through the vendor-agnostic - * [Standard Schema](https://standardschema.dev/) interface. - * @internal - */ -export const validateTaskData = async ( - schema: S, - data: unknown, -): Promise> => { - const result = await schema["~standard"].validate(data); - if (result.issues) { - throw new TypeError( - `Task data failed schema validation: ${JSON.stringify(result.issues)}`, - ); - } - return result.value; -}; diff --git a/packages/fedify/src/federation/tasks/codec.ts b/packages/fedify/src/federation/tasks/codec.ts index 0681ce444..6f5b0f005 100644 --- a/packages/fedify/src/federation/tasks/codec.ts +++ b/packages/fedify/src/federation/tasks/codec.ts @@ -1,7 +1,8 @@ import { Link, Object as APObject } from "@fedify/vocab"; +import type { DocumentLoader } from "@fedify/vocab-runtime"; +import type { TracerProvider } from "@opentelemetry/api"; import type { StandardSchemaV1 } from "@standard-schema/spec"; import { parse, stringifyAsync } from "devalue"; -import type { TaskCodecLoaders } from "./codec-fn.ts"; export default class TaskCodec { constructor(readonly options: TaskCodecLoaders) {} @@ -128,6 +129,18 @@ function assertSchema( } } +/** + * The loaders a worker {@link Context} already exposes; both decode passes + * use them. + * @internal + */ +interface TaskCodecLoaders { + readonly contextLoader?: DocumentLoader; + readonly documentLoader?: DocumentLoader; + readonly tracerProvider?: TracerProvider; + readonly baseUrl?: URL; +} + /** Which `fromJsonLd` entry point rebuilds a given vocabulary object. */ type VocabKind = "object" | "link"; diff --git a/packages/fedify/src/federation/tasks/mod.ts b/packages/fedify/src/federation/tasks/mod.ts index a40b57bd3..107e0e445 100644 --- a/packages/fedify/src/federation/tasks/mod.ts +++ b/packages/fedify/src/federation/tasks/mod.ts @@ -6,6 +6,5 @@ * * @module */ -export * from "./codec-fn.ts"; export { default as TaskCodec } from "./codec.ts"; export * from "./task.ts"; From 42153cb8397b8204c25c60399d0878cf76174b44 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 19:57:24 +0000 Subject: [PATCH 11/57] Document the idempotent-validation requirement for task schemas The task payload schema validates on both sides of the queue: at enqueue time and again at dequeue time. The wire therefore carries the validated *output*, which the same schema must re-accept as input, so transforming schemas (e.g., Zod's .transform()) whose output differs in shape from their input cannot round-trip. This constraint was neither documented nor tested; state it in the manual and the schema option's JSDoc, and pin it with a regression test. https://github.com/fedify-dev/fedify/pull/803 Assisted-by: Claude Code:claude-fable-5 --- docs/manual/tasks.md | 6 +++++ .../fedify/src/federation/tasks/codec.test.ts | 23 +++++++++++++++++++ packages/fedify/src/federation/tasks/task.ts | 5 +++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index dd52e1ef4..2c266ee1b 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -91,6 +91,12 @@ that an older deployment enqueued. A payload that fails dequeue-time validation (or cannot be decoded at all) is dropped with an error log rather than retried, because retrying cannot make it valid. +Because the same schema validates on both sides of the queue, its validation +must be *idempotent*: the validated output must itself be a valid input. +Transforming schemas (e.g., Zod's `.transform()`) whose output differs in +shape from their input are not supported—the payload type is inferred as the +schema's *output*, so the call site already fails validation at enqueue time. + [devalue]: https://github.com/sveltejs/devalue diff --git a/packages/fedify/src/federation/tasks/codec.test.ts b/packages/fedify/src/federation/tasks/codec.test.ts index d45d39ea1..ca0c65f19 100644 --- a/packages/fedify/src/federation/tasks/codec.test.ts +++ b/packages/fedify/src/federation/tasks/codec.test.ts @@ -295,4 +295,27 @@ test("TaskCodec.encode() / decode()", async (t) => { ); }, ); + + await t.step( + "a non-idempotent (transforming) schema fails to round-trip", + async () => { + // Validation must be idempotent: the wire carries the validated + // output, which the same schema re-validates as input at dequeue. + const transforming: StandardSchemaV1 = { + "~standard": { + version: 1, + vendor: "fedify-test", + validate: (value: unknown) => + typeof value === "string" + ? { value: value.length } + : { issues: [{ message: "Expected a string." }] }, + }, + }; + const wire = await codec.encode(transforming, "hello"); + await rejects( + () => codec.decode(transforming, wire), + { name: "TypeError", message: /Task data failed schema validation/ }, + ); + }, + ); }); diff --git a/packages/fedify/src/federation/tasks/task.ts b/packages/fedify/src/federation/tasks/task.ts index 1318d647b..f7012f5f0 100644 --- a/packages/fedify/src/federation/tasks/task.ts +++ b/packages/fedify/src/federation/tasks/task.ts @@ -34,7 +34,10 @@ export interface TaskDefinitionOptions< * * The payload is validated twice: once at enqueue time (fail fast) and * once at dequeue time (drift protection against payloads enqueued by an - * older deployment). + * older deployment). Because the same schema runs on both sides, its + * validation must be idempotent: the validated output must itself be + * a valid input. Transforming schemas (e.g., Zod's `.transform()`) whose + * output differs in shape from their input are not supported. */ readonly schema: TSchema; From 0cd315539765c9c491e20c24061f60e784faf366 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 20:02:10 +0000 Subject: [PATCH 12/57] Reject enqueuing a task unknown to the federation resolveTaskQueue() returns the fallback queue even for a task name with no registered definition, so enqueuing a handle created by a different federation instance silently succeeded and the worker later dropped the message with only a warning. The task API's contract is to fail fast at the enqueue call site (it already validates the payload there), so check the registry before resolving a queue and throw a TypeError instead. https://github.com/fedify-dev/fedify/pull/803 Assisted-by: Claude Code:claude-fable-5 --- packages/fedify/src/federation/context.ts | 6 ++-- packages/fedify/src/federation/middleware.ts | 10 +++++++ .../fedify/src/federation/tasks/tasks.test.ts | 28 +++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/packages/fedify/src/federation/context.ts b/packages/fedify/src/federation/context.ts index 8375e6a76..a7a629c7c 100644 --- a/packages/fedify/src/federation/context.ts +++ b/packages/fedify/src/federation/context.ts @@ -446,7 +446,8 @@ export interface Context { * @param data The task payload. It is validated against the task's * schema before being enqueued. * @param options Options for enqueuing the task. - * @throws {TypeError} If no message queue is configured for tasks, or if + * @throws {TypeError} If the task is not defined on this federation, + * if no message queue is configured for tasks, or if * the payload fails schema validation. * @since 2.3.0 */ @@ -466,7 +467,8 @@ export interface Context { * @param payloads The task payloads. Each is validated against the * task's schema before being enqueued. * @param options Options for enqueuing the tasks. - * @throws {TypeError} If no message queue is configured for tasks, or if + * @throws {TypeError} If the task is not defined on this federation, + * if no message queue is configured for tasks, or if * a payload fails schema validation. * @since 2.3.0 */ diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 3ecfa7047..1e53960ff 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -3708,6 +3708,16 @@ export class ContextImpl implements Context { items: readonly TData[], options: TaskEnqueueOptions, ): Promise { + // Fail fast on a handle from another federation instance; without this + // check the message would enqueue fine and be dropped by the worker. + if (!this.federation.taskDefinitions.has(task.name)) { + throw new TypeError( + `Task ${ + JSON.stringify(task.name) + } is not defined on this federation; ` + + "pass a handle returned by its defineTask().", + ); + } const queue = this.federation.resolveTaskQueue(task.name); if (queue == null) { throw new TypeError( diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index cbaca519b..7a4bba48f 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -300,6 +300,34 @@ test("Context.enqueueTask() end-to-end", async (t) => { strictEqual(queue.enqueued.length, 0); }); + await t.step( + "rejects a handle from another federation at enqueue", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const other = createFederation({ + ...baseOptions, + queue: { task: new MockQueue() }, + }); + const foreignTask = other.defineTask("foreign", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + () => ctx.enqueueTask(foreignTask, "data"), + { name: "TypeError", message: /is not defined on this federation/ }, + ); + strictEqual(queue.enqueued.length, 0); + }, + ); + await t.step("passes delay and orderingKey through", async () => { const queue = new MockQueue(); const federation = createFederation({ From bd3f2ac190dabb73b957f1c56fdb1f43c43d089f Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 20:05:01 +0000 Subject: [PATCH 13/57] Polish task enqueue path and queue-isolation docs Small follow-ups from review: - Document that tasks must be defined before startQueue() (or the first request); workers for dedicated per-task queues are only registered when the queue machinery starts, so a queue defined later never gets a worker. - Return early from the enqueue path when no payloads are given, instead of reaching enqueueMany()/Promise.all with an empty batch, whose backend behavior is undefined. - Rename #enqueueSingular to #encodeTaskMessage; it encodes and builds a TaskMessage but does not enqueue anything. - Fix a comment typo in the codec. https://github.com/fedify-dev/fedify/pull/803 Assisted-by: Claude Code:claude-fable-5 --- docs/manual/tasks.md | 6 +++++ packages/fedify/src/federation/middleware.ts | 5 +++-- packages/fedify/src/federation/tasks/codec.ts | 2 +- .../fedify/src/federation/tasks/tasks.test.ts | 22 +++++++++++++++++++ 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index 2c266ee1b..cde246b5c 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -215,6 +215,12 @@ const transcodeVideo = federation.defineTask("transcodeVideo", { }); ~~~~ +Workers for dedicated per-task queues are registered when the queue +machinery starts, so define every task before `~Federation.startQueue()` +is called (or, without `~FederationOptions.manuallyStartQueue`, before the +first request is handled); a per-task queue defined later never gets +a worker. + The queue for a task is resolved in order: the per-task `queue`, then the federation's `task` queue, then the outbox queue. Deployments that must *not* silently share the outbox queue can opt out of the last step with diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 1e53960ff..6692c8d23 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -3725,6 +3725,7 @@ export class ContextImpl implements Context { "createFederation() or to defineTask().", ); } + if (items.length < 1) return; const delay = options.delay == null ? undefined : Temporal.Duration.from(options.delay); @@ -3733,7 +3734,7 @@ export class ContextImpl implements Context { // `map` preserves order, and a rejected encode (validation failure) rejects // the whole batch before anything is enqueued, keeping fail-fast intact. const messages: TaskMessage[] = await Promise.all( - items.map(this.#enqueueSingular(task, options)), + items.map(this.#encodeTaskMessage(task, options)), ); const enqueueOptions = { delay, orderingKey: options.orderingKey }; if (messages.length === 1) { @@ -3745,7 +3746,7 @@ export class ContextImpl implements Context { } } - #enqueueSingular = ( + #encodeTaskMessage = ( task: TaskDefinition, options: TaskEnqueueOptions, ) => diff --git a/packages/fedify/src/federation/tasks/codec.ts b/packages/fedify/src/federation/tasks/codec.ts index 6f5b0f005..d0620abb2 100644 --- a/packages/fedify/src/federation/tasks/codec.ts +++ b/packages/fedify/src/federation/tasks/codec.ts @@ -43,7 +43,7 @@ export default class TaskCodec { if (node === null || typeof node !== "object") return node; if (seen.has(node)) return seen.get(node); const reviver = this.#classRevivers.find(([filter]) => filter(node)); - // devalue can handled non-container objects. + // devalue can handle non-container objects. if (reviver == null) return node; const [, init, set] = reviver; // @ts-ignore tsc faults diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index 7a4bba48f..e7582766e 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -399,6 +399,28 @@ test("Context.enqueueTask() end-to-end", async (t) => { strictEqual(queue.enqueued.length, 2); }, ); + + await t.step( + "enqueueTaskMany() with no payloads touches no queue", + async () => { + const queue = new MockQueue({ supportsEnqueueMany: true }); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("bulk-empty", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTaskMany(task, []); + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 0); + }, + ); }); test("task queue routing", async (t) => { From 29a5dcb9cdb85fc1786a46c3b8cbcd76dee20615 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 21:37:06 +0000 Subject: [PATCH 14/57] Type the task codec's reviver dispatch The revival dispatch pulled init/set out of a heterogeneous tuple list, losing the correlation between each tuple's filter and its init/set node type, which forced two @ts-ignore suppressions at the call site. Such suppressions hide any future error on those lines, so they are unfit for a permanent implementation. Each entry is now built by a generic classReviver() factory whose single type parameter ties the filter to its init/set, letting the compiler check the calls it previously could not. Also bind the recursive reviver to one inner closure per decode pass instead of allocating a fresh closure on every dispatch. https://github.com/fedify-dev/fedify/pull/803#issuecomment-4683028319 Assisted-by: Claude Code:claude-fable-5 --- packages/fedify/src/federation/tasks/codec.ts | 105 ++++++++++-------- 1 file changed, 60 insertions(+), 45 deletions(-) diff --git a/packages/fedify/src/federation/tasks/codec.ts b/packages/fedify/src/federation/tasks/codec.ts index d0620abb2..b93366c0b 100644 --- a/packages/fedify/src/federation/tasks/codec.ts +++ b/packages/fedify/src/federation/tasks/codec.ts @@ -39,65 +39,60 @@ export default class TaskCodec { jsonLd: await value.toJsonLd({ format: "expand", ...this.options }), }); - #revive = (seen: Seen): Revive => async (node: unknown): Promise => { - if (node === null || typeof node !== "object") return node; - if (seen.has(node)) return seen.get(node); - const reviver = this.#classRevivers.find(([filter]) => filter(node)); - // devalue can handle non-container objects. - if (reviver == null) return node; - const [, init, set] = reviver; - // @ts-ignore tsc faults - const out: Revived = await init(node); - seen.set(node, out); - // @ts-ignore tsc faults - await set(this.#revive(seen), node, out); - return out; + #revive = (seen: Seen): Revive => { + const inner: Revive = async (node) => { + if (node === null || typeof node !== "object") return node; + if (seen.has(node)) return seen.get(node); + for (const reviver of this.#classRevivers) { + const out = reviver(seen, inner, node); + if (out !== undefined) return await out; + } + // devalue can handle non-container objects. + return node; + }; + return inner; }; - #classRevivers = [ - [ + #classRevivers: readonly ClassReviver[] = [ + classReviver( isInstanceOf(VocabHolder), - ({ kind, jsonLd }: VocabWire): Promise => + ({ kind, jsonLd }): Promise => kind === "link" ? Link.fromJsonLd(jsonLd, this.options) : APObject.fromJsonLd(jsonLd, this.options), - () => Promise.resolve(), - ], - [ + () => {}, + ), + classReviver( isInstanceOf(Array), (): unknown[] => [], - async (revive: Revive, node: unknown[], arr: typeof node) => { + async (revive, node, arr) => { arr.push(...await Array.fromAsync(node, revive)); }, - ], - [ + ), + classReviver( isInstanceOf(Map), - () => new Map(), - async (revive: Revive, node: Map, map: typeof node) => { + () => new Map(), + async (revive, node, map) => { for (const [k, v] of node) map.set(await revive(k), await revive(v)); }, - ], - [ + ), + classReviver( isInstanceOf(Set), - () => new Set(), - async (revive: Revive, node: Set, set: typeof node) => { + () => new Set(), + async (revive, node, set) => { for (const v of await Array.fromAsync(node, revive)) set.add(v); }, - ], - [ + ), + classReviver( isPlainObject, - () => ({}), - async ( - revive: Revive, - node: Record, - obj: typeof node, - ) => { + (): Record => ({}), + async (revive, node, obj) => { for (const [k, v] of globalThis.Object.entries(node)) { obj[k] = await revive(v); } }, - ], - ] as const; + ), + ]; } const isVocab = (value: unknown): value is APObject | Link => @@ -165,12 +160,32 @@ type Seen = Map; /** Revives one node, sharing the per-decode {@link Seen} map via closure. */ type Revive = (node: unknown) => Promise; -type Container = - | VocabHolder - | Map - | Set - | Array - | Record; -type Revived = Exclude | APObject | Link; +/** Revives one matched container, or `undefined` when the node isn't its kind. */ +type ClassReviver = ( + seen: Seen, + revive: Revive, + node: object, +) => Promise | undefined; + +/** + * Ties a container filter to its empty-shell `init` and child-filling `set` + * through one type parameter—a correlation the heterogeneous reviver list + * cannot carry, which previously forced `@ts-ignore` at the dispatch site. + */ +const classReviver = ( + filter: (v: unknown) => v is TNode, + init: (node: TNode) => TOut | Promise, + set: (revive: Revive, node: TNode, out: TOut) => void | Promise, +): ClassReviver => +(seen, revive, node) => { + if (!filter(node)) return undefined; + return (async () => { + const out = await init(node); + seen.set(node, out); + await set(revive, node, out); + return out; + })(); +}; + // deno-lint-ignore no-explicit-any type Constructor = new (...arg: any[]) => T; From f935cd5ac322818927fb3e6b6e34d6391076bfd9 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 21:38:44 +0000 Subject: [PATCH 15/57] Hide the task handle's phantom context-data marker The __contextData phantom field binds a TaskDefinition handle to its federation's context data type, but as a string-keyed property it leaked into user-facing docs and IDE completions despite its @internal tag. Replace it with a module-private unique symbol key: no value exists at runtime, the marker disappears from completions, and cross-federation handle rejection still type-checks, now guarded by a regression test. Also replace the tasks barrel's wildcard re-export of task.ts with explicit named exports of the six types its consumers actually use, so nothing new falls through the barrel unnoticed. https://github.com/fedify-dev/fedify/pull/803#issuecomment-4683028319 Assisted-by: Claude Code:claude-fable-5 --- packages/fedify/src/federation/tasks/mod.ts | 9 ++++++++- packages/fedify/src/federation/tasks/task.ts | 9 ++++++++- packages/fedify/src/federation/tasks/tasks.test.ts | 8 ++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/fedify/src/federation/tasks/mod.ts b/packages/fedify/src/federation/tasks/mod.ts index 107e0e445..e151fa576 100644 --- a/packages/fedify/src/federation/tasks/mod.ts +++ b/packages/fedify/src/federation/tasks/mod.ts @@ -7,4 +7,11 @@ * @module */ export { default as TaskCodec } from "./codec.ts"; -export * from "./task.ts"; +export type { + TaskDefinition, + TaskDefinitionInternal, + TaskDefinitionOptions, + TaskEnqueueOptions, + TaskHandler, + TaskRegistry, +} from "./task.ts"; diff --git a/packages/fedify/src/federation/tasks/task.ts b/packages/fedify/src/federation/tasks/task.ts index f7012f5f0..836e4802f 100644 --- a/packages/fedify/src/federation/tasks/task.ts +++ b/packages/fedify/src/federation/tasks/task.ts @@ -77,6 +77,13 @@ export interface TaskDefinitionOptions< readonly queue?: MessageQueue; } +/** + * Phantom key binding a {@link TaskDefinition} to its federation's context + * data type. Declared only—no value exists at runtime, and the symbol is + * not exported, so the marker stays out of user-facing completions. + */ +declare const contextDataBrand: unique symbol; + /** * The handle returned by {@link TaskRegistry.defineTask}. It carries the * task name and schema so that {@link Context.enqueueTask} can validate the @@ -101,7 +108,7 @@ export interface TaskDefinition { /** * @internal Phantom marker binding the handle to its federation. */ - readonly __contextData?: TContextData; + readonly [contextDataBrand]?: TContextData; } /** diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index e7582766e..5f366b4b8 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -235,6 +235,14 @@ test("task type-level guards", () => { // @ts-expect-error: a wrong-shaped payload must not type-check. return ctx.enqueueTask(task, { n: "not a number" }); }; + const _crossContextHandleIsACompileError = ( + ctx: Context, + task: TaskDefinition<{ tenant: string }, { n: number }>, + ) => { + // @ts-expect-error: a handle bound to different context data must not + // type-check. + return ctx.enqueueTask(task, { n: 1 }); + }; }); test("Context.enqueueTask() end-to-end", async (t) => { From 7be32def86bc170447eb2939c05958062925e705 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 22:11:58 +0000 Subject: [PATCH 16/57] Pin deep-nesting support in the task codec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automated reviewers keep proposing a fixed recursion depth cap (~100) in TaskCodec's #revive to guard against stack overflow from deeply nested payloads. The concern does not apply: the revive traversal suspends at an await on every level, so nesting depth consumes heap (promise chains) rather than native stack, and a structure deep enough to threaten the stack would fail inside devalue.parse() before #revive ever ran. A cap would only reject legitimate payloads. Add a regression test that round-trips a payload nested 1,000 levels deep—an order of magnitude above any proposed cap—through alternating objects and arrays down to a vocab leaf, so introducing such a cap now fails the suite. https://github.com/fedify-dev/fedify/pull/803#discussion_r3399383165 Assisted-by: Claude Code:claude-fable-5 --- .../fedify/src/federation/tasks/codec.test.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/fedify/src/federation/tasks/codec.test.ts b/packages/fedify/src/federation/tasks/codec.test.ts index ca0c65f19..b194cda6c 100644 --- a/packages/fedify/src/federation/tasks/codec.test.ts +++ b/packages/fedify/src/federation/tasks/codec.test.ts @@ -172,6 +172,31 @@ test("TaskCodec (fresh instance per operation)", async (t) => { strictEqual(decoded.wrap.note.content?.toString(), "Hello, world!"); }, ); + + await t.step( + "revives a payload nested far deeper than any fixed depth cap", + async () => { + // `#revive` suspends at an `await` on every level, so nesting depth + // consumes heap (promise chains) rather than native stack—deep + // payloads cannot overflow it, and a fixed depth cap would only + // reject legitimate data. Pins a depth an order of magnitude above + // any such cap. + const depth = 1000; + let payload: unknown = new Note({ content: "deep" }); + for (let i = 0; i < depth; i++) { + payload = i % 2 === 0 ? { inner: payload } : [payload]; + } + const encoded = await codec.serialize(payload); + let decoded = await codec.deserialize(encoded); + for (let i = depth - 1; i >= 0; i--) { + decoded = i % 2 === 0 + ? (decoded as { inner: unknown }).inner + : (decoded as unknown[])[0]; + } + ok(decoded instanceof Note); + strictEqual(decoded.content?.toString(), "deep"); + }, + ); }); test("TaskCodec (one instance reused across decodes)", async (t) => { From 5e14a015c3b4ba732f7c8c09b02f52bdf0e3378a Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Fri, 12 Jun 2026 05:11:23 +0000 Subject: [PATCH 17/57] Reject foreign task handles by identity, not name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The enqueue guard only checked that the handle's task name existed in the local registry, so once two federation instances defined the same task name, a handle from the other instance slipped through: the local context encoded the payload under the schema carried by the foreign handle while the worker decoded it under the local definition's schema. A payload the local schema would have rejected at enqueue thus landed in the queue anyway, only to be dropped at decode time—defeating the fail-fast purpose the guard exists for. defineTask() now stores the exact handle object it returns alongside the internal definition, and enqueueTask()/enqueueTaskMany() compare that handle by identity. Handles still work on every federation built from the same builder, since build() shares the stored definitions. The cross-federation regression test now covers the same-name case in addition to the undefined-name case. https://github.com/fedify-dev/fedify/pull/803#discussion_r3399385300 Assisted-by: Claude Code:claude-fable-5 --- packages/fedify/src/federation/builder.ts | 7 ++- packages/fedify/src/federation/middleware.ts | 7 ++- packages/fedify/src/federation/tasks/task.ts | 7 +++ .../fedify/src/federation/tasks/tasks.test.ts | 45 +++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/packages/fedify/src/federation/builder.ts b/packages/fedify/src/federation/builder.ts index 59460f3f9..e1c35b6fc 100644 --- a/packages/fedify/src/federation/builder.ts +++ b/packages/fedify/src/federation/builder.ts @@ -610,16 +610,21 @@ export class FederationBuilderImpl if (this.taskDefinitions.has(name)) { throw new TypeError(`Task ${JSON.stringify(name)} is already defined.`); } + const handle: TaskDefinition< + TContextData, + StandardSchemaV1.InferOutput + > = { name, schema: options.schema }; this.taskDefinitions.set(name, { name, schema: options.schema, + handle, handler: options.handler as TaskHandler, onError: options .onError as TaskDefinitionInternal["onError"], retryPolicy: options.retryPolicy, queue: options.queue, }); - return { name, schema: options.schema }; + return handle; } /** diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 6692c8d23..a7e0d5d14 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -3710,7 +3710,12 @@ export class ContextImpl implements Context { ): Promise { // Fail fast on a handle from another federation instance; without this // check the message would enqueue fine and be dropped by the worker. - if (!this.federation.taskDefinitions.has(task.name)) { + // Compare the registered handle by identity, not just the name: another + // instance may define the same task name with a different schema, and + // its handle would otherwise encode under that foreign schema here + // while the worker decodes under the local one. + const def = this.federation.taskDefinitions.get(task.name); + if (def == null || def.handle !== task) { throw new TypeError( `Task ${ JSON.stringify(task.name) diff --git a/packages/fedify/src/federation/tasks/task.ts b/packages/fedify/src/federation/tasks/task.ts index 836e4802f..a11ba5f9a 100644 --- a/packages/fedify/src/federation/tasks/task.ts +++ b/packages/fedify/src/federation/tasks/task.ts @@ -168,6 +168,13 @@ export interface TaskEnqueueOptions { export interface TaskDefinitionInternal { readonly name: string; readonly schema: StandardSchemaV1; + /** + * The exact handle object {@link TaskRegistry.defineTask} returned for + * this definition. {@link Context.enqueueTask} compares it by identity: + * another federation instance may define the same task name with a + * different schema, so name lookup alone cannot tell its handle apart. + */ + readonly handle: TaskDefinition; readonly handler: TaskHandler; readonly retryPolicy?: RetryPolicy; readonly onError?: ( diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index 5f366b4b8..8ce9bf15f 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -336,6 +336,51 @@ test("Context.enqueueTask() end-to-end", async (t) => { }, ); + await t.step( + "rejects a same-named handle from another federation", + async () => { + // Name lookup alone cannot tell a foreign handle apart once both + // instances define the same task name: the local context would + // encode under the *schema carried by the foreign handle*, so a + // payload the local schema rejects would enqueue anyway, only to be + // dropped by the worker decoding under the local schema. Both + // instances share TContextData = void, so the phantom-brand check + // cannot reject this at compile time; the handle-identity guard is + // the only defense. + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + let called = 0; + federation.defineTask("rename", { + schema: numberSchema, // the local "rename" takes a number… + handler: () => { + called++; + }, + }); + const other = createFederation({ + ...baseOptions, + queue: { task: new MockQueue() }, + }); + // …while the other instance's "rename" takes a string: + const foreignTask = other.defineTask("rename", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + () => ctx.enqueueTask(foreignTask, "not a number"), + { name: "TypeError", message: /is not defined on this federation/ }, + ); + strictEqual(queue.enqueued.length, 0); + strictEqual(called, 0); + }, + ); + await t.step("passes delay and orderingKey through", async () => { const queue = new MockQueue(); const federation = createFederation({ From f7a9f388d72c52e5e3fdf5f0be0540b0ce85d02d Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Fri, 12 Jun 2026 05:47:07 +0000 Subject: [PATCH 18/57] Validate task payloads in the testing mock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MockContext.enqueueTask() invoked the handler with the raw input, while production enqueueTask() validates the payload against the task schema and hands the validated output to the handler. Tests written against @fedify/testing therefore accepted payloads that production rejects at enqueue, and observed the raw input rather than the coerced or normalized value a transforming schema produces—masking integration bugs the mock exists to surface. The mock now runs the registered schema's Standard Schema validator before invoking the handler, throwing the same TypeError production throws on failure and passing the validated output through. enqueueTaskMany() inherits this since it delegates to enqueueTask(). Added tests covering a rejected payload, a coercing schema whose validated output reaches the handler, and per-item validation in the batch path. https://github.com/fedify-dev/fedify/pull/803#discussion_r3399385307 Assisted-by: Claude Code:claude-fable-5 --- packages/testing/package.json | 1 + packages/testing/src/mock.test.ts | 87 +++++++++++++++++++++++++++++++ packages/testing/src/mock.ts | 12 ++++- pnpm-lock.yaml | 3 ++ 4 files changed, 102 insertions(+), 1 deletion(-) diff --git a/packages/testing/package.json b/packages/testing/package.json index c55790186..39efce142 100644 --- a/packages/testing/package.json +++ b/packages/testing/package.json @@ -55,6 +55,7 @@ "devDependencies": { "@fedify/fixture": "workspace:^", "@js-temporal/polyfill": "catalog:", + "@standard-schema/spec": "catalog:", "@std/assert": "catalog:", "@std/async": "catalog:", "tsdown": "catalog:", diff --git a/packages/testing/src/mock.test.ts b/packages/testing/src/mock.test.ts index 9f2fcd85e..cd14c343c 100644 --- a/packages/testing/src/mock.test.ts +++ b/packages/testing/src/mock.test.ts @@ -1,5 +1,6 @@ import type { InboxContext, OutboxContext } from "@fedify/fedify/federation"; import { signJsonLd } from "@fedify/fedify/sig"; +import type { StandardSchemaV1 } from "@standard-schema/spec"; import { mockDocumentLoader, test } from "@fedify/fixture"; import { Activity, @@ -1711,3 +1712,89 @@ test("MockContext.getActorKeyPairs() returns empty array when no dispatcher regi const keyPairs = await context.getActorKeyPairs("alice"); assertEquals(keyPairs, []); }); + +// A Standard Schema that only accepts numbers; mirrors a strict task schema. +const numberSchema: StandardSchemaV1 = { + "~standard": { + version: 1, + vendor: "fedify-test", + validate: (value: unknown) => + typeof value === "number" + ? { value } + : { issues: [{ message: "Expected a number." }] }, + }, +}; + +test("MockContext.enqueueTask rejects a payload the schema refuses", async () => { + const federation = createFederation(); + let called = 0; + const task = federation.defineTask("count", { + schema: numberSchema, + handler: () => { + called++; + }, + }); + const context = federation.createContext( + new URL("https://example.com"), + undefined, + ); + await assertRejects( + () => context.enqueueTask(task, "not a number" as unknown as number), + TypeError, + "Task data failed schema validation", + ); + assertEquals(called, 0); // production fails fast at enqueue; so must the mock +}); + +test("MockContext.enqueueTask passes the schema's validated output to the handler", async () => { + const federation = createFederation(); + // A coercing schema: uppercases the string. Input and output share the + // same type, but the validated value differs from the raw input. + const upper: StandardSchemaV1 = { + "~standard": { + version: 1, + vendor: "fedify-test", + validate: (value: unknown) => + typeof value === "string" + ? { value: value.toUpperCase() } + : { issues: [{ message: "Expected a string." }] }, + }, + }; + let received = "UNSET"; + const task = federation.defineTask("shout", { + schema: upper, + handler: (_ctx, data) => { + received = data; + }, + }); + const context = federation.createContext( + new URL("https://example.com"), + undefined, + ); + await context.enqueueTask(task, "hi"); + // The handler observes the validated output, not the raw "hi". + assertEquals(received, "HI"); +}); + +test("MockContext.enqueueTaskMany validates every payload", async () => { + const federation = createFederation(); + const seen: number[] = []; + const task = federation.defineTask("count-many", { + schema: numberSchema, + handler: (_ctx, data) => { + seen.push(data); + }, + }); + const context = federation.createContext( + new URL("https://example.com"), + undefined, + ); + // The second item is invalid: the batch rejects and the first item, which + // ran before it, is the only one the handler saw. + await assertRejects( + () => context.enqueueTaskMany(task, [1, "two" as unknown as number]), + TypeError, + "Task data failed schema validation", + ); + assertEquals(seen, [1]); +}); diff --git a/packages/testing/src/mock.ts b/packages/testing/src/mock.ts index 74cfe1a0d..6c1101fa2 100644 --- a/packages/testing/src/mock.ts +++ b/packages/testing/src/mock.ts @@ -977,7 +977,17 @@ class MockContext implements Context { if (def == null) { throw new TypeError(`Task ${JSON.stringify(task.name)} is not defined.`); } - await def.handler(this, data); + // Mirror production: validate against the schema and hand the *validated* + // output to the handler. Without this, the mock would accept payloads + // that production rejects at enqueue, and a normalizing schema's output + // (defaults, coercions) would differ between tests and production. + const result = await def.schema["~standard"].validate(data); + if (result.issues != null && result.issues.length > 0) { + throw new TypeError( + `Task data failed schema validation: ${JSON.stringify(result.issues)}`, + ); + } + await def.handler(this, result.value); } async enqueueTaskMany( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 63cedfc77..3bf0cf09c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1736,6 +1736,9 @@ importers: '@js-temporal/polyfill': specifier: 'catalog:' version: 0.5.1 + '@standard-schema/spec': + specifier: 'catalog:' + version: 1.1.0 '@std/assert': specifier: 'catalog:' version: '@jsr/std__assert@1.0.13' From a2deb39f381d307ef84e7d7b3cb6aa72d6c750d7 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Fri, 12 Jun 2026 06:06:18 +0000 Subject: [PATCH 19/57] Hoist @standard-schema/spec to the workspace root The @standard-schema/spec import is shared by the fedify and testing packages, so it belongs at the workspace level rather than being declared per package. The root deno.json already lists it and workspace members inherit the root import map, making the copy in the fedify package's deno.json redundant; drop it. The pnpm side already sources the version from the catalog in pnpm-workspace.yaml, with each package.json referencing it as "catalog:". Assisted-by: Claude Code:claude-fable-5 --- deno.lock | 1 - packages/fedify/deno.json | 1 - 2 files changed, 2 deletions(-) diff --git a/deno.lock b/deno.lock index b24477b71..0f12de536 100644 --- a/deno.lock +++ b/deno.lock @@ -9627,7 +9627,6 @@ }, "packages/fedify": { "dependencies": [ - "jsr:@standard-schema/spec@^1.1.0", "jsr:@std/assert@0.226", "jsr:@std/url@~0.225.1", "npm:@multiformats/base-x@^4.0.1", diff --git a/packages/fedify/deno.json b/packages/fedify/deno.json index 910c4cd2d..4c0b38b5e 100644 --- a/packages/fedify/deno.json +++ b/packages/fedify/deno.json @@ -15,7 +15,6 @@ }, "imports": { "@multiformats/base-x": "npm:@multiformats/base-x@^4.0.1", - "@standard-schema/spec": "jsr:@standard-schema/spec@^1.1.0", "@std/assert": "jsr:@std/assert@^0.226.0", "@std/url": "jsr:@std/url@^0.225.1", "devalue": "npm:devalue@^5.8.1", From 33dfd68350ad4604304e3efdf88b5e307194b76b Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Fri, 12 Jun 2026 08:04:19 +0000 Subject: [PATCH 20/57] Start the queue worker on custom task enqueue Every other enqueue path (inbox, outbox, fanout, forwarding) calls _startQueueInternal() right before enqueuing unless manuallyStartQueue is set, but #enqueueTasks did not. An application that only uses the custom task API never sends an activity, so with the default configuration its first enqueueTask() accepted the message while no worker ever listened: tasks piled up in the queue unprocessed until startQueue() was called explicitly or an activity happened to be sent. Add the same guard to #enqueueTasks, plus a regression test asserting that the first enqueue starts the task worker exactly once and that a second enqueue does not start another listener. https://github.com/fedify-dev/fedify/pull/803#discussion_r3401351190 Assisted-by: Claude Code:claude-fable-5 --- packages/fedify/src/federation/middleware.ts | 3 ++ .../fedify/src/federation/tasks/tasks.test.ts | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index a7e0d5d14..f4f2fa2cd 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -3741,6 +3741,9 @@ export class ContextImpl implements Context { const messages: TaskMessage[] = await Promise.all( items.map(this.#encodeTaskMessage(task, options)), ); + if (!this.federation.manuallyStartQueue) { + this.federation._startQueueInternal(this.data); + } const enqueueOptions = { delay, orderingKey: options.orderingKey }; if (messages.length === 1) { await queue.enqueue(messages[0], enqueueOptions); diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index 8ce9bf15f..87477c336 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -308,6 +308,35 @@ test("Context.enqueueTask() end-to-end", async (t) => { strictEqual(queue.enqueued.length, 0); }); + await t.step( + "starts the task worker on first enqueue without startQueue()", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + manuallyStartQueue: false, + queue: { task: queue }, + }); + const task = federation.defineTask("auto-start", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + // An app that only uses the custom task API never sends an activity, + // so enqueueTask() itself must start the worker like the other + // enqueue paths do; otherwise tasks pile up unprocessed forever. + await ctx.enqueueTask(task, "first"); + strictEqual(queue.listenCount, 1); + // The started flag keeps a second enqueue from re-listening. + await ctx.enqueueTask(task, "second"); + strictEqual(queue.listenCount, 1); + strictEqual(queue.enqueued.length, 2); + }, + ); + await t.step( "rejects a handle from another federation at enqueue", async () => { From 8e41a7ed3b8daa557fde2883c65300700793d2f1 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Fri, 12 Jun 2026 08:15:19 +0000 Subject: [PATCH 21/57] Vet the whole mock task batch before any handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production's enqueueTaskMany() validates and encodes every payload with Promise.all() before enqueuing anything, so a batch with one invalid item rejects with no effect. The mock looped enqueueTask() per item instead, invoking handlers for earlier payloads before a later one failed validation—tests could observe a partial processing state that cannot occur in production. Split the definition lookup and the schema validation out of enqueueTask() into helpers, and make enqueueTaskMany() validate the whole batch up front, running handlers only once every payload has passed. The existing batch-validation test now pins that no handler runs at all when the batch rejects. https://github.com/fedify-dev/fedify/pull/803#discussion_r3401351204 https://github.com/fedify-dev/fedify/pull/803#discussion_r3401416242 Assisted-by: Claude Code:claude-fable-5 --- packages/testing/src/mock.test.ts | 9 ++++---- packages/testing/src/mock.ts | 35 ++++++++++++++++++++++--------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/packages/testing/src/mock.test.ts b/packages/testing/src/mock.test.ts index cd14c343c..eaf07c4e8 100644 --- a/packages/testing/src/mock.test.ts +++ b/packages/testing/src/mock.test.ts @@ -1776,7 +1776,7 @@ test("MockContext.enqueueTask passes the schema's validated output to the handle assertEquals(received, "HI"); }); -test("MockContext.enqueueTaskMany validates every payload", async () => { +test("MockContext.enqueueTaskMany validates the whole batch before any handler runs", async () => { const federation = createFederation(); const seen: number[] = []; const task = federation.defineTask("count-many", { @@ -1789,12 +1789,13 @@ test("MockContext.enqueueTaskMany validates every payload", async () => { new URL("https://example.com"), undefined, ); - // The second item is invalid: the batch rejects and the first item, which - // ran before it, is the only one the handler saw. + // The second item is invalid: production validates every payload before + // enqueuing anything, so the whole batch rejects with no effect. The + // mock must not let the first handler run before the batch is vetted. await assertRejects( () => context.enqueueTaskMany(task, [1, "two" as unknown as number]), TypeError, "Task data failed schema validation", ); - assertEquals(seen, [1]); + assertEquals(seen, []); }); diff --git a/packages/testing/src/mock.ts b/packages/testing/src/mock.ts index 6c1101fa2..24dfb9636 100644 --- a/packages/testing/src/mock.ts +++ b/packages/testing/src/mock.ts @@ -967,9 +967,7 @@ class MockContext implements Context { return Promise.resolve(null); } - // No queue in mock type: the task handler is invoked immediately, - // mirroring how processQueuedTask() processes immediately. - async enqueueTask(task: any, data: any, _options?: any): Promise { + #resolveTaskDefinition(task: any): any { if (!(this.federation instanceof MockFederation)) { throw new TypeError("No task definitions are available."); } @@ -977,25 +975,42 @@ class MockContext implements Context { if (def == null) { throw new TypeError(`Task ${JSON.stringify(task.name)} is not defined.`); } - // Mirror production: validate against the schema and hand the *validated* - // output to the handler. Without this, the mock would accept payloads - // that production rejects at enqueue, and a normalizing schema's output - // (defaults, coercions) would differ between tests and production. + return def; + } + + // Mirror production: validate against the schema and hand the *validated* + // output to the handler. Without this, the mock would accept payloads + // that production rejects at enqueue, and a normalizing schema's output + // (defaults, coercions) would differ between tests and production. + async #validateTaskPayload(def: any, data: any): Promise { const result = await def.schema["~standard"].validate(data); if (result.issues != null && result.issues.length > 0) { throw new TypeError( `Task data failed schema validation: ${JSON.stringify(result.issues)}`, ); } - await def.handler(this, result.value); + return result.value; + } + + // No queue in mock type: the task handler is invoked immediately, + // mirroring how processQueuedTask() processes immediately. + async enqueueTask(task: any, data: any, _options?: any): Promise { + const def = this.#resolveTaskDefinition(task); + await def.handler(this, await this.#validateTaskPayload(def, data)); } async enqueueTaskMany( task: any, payloads: readonly any[], - options?: any, + _options?: any, ): Promise { - for (const data of payloads) await this.enqueueTask(task, data, options); + const def = this.#resolveTaskDefinition(task); + // Mirror production: the whole batch validates before anything runs, so + // a failing payload rejects the batch with no partial processing. + const values = await Promise.all( + payloads.map((data) => this.#validateTaskPayload(def, data)), + ); + for (const value of values) await def.handler(this, value); } clone(data: TContextData): TestContext { From 45b4e895defe66f538524f42f96b2040490891db Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Fri, 12 Jun 2026 08:19:16 +0000 Subject: [PATCH 22/57] Reject foreign task handles in the testing mock Production compares the registered handle by identity (14313a105), so passing a handle from another federation instance throws even when both instances define the same task name. The mock looked definitions up by name only, and defineTask() did not keep the handle it returned, so an identity check was impossible: tests could pass with a handle the real federation rejects. Store the returned handle with the definition and require the enqueued handle to be that very object, with the same error message production uses. A regression test defines the same task name on two mock federations and asserts the foreign handle is rejected without running any handler. https://github.com/fedify-dev/fedify/pull/803#discussion_r3401351212 Assisted-by: Claude Code:claude-fable-5 --- packages/testing/src/mock.test.ts | 29 +++++++++++++++++++++++++++++ packages/testing/src/mock.ts | 17 +++++++++++++---- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/packages/testing/src/mock.test.ts b/packages/testing/src/mock.test.ts index eaf07c4e8..c6048b5a6 100644 --- a/packages/testing/src/mock.test.ts +++ b/packages/testing/src/mock.test.ts @@ -1799,3 +1799,32 @@ test("MockContext.enqueueTaskMany validates the whole batch before any handler r ); assertEquals(seen, []); }); + +test("MockContext.enqueueTask rejects a handle from another federation", async () => { + const federation = createFederation(); + const other = createFederation(); + let called = 0; + federation.defineTask("shared-name", { + schema: numberSchema, + handler: () => { + called++; + }, + }); + // Same task name on another federation: production compares the registered + // handle by identity and rejects the foreign one, so a name-only lookup in + // the mock would let tests pass with a handle the real federation refuses. + const foreign = other.defineTask("shared-name", { + schema: numberSchema, + handler: () => {}, + }); + const context = federation.createContext( + new URL("https://example.com"), + undefined, + ); + await assertRejects( + () => context.enqueueTask(foreign, 1), + TypeError, + "is not defined on this federation", + ); + assertEquals(called, 0); +}); diff --git a/packages/testing/src/mock.ts b/packages/testing/src/mock.ts index 24dfb9636..60fb98db8 100644 --- a/packages/testing/src/mock.ts +++ b/packages/testing/src/mock.ts @@ -324,8 +324,12 @@ class MockFederation implements Federation { if (this.taskDefinitions.has(name)) { throw new TypeError(`Task ${JSON.stringify(name)} is already defined.`); } - this.taskDefinitions.set(name, { name, ...options }); - return { name, schema: options.schema }; + // Keep the returned handle with the definition: enqueue compares the + // handle by identity, as production does, so a same-named handle from + // another federation instance is rejected rather than looked up by name. + const handle = { name, schema: options.schema }; + this.taskDefinitions.set(name, { name, ...options, handle }); + return handle; } // Note: Parameter type is `any` instead of WebFingerLinksDispatcher to avoid @@ -972,8 +976,13 @@ class MockContext implements Context { throw new TypeError("No task definitions are available."); } const def = this.federation.taskDefinitions.get(task.name); - if (def == null) { - throw new TypeError(`Task ${JSON.stringify(task.name)} is not defined.`); + if (def == null || def.handle !== task) { + throw new TypeError( + `Task ${ + JSON.stringify(task.name) + } is not defined on this federation; ` + + "pass a handle returned by its defineTask().", + ); } return def; } From 80b1d129e35a325ed8c9b4ba96c9c11eba448d5d Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Fri, 12 Jun 2026 18:11:48 +0000 Subject: [PATCH 23/57] Use 2.x.x @since for new task APIs The custom background task APIs added on this branch were annotated with @since 2.3.0, but the release that will include them is not yet decided. Replace those tags with the placeholder 2.x.x so the documentation does not promise a specific version prematurely. Affected APIs: Context.enqueueTask and enqueueTaskMany, the taskRetryPolicy and taskQueueResolution federation options, the task queue option, TaskMessage, and the task definition types (TaskHandler, TaskDefinitionOptions, TaskDefinition, TaskRegistry, and TaskEnqueueOptions). Assisted-by: Claude Code:claude-opus-4-8 --- packages/fedify/src/federation/context.ts | 4 ++-- packages/fedify/src/federation/federation.ts | 4 ++-- packages/fedify/src/federation/middleware.ts | 2 +- packages/fedify/src/federation/queue.ts | 2 +- packages/fedify/src/federation/tasks/task.ts | 10 +++++----- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/fedify/src/federation/context.ts b/packages/fedify/src/federation/context.ts index a7a629c7c..fda194530 100644 --- a/packages/fedify/src/federation/context.ts +++ b/packages/fedify/src/federation/context.ts @@ -449,7 +449,7 @@ export interface Context { * @throws {TypeError} If the task is not defined on this federation, * if no message queue is configured for tasks, or if * the payload fails schema validation. - * @since 2.3.0 + * @since 2.x.x */ enqueueTask( task: TaskDefinition, @@ -470,7 +470,7 @@ export interface Context { * @throws {TypeError} If the task is not defined on this federation, * if no message queue is configured for tasks, or if * a payload fails schema validation. - * @since 2.3.0 + * @since 2.x.x */ enqueueTaskMany( task: TaskDefinition, diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index dbb5231f1..baa00edf2 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -1086,7 +1086,7 @@ export interface FederationOptions { * this uses an exponential backoff strategy with a maximum of 10 attempts * and a maximum delay of 12 hours. A per-task retry policy * ({@link TaskDefinitionOptions.retryPolicy}) overrides this. - * @since 2.3.0 + * @since 2.x.x */ taskRetryPolicy?: RetryPolicy; @@ -1099,7 +1099,7 @@ export interface FederationOptions { * - `"strict"`: no fallback; enqueuing the task throws instead of * silently sharing the outbox queue. * @default `"fallback"` - * @since 2.3.0 + * @since 2.x.x */ taskQueueResolution?: "fallback" | "strict"; diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index f4f2fa2cd..e4b956dd5 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -495,7 +495,7 @@ export interface FederationQueueOptions { * The message queue for custom background tasks. If not provided, * tasks are routed to the outbox queue (unless * {@link FederationOptions.taskQueueResolution} is `"strict"`). - * @since 2.3.0 + * @since 2.x.x */ readonly task?: MessageQueue; } diff --git a/packages/fedify/src/federation/queue.ts b/packages/fedify/src/federation/queue.ts index 5bef2abfc..a8e862a1e 100644 --- a/packages/fedify/src/federation/queue.ts +++ b/packages/fedify/src/federation/queue.ts @@ -79,7 +79,7 @@ export interface OutboxMessage { * A message that carries a custom background task. Every field is * a string, number, or plain record so that the message survives both * JSON serialization and structured clone on every queue backend. - * @since 2.3.0 + * @since 2.x.x */ export interface TaskMessage { readonly type: "task"; diff --git a/packages/fedify/src/federation/tasks/task.ts b/packages/fedify/src/federation/tasks/task.ts index a11ba5f9a..0105e8e21 100644 --- a/packages/fedify/src/federation/tasks/task.ts +++ b/packages/fedify/src/federation/tasks/task.ts @@ -10,7 +10,7 @@ import type { RetryPolicy } from "../retry.ts"; * schema. * @param ctx The context for the worker processing the task. * @param data The decoded and validated task payload. - * @since 2.3.0 + * @since 2.x.x */ export type TaskHandler = ( ctx: Context, @@ -22,7 +22,7 @@ export type TaskHandler = ( * @template TContextData The context data to pass to the {@link Context}. * @template TSchema The [Standard Schema](https://standardschema.dev/) that * validates the task payload. - * @since 2.3.0 + * @since 2.x.x */ export interface TaskDefinitionOptions< TContextData, @@ -91,7 +91,7 @@ declare const contextDataBrand: unique symbol; * @template TContextData The context data to pass to the {@link Context}. * @template TData The type of the task payload, inferred from the task's * schema. - * @since 2.3.0 + * @since 2.x.x */ export interface TaskDefinition { /** @@ -115,7 +115,7 @@ export interface TaskDefinition { * Registration of custom background tasks. Both {@link Federation} and * {@link FederationBuilder} implement this interface. * @template TContextData The context data to pass to the {@link Context}. - * @since 2.3.0 + * @since 2.x.x */ export interface TaskRegistry { /** @@ -146,7 +146,7 @@ export interface TaskRegistry { /** * Options for {@link Context.enqueueTask} and {@link Context.enqueueTaskMany}. - * @since 2.3.0 + * @since 2.x.x */ export interface TaskEnqueueOptions { /** From 193025ba97ec544f1fdb5b0e537f0cb15131d871 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Fri, 12 Jun 2026 18:34:25 +0000 Subject: [PATCH 24/57] Fix doc and comment inaccuracies found in review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local review of the custom background task PR flagged several documentation-level problems; no runtime behavior is affected: - The tasks manual claimed the API ships in Fedify 2.3.0, while the new APIs' JSDoc had already moved to `@since 2.x.x` because the containing release is undecided. Align the manual with the JSDoc. - The manual also claimed a per-task queue defined after the queue machinery starts "never gets a worker." Without `manuallyStartQueue`, the next request or enqueue starts the worker, so soften the claim to match the implementation. - A comment in codec.test.ts described an instance-level `#seen` map that does not exist—each `deserialize()` call builds its own per-decode map—and the first test's title claimed a fresh instance per operation while the tests share one module-level codec. Correct both. - Fix grammar errors in the new *AGENTS.md* paragraph about `mise tasks`. https://github.com/fedify-dev/fedify/pull/803 Assisted-by: Claude Code:claude-fable-5 Assisted-by: Codex:gpt-5-5 --- AGENTS.md | 6 +++--- docs/manual/tasks.md | 6 +++--- packages/fedify/src/federation/tasks/codec.test.ts | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 458069844..653114446 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -190,9 +190,9 @@ Common tasks ### **BE WELL-ACQUAINTED WITH `mise tasks`** *mise.toml* has useful tasks. **Acquaint all of them** and use them in the right -place at the right time. If it has too many information, use `mise tasks`. This -command show the summary of the tasks and descriptions. If `mise tasks` didn't -not make it sure, use `mise tasks ` to check the details for the task. +place at the right time. If it has too much information, use `mise tasks`. This +command shows the summary of the tasks and descriptions. If `mise tasks` does +not make it clear, use `mise tasks ` to check the details for the task. ### Adding ActivityPub vocabulary types diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index cde246b5c..ec5ca09bf 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -1,7 +1,7 @@ Background tasks ================ -*This API is available since Fedify 2.3.0.* +*This API is available since Fedify 2.x.x.* Fedify already processes outgoing and incoming activities on background workers through its [message queue](./mq.md). The custom background task API @@ -218,8 +218,8 @@ const transcodeVideo = federation.defineTask("transcodeVideo", { Workers for dedicated per-task queues are registered when the queue machinery starts, so define every task before `~Federation.startQueue()` is called (or, without `~FederationOptions.manuallyStartQueue`, before the -first request is handled); a per-task queue defined later never gets -a worker. +first request is handled); a per-task queue defined later may not get +a worker until the queue machinery is next started. The queue for a task is resolved in order: the per-task `queue`, then the federation's `task` queue, then the outbox queue. Deployments that must diff --git a/packages/fedify/src/federation/tasks/codec.test.ts b/packages/fedify/src/federation/tasks/codec.test.ts index b194cda6c..31bc43c33 100644 --- a/packages/fedify/src/federation/tasks/codec.test.ts +++ b/packages/fedify/src/federation/tasks/codec.test.ts @@ -27,7 +27,7 @@ function makeSchema( }; } -test("TaskCodec (fresh instance per operation)", async (t) => { +test("TaskCodec.serialize() / deserialize()", async (t) => { const note = new Note({ id: new URL("https://example.com/notes/1"), content: "Hello, world!", @@ -200,9 +200,9 @@ test("TaskCodec (fresh instance per operation)", async (t) => { }); test("TaskCodec (one instance reused across decodes)", async (t) => { - // The instance carries the cycle-tracking `#seen` map across decodes, but - // each decode parses a fresh object graph with distinct identities, so a - // reused instance still decodes every payload independently. + // Each deserialize() call builds its own per-decode `seen` map, so no + // cycle-tracking state crosses calls and a reused instance decodes every + // payload independently. await t.step("two sequential decodes stay independent", async () => { const codec = new TaskCodec(loaders); const first = await codec.serialize({ From 4303ac42efee4cf4df758176fa330f921912207f Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Sun, 14 Jun 2026 12:07:29 +0000 Subject: [PATCH 25/57] Cover enqueueTask(Many) in middleware tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The custom task API's producer side—Context.enqueueTask() and enqueueTaskMany()—had no direct coverage at the middleware layer. Add tests that drive a real ContextImpl against a recording queue and assert on what it enqueues: - enqueueTask() builds a well-formed task message (type, taskName, baseUrl, attempt, UUID id, parseable started instant, trace context) and round-trips a vocab payload through the codec as JSON-LD. - enqueueTaskMany() routes a multi-item batch through enqueueMany(), preserving order and forwarding delay/orderingKey, while a single-item batch uses enqueue() instead. - When the queue lacks enqueueMany(), the batch falls back to concurrent single enqueues—verified with a rendezvous queue that blocks until both are in flight—still preserving order and options. - An invalid payload anywhere in the batch rejects with a schema TypeError and enqueues nothing. To avoid duplicating fixtures, the MockQueue and Standard Schema test helpers that tasks.test.ts defined inline move to testing/tasks.ts (re-exported by testing/mod.ts); both suites now import the single implementation, and the fixture-usage allowlist covers the new file. https://github.com/fedify-dev/fedify/pull/803 Assisted-by: Claude Code:claude-opus-4-8 --- .../fedify/src/federation/middleware.test.ts | 286 +++++++++++++++++- .../fedify/src/federation/tasks/tasks.test.ts | 90 +----- packages/fedify/src/testing/mod.ts | 9 + packages/fedify/src/testing/tasks.ts | 101 +++++++ scripts/check_fixture_usage.ts | 2 + 5 files changed, 391 insertions(+), 97 deletions(-) create mode 100644 packages/fedify/src/testing/tasks.ts diff --git a/packages/fedify/src/federation/middleware.test.ts b/packages/fedify/src/federation/middleware.test.ts index ad073b5df..40b67f189 100644 --- a/packages/fedify/src/federation/middleware.test.ts +++ b/packages/fedify/src/federation/middleware.test.ts @@ -5,9 +5,23 @@ import { test, } from "@fedify/fixture"; import { RouterError } from "@fedify/uri-template"; -import { configure, type LogRecord, reset } from "@logtape/logtape"; import * as vocab from "@fedify/vocab"; -import { Create, getTypeId, lookupObject, Offer, Person } from "@fedify/vocab"; +import { + Create, + getTypeId, + lookupObject, + Note, + Offer, + Person, +} from "@fedify/vocab"; +import { FetchError, getDocumentLoader } from "@fedify/vocab-runtime"; +import { configure, type LogRecord, reset } from "@logtape/logtape"; +import { metrics, SpanStatusCode } from "@opentelemetry/api"; +import { + DataPointType, + MeterProvider, + MetricReader, +} from "@opentelemetry/sdk-metrics"; import { assert, assertEquals, @@ -21,6 +35,7 @@ import { } from "@std/assert"; import fetchMock from "fetch-mock"; import serialize from "json-canon"; +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; import createFixture from "../../../fixture/src/fixtures/example.com/create.json" with { type: "json", }; @@ -49,20 +64,12 @@ import { rsaPublicKey2, rsaPublicKey3, } from "../testing/keys.ts"; -import { FetchError, getDocumentLoader } from "@fedify/vocab-runtime"; -import { metrics, SpanStatusCode } from "@opentelemetry/api"; -import { - DataPointType, - MeterProvider, - MetricReader, -} from "@opentelemetry/sdk-metrics"; import { getAuthenticatedDocumentLoader } from "../utils/docloader.ts"; -import { CircuitBreaker } from "./circuit-breaker.ts"; import { handleBenchmarkTrigger } from "./bench.ts"; - -const documentLoader = getDocumentLoader(); +import { CircuitBreaker } from "./circuit-breaker.ts"; import type { Context, GetActorOptions } from "./context.ts"; import { MemoryKvStore } from "./kv.ts"; +import { recordInboxActivity } from "./metrics.ts"; import { ContextImpl, createFederation, @@ -70,9 +77,26 @@ import { InboxContextImpl, KvSpecDeterminer, } from "./middleware.ts"; -import { recordInboxActivity } from "./metrics.ts"; -import type { MessageQueue } from "./mq.ts"; -import type { InboxMessage, Message, OutboxMessage } from "./queue.ts"; +import type { + MessageQueue, + MessageQueueEnqueueOptions, + MessageQueueListenOptions, +} from "./mq.ts"; +import type { + InboxMessage, + Message, + OutboxMessage, + TaskMessage, +} from "./queue.ts"; +import TaskCodec from "./tasks/codec.ts"; +import { + type Envelope, + envelopeSchema, + MockQueue, + numberSchema, +} from "../testing/mod.ts"; + +const documentLoader = getDocumentLoader(); type IsEqual = (() => T extends A ? 1 : 2) extends (() => T extends B ? 1 : 2) ? true : false; @@ -10480,3 +10504,235 @@ test("createFederation() omits instrumentation when no meterProvider is set", () assertStrictEquals(ctx.documentLoader, mockDocumentLoader); assertStrictEquals(ctx.contextLoader, mockDocumentLoader); }); + +const taskCodec = new TaskCodec({ contextLoader: mockDocumentLoader }); +const decodeEnvelope = (message: TaskMessage): Promise => + taskCodec.decode(envelopeSchema, message.data); +const envelope = (title: string): Envelope => ({ + note: new Note({ content: title }), + title, +}); + +class RendezvousQueue implements MessageQueue { + readonly enqueued: { + message: TaskMessage; + options?: MessageQueueEnqueueOptions; + }[] = []; + #count = 0; + #markDispatched!: () => void; + #openGate!: () => void; + readonly dispatched: Promise; + readonly #gate: Promise; + + constructor(readonly expected: number) { + this.dispatched = new Promise((resolve) => { + this.#markDispatched = resolve; + }); + this.#gate = new Promise((resolve) => { + this.#openGate = resolve; + }); + } + + release(): void { + this.#openGate(); + } + + // deno-lint-ignore no-explicit-any + enqueue(message: any, options?: MessageQueueEnqueueOptions): Promise { + this.enqueued.push({ message, options }); + if (++this.#count >= this.expected) this.#markDispatched(); + return this.#gate; + } + + listen( + // deno-lint-ignore no-explicit-any + _handler: (message: any) => Promise | void, + _options?: MessageQueueListenOptions, + ): Promise { + return new Promise(() => {}); + } +} + +const withTimeout = ( + promise: Promise, + ms: number, + message: string, +): Promise => { + let timer: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(message)), ms); + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +}; + +const taskFederationOptions = { + kv: new MemoryKvStore(), + documentLoaderFactory: () => mockDocumentLoader, + contextLoaderFactory: () => mockDocumentLoader, + manuallyStartQueue: true, +}; + +test("ContextImpl.enqueueTask()", async (t) => { + await t.step( + "builds the task message envelope and round-trips a vocab payload", + async () => { + const queue = new MockQueue({ supportsEnqueueMany: true }); + const federation = createFederation({ + ...taskFederationOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("greet", { + schema: envelopeSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + ok(ctx instanceof ContextImpl); + await ctx.enqueueTask(task, envelope("greeting")); + strictEqual(queue.enqueuedMany.length, 0); + strictEqual(queue.enqueued.length, 1); + const { message } = queue.enqueued[0]; + strictEqual(message.type, "task"); + strictEqual(message.taskName, "greet"); + strictEqual(message.baseUrl, "https://example.com"); + strictEqual(message.attempt, 0); + ok(/^[0-9a-f-]{36}$/i.test(message.id)); + // `started` must be a parseable instant; Temporal.Instant.from throws + // otherwise. + ok(Temporal.Instant.from(message.started) instanceof Temporal.Instant); + strictEqual(typeof message.traceContext, "object"); + ok(message.traceContext != null); + // A vocab object must survive the producer-side encode as JSON-LD. + const decoded = await decodeEnvelope(message); + ok(decoded.note instanceof Note); + strictEqual(decoded.note.content?.toString(), "greeting"); + strictEqual(decoded.title, "greeting"); + }, + ); +}); + +test("ContextImpl.enqueueTaskMany()", async (t) => { + await t.step( + "round-trips every payload through enqueueMany in order, forwarding options", + async () => { + const queue = new MockQueue({ supportsEnqueueMany: true }); + const federation = createFederation({ + ...taskFederationOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("bulk", { + schema: envelopeSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + const payloads = [envelope("a"), envelope("b"), envelope("c")]; + await ctx.enqueueTaskMany(task, payloads, { + delay: { seconds: 30 }, + orderingKey: "batch", + }); + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 1); + const { messages, options } = queue.enqueuedMany[0]; + ok(options?.delay instanceof Temporal.Duration); + strictEqual(options.delay.total("second"), 30); + strictEqual(options.orderingKey, "batch"); + const decoded = await Promise.all(messages.map(decodeEnvelope)); + deepStrictEqual(decoded.map((d) => d.title), ["a", "b", "c"]); + for (const message of messages) strictEqual(message.orderingKey, "batch"); + }, + ); + + await t.step( + "with a single payload uses enqueue() instead of enqueueMany", + async () => { + const queue = new MockQueue({ supportsEnqueueMany: true }); + const federation = createFederation({ + ...taskFederationOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("bulk-single", { + schema: envelopeSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTaskMany(task, [envelope("solo")]); + strictEqual(queue.enqueuedMany.length, 0); + strictEqual(queue.enqueued.length, 1); + }, + ); + + await t.step( + "falls back to concurrent single enqueues, preserving order and options", + async () => { + const queue = new RendezvousQueue(2); + const federation = createFederation({ + ...taskFederationOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("bulk-fallback", { + schema: envelopeSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + const payloads = [envelope("x"), envelope("y")]; + const pending = ctx.enqueueTaskMany(task, payloads, { + orderingKey: "batch", + }); + try { + await withTimeout( + queue.dispatched, + 2000, + "fallback did not dispatch enqueues concurrently", + ); + strictEqual(queue.enqueued.length, 2); + } finally { + queue.release(); + await pending; + } + const decoded = await Promise.all( + queue.enqueued.map(({ message }) => decodeEnvelope(message)), + ); + deepStrictEqual(decoded.map((d) => d.title), ["x", "y"]); + for (const { message, options } of queue.enqueued) { + strictEqual(message.orderingKey, "batch"); + strictEqual(options?.orderingKey, "batch"); + } + }, + ); + + await t.step( + "fallback path aborts the whole batch when one payload is invalid", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...taskFederationOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("bulk-typed", { + schema: numberSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + // deno-lint-ignore no-explicit-any + () => ctx.enqueueTaskMany(task, [1, "two", 3] as any), + { name: "TypeError", message: /Task data failed schema validation/ }, + ); + strictEqual(queue.enqueued.length, 0); + }, + ); +}); diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index 87477c336..6ac05255a 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -1,6 +1,5 @@ import { mockDocumentLoader, test } from "@fedify/fixture"; import { Note } from "@fedify/vocab"; -import type { StandardSchemaV1 } from "@standard-schema/spec"; import { delay } from "es-toolkit"; import { deepStrictEqual, @@ -14,93 +13,20 @@ import type { Context } from "../context.ts"; import type { Federatable, FederationOptions } from "../federation.ts"; import { MemoryKvStore } from "../kv.ts"; import { createFederation, type FederationImpl } from "../middleware.ts"; -import { - InProcessMessageQueue, - type MessageQueue, - type MessageQueueEnqueueOptions, - type MessageQueueListenOptions, -} from "../mq.ts"; +import { InProcessMessageQueue } from "../mq.ts"; import type { TaskMessage } from "../queue.ts"; import TaskCodec from "./codec.ts"; import type { TaskDefinition, TaskRegistry } from "./task.ts"; +import { + type Envelope, + envelopeSchema, + MockQueue, + numberSchema, + stringSchema, +} from "../../testing/mod.ts"; type Assert = T; -const makeSchema = ( - check: (data: unknown) => data is T, -): StandardSchemaV1 => ({ - "~standard": { - version: 1, - vendor: "fedify-test", - validate: (value: unknown) => - check(value) - ? { value } - : { issues: [{ message: "Invalid task data." }] }, - }, -}); - -interface Envelope { - note: Note; - title: string; -} - -const envelopeSchema = makeSchema( - (data): data is Envelope => - typeof data === "object" && data != null && - (data as Envelope).note instanceof Note && - typeof (data as Envelope).title === "string", -); - -const stringSchema = makeSchema((d): d is string => typeof d === "string"); -const numberSchema = makeSchema((d): d is number => typeof d === "number"); - -class MockQueue implements MessageQueue { - readonly nativeRetrial: boolean; - readonly enqueued: { - message: TaskMessage; - options?: MessageQueueEnqueueOptions; - }[] = []; - readonly enqueuedMany: { - messages: readonly TaskMessage[]; - options?: MessageQueueEnqueueOptions; - }[] = []; - listenCount = 0; - enqueueMany?: ( - messages: readonly TaskMessage[], - options?: MessageQueueEnqueueOptions, - ) => Promise; - - constructor( - options: { nativeRetrial?: boolean; supportsEnqueueMany?: boolean } = {}, - ) { - this.nativeRetrial = options.nativeRetrial ?? false; - if (options.supportsEnqueueMany) { - this.enqueueMany = (messages, opts) => { - this.enqueuedMany.push({ messages, options: opts }); - return Promise.resolve(); - }; - } - } - - enqueue( - message: TaskMessage, - options?: MessageQueueEnqueueOptions, - ): Promise { - this.enqueued.push({ message, options }); - return Promise.resolve(); - } - - listen( - _handler: (message: TaskMessage) => Promise | void, - options?: MessageQueueListenOptions, - ): Promise { - this.listenCount++; - return new Promise((resolve) => { - options?.signal?.addEventListener("abort", () => resolve()); - }); - } -} - const baseOptions: Omit, "queue"> = { kv: new MemoryKvStore(), documentLoaderFactory: () => mockDocumentLoader, diff --git a/packages/fedify/src/testing/mod.ts b/packages/fedify/src/testing/mod.ts index fe72cbdda..27cf5bdc9 100644 --- a/packages/fedify/src/testing/mod.ts +++ b/packages/fedify/src/testing/mod.ts @@ -3,5 +3,14 @@ export { createOutboxContext, createRequestContext, } from "./context.ts"; +export { + type Envelope, + envelopeSchema, + makeSchema, + MockQueue, + type MockQueueOptions, + numberSchema, + stringSchema, +} from "./tasks.ts"; // Without the export below, `test:cfworkers` makes an error. export { testDefinitions } from "@fedify/fixture"; diff --git a/packages/fedify/src/testing/tasks.ts b/packages/fedify/src/testing/tasks.ts new file mode 100644 index 000000000..2dc50c83d --- /dev/null +++ b/packages/fedify/src/testing/tasks.ts @@ -0,0 +1,101 @@ +import { Note } from "@fedify/vocab"; +import type { StandardSchemaV1 } from "@standard-schema/spec"; +import type { + MessageQueue, + MessageQueueEnqueueOptions, + MessageQueueListenOptions, +} from "../federation/mq.ts"; +import type { TaskMessage } from "../federation/queue.ts"; + +/** + * Builds a minimal [Standard Schema](https://standardschema.dev/) from a type + * guard, for use as a task payload schema in tests. + */ +export const makeSchema = ( + check: (data: unknown) => data is T, +): StandardSchemaV1 => ({ + "~standard": { + version: 1, + vendor: "fedify-test", + validate: (value: unknown) => + check(value) + ? { value } + : { issues: [{ message: "Invalid task data." }] }, + }, +}); + +export const stringSchema = makeSchema( + (d): d is string => typeof d === "string", +); +export const numberSchema = makeSchema( + (d): d is number => typeof d === "number", +); + +/** A task payload that carries a vocabulary object, to exercise the codec's + * vocab-to-JSON-LD bridging. */ +export interface Envelope { + note: Note; + title: string; +} + +export const envelopeSchema = makeSchema( + (data): data is Envelope => + typeof data === "object" && data != null && + (data as Envelope).note instanceof Note && + typeof (data as Envelope).title === "string", +); + +/** Options for {@link MockQueue}. */ +export interface MockQueueOptions { + readonly nativeRetrial?: boolean; + readonly supportsEnqueueMany?: boolean; +} + +/** + * A {@link MessageQueue} that records what it was asked to enqueue and resolves + * its `listen()` when the abort signal fires, so tests can inspect dispatch + * without a real backend. + */ +export class MockQueue implements MessageQueue { + readonly nativeRetrial: boolean; + readonly enqueued: { + message: TaskMessage; + options?: MessageQueueEnqueueOptions; + }[] = []; + readonly enqueuedMany: { + messages: readonly TaskMessage[]; + options?: MessageQueueEnqueueOptions; + }[] = []; + listenCount = 0; + enqueueMany?: ( + messages: readonly TaskMessage[], + options?: MessageQueueEnqueueOptions, + ) => Promise; + + constructor(options: MockQueueOptions = {}) { + this.nativeRetrial = options.nativeRetrial ?? false; + if (options.supportsEnqueueMany) { + this.enqueueMany = (messages, opts) => { + this.enqueuedMany.push({ messages, options: opts }); + return Promise.resolve(); + }; + } + } + + // deno-lint-ignore no-explicit-any + enqueue(message: any, options?: MessageQueueEnqueueOptions): Promise { + this.enqueued.push({ message, options }); + return Promise.resolve(); + } + + listen( + // deno-lint-ignore no-explicit-any + _handler: (message: any) => Promise | void, + options?: MessageQueueListenOptions, + ): Promise { + this.listenCount++; + return new Promise((resolve) => { + options?.signal?.addEventListener("abort", () => resolve()); + }); + } +} diff --git a/scripts/check_fixture_usage.ts b/scripts/check_fixture_usage.ts index 7ca5e28ff..73ef518d9 100644 --- a/scripts/check_fixture_usage.ts +++ b/scripts/check_fixture_usage.ts @@ -35,6 +35,8 @@ const ALLOWLIST: readonly string[] = [ // tsdown `noExternal` so consumers never resolve `@fedify/fixture` at // runtime. "packages/fedify/src/testing/mod.ts", + // Test utils for custom tasks + "packages/fedify/src/testing/tasks.ts", // JSDoc `@example` block mentions `import { test } from "@fedify/fixture"` // as documentation; not a real runtime import. "packages/testing/src/mq-tester.ts", From 1d82b198e98fd8f0d500f28a08eca43cf80b87c9 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 18 Jun 2026 18:14:40 +0000 Subject: [PATCH 26/57] Append revived array elements without spread The array reviver in TaskCodec restored elements with `arr.push(...await Array.fromAsync(node, revive))`. Spreading the revived elements into a single call hits the engine's argument-count limit, so a large enough array throws `RangeError: Maximum call stack size exceeded` during decode. Since the worker drops decode failures without retry, an otherwise-valid payload that enqueued fine is silently lost on the dequeue side. Replace the spread with a per-item loop append, matching the existing Map and Set revivers, so revival no longer depends on the array length. Add a regression test that round-trips a 200,000-element array. https://github.com/fedify-dev/fedify/pull/803#discussion_r3418709602 Assisted-by: Claude Code:claude-opus-4-8 --- packages/fedify/src/federation/tasks/codec.test.ts | 13 +++++++++++++ packages/fedify/src/federation/tasks/codec.ts | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/fedify/src/federation/tasks/codec.test.ts b/packages/fedify/src/federation/tasks/codec.test.ts index 31bc43c33..c90374f6b 100644 --- a/packages/fedify/src/federation/tasks/codec.test.ts +++ b/packages/fedify/src/federation/tasks/codec.test.ts @@ -197,6 +197,19 @@ test("TaskCodec.serialize() / deserialize()", async (t) => { strictEqual(decoded.content?.toString(), "deep"); }, ); + + await t.step( + "round-trips an array too large to spread into push()", + async () => { + const length = 200_000; + const payload = { big: Array.from({ length }, (_, i) => i) }; + const encoded = await codec.serialize(payload); + const decoded = await codec.deserialize(encoded) as { big: number[] }; + strictEqual(decoded.big.length, length); + strictEqual(decoded.big[0], 0); + strictEqual(decoded.big[length - 1], length - 1); + }, + ); }); test("TaskCodec (one instance reused across decodes)", async (t) => { diff --git a/packages/fedify/src/federation/tasks/codec.ts b/packages/fedify/src/federation/tasks/codec.ts index b93366c0b..f13d85581 100644 --- a/packages/fedify/src/federation/tasks/codec.ts +++ b/packages/fedify/src/federation/tasks/codec.ts @@ -66,7 +66,7 @@ export default class TaskCodec { isInstanceOf(Array), (): unknown[] => [], async (revive, node, arr) => { - arr.push(...await Array.fromAsync(node, revive)); + for (const item of await Array.fromAsync(node, revive)) arr.push(item); }, ), classReviver( From 369e01f27a9f92796ea16d38959b3a54c1c8a3ad Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 18 Jun 2026 18:24:28 +0000 Subject: [PATCH 27/57] Polish task payload docs and cover Temporal Three documentation points raised on the task API review, plus a regression test backing the Temporal claim: - The payload codec round-trips devalue's built-in Temporal types with no extra code, but the supported-payload list omitted them. List `Temporal` (with `Temporal.Instant` / `Temporal.Duration` examples) and add a serialize/deserialize round-trip test so the documented support stays covered. - The vocab import example used the compatibility path `@fedify/fedify/vocab`. Switch it to `@fedify/vocab`, matching the surrounding docs and the current package boundary so copied code does not bind to a path slated for removal. - Task payloads now cross durable queue storage and can hold arbitrary application data. Add a trust-boundary security note to the queue isolation section: treat the backend and payloads as internal trusted storage, pass identifiers the worker resolves rather than long-lived secrets, and use a dedicated task queue with `taskQueueResolution: "strict"` when isolation is required. https://github.com/fedify-dev/fedify/pull/803#discussion_r3418856889 https://github.com/fedify-dev/fedify/pull/803#discussion_r3418709609 https://github.com/fedify-dev/fedify/pull/803#discussion_r3418793171 Assisted-by: Claude Code:claude-opus-4-8 --- docs/manual/tasks.md | 15 ++++++++++++--- .../fedify/src/federation/tasks/codec.test.ts | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index ec5ca09bf..55adef8dc 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -57,8 +57,9 @@ Task payloads cross a message queue, so they are serialized on enqueue and deserialized on dispatch. Fedify owns this codec—applications never encode payloads themselves. The codec is built on [devalue], which means payloads are not limited to JSON: `Date`, `Map`, `Set`, `URL`, `RegExp`, `bigint`, -typed arrays, circular references, and repeated references all round-trip -faithfully. +typed arrays, `Temporal` values (e.g., `Temporal.Instant`, +`Temporal.Duration`), circular references, and repeated references all +round-trip faithfully. Activity Vocabulary objects (`Note`, `Create`, `Person`, `Link`, and so on) are also supported as payload values. Each vocabulary object is bridged @@ -66,7 +67,7 @@ through expanded JSON-LD on the wire and comes back as a real instance, so the handler can call its methods and getters as usual: ~~~~ typescript -import { Note } from "@fedify/fedify/vocab"; +import { Note } from "@fedify/vocab"; import { z } from "zod"; const indexNote = federation.defineTask("indexNote", { @@ -247,6 +248,14 @@ A task that falls back to the outbox queue needs no dedicated worker; the outbox worker dispatches every message by its type regardless of which queue delivered it. +> [!CAUTION] +> Task payloads cross durable queue storage, so treat the queue backend and +> its payloads as internal, trusted storage. Do not place long-lived secrets +> or credentials directly in a task payload; pass an identifier that the +> worker resolves from your application storage instead. When task workloads +> must stay isolated from ActivityPub delivery, give them a dedicated task +> queue and set `taskQueueResolution: "strict"`. + Limitations ----------- diff --git a/packages/fedify/src/federation/tasks/codec.test.ts b/packages/fedify/src/federation/tasks/codec.test.ts index c90374f6b..a3f2259d4 100644 --- a/packages/fedify/src/federation/tasks/codec.test.ts +++ b/packages/fedify/src/federation/tasks/codec.test.ts @@ -210,6 +210,22 @@ test("TaskCodec.serialize() / deserialize()", async (t) => { strictEqual(decoded.big[length - 1], length - 1); }, ); + + await t.step("round-trips Temporal values", async () => { + const payload = { + instant: Temporal.Instant.from("2026-01-02T03:04:05Z"), + duration: Temporal.Duration.from({ hours: 1, minutes: 30 }), + }; + const encoded = await codec.serialize(payload); + const decoded = await codec.deserialize(encoded) as { + instant: Temporal.Instant; + duration: Temporal.Duration; + }; + ok(decoded.instant instanceof Temporal.Instant); + ok(decoded.instant.equals(payload.instant)); + ok(decoded.duration instanceof Temporal.Duration); + strictEqual(decoded.duration.toString(), payload.duration.toString()); + }); }); test("TaskCodec (one instance reused across decodes)", async (t) => { From de7cad5a28c3fb56443d36edb20f22842854683c Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 18 Jun 2026 19:23:07 +0000 Subject: [PATCH 28/57] Ignore `Temporal` test in Bun --- .../fedify/src/federation/tasks/codec.test.ts | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/fedify/src/federation/tasks/codec.test.ts b/packages/fedify/src/federation/tasks/codec.test.ts index a3f2259d4..fcc84de0e 100644 --- a/packages/fedify/src/federation/tasks/codec.test.ts +++ b/packages/fedify/src/federation/tasks/codec.test.ts @@ -211,20 +211,24 @@ test("TaskCodec.serialize() / deserialize()", async (t) => { }, ); - await t.step("round-trips Temporal values", async () => { - const payload = { - instant: Temporal.Instant.from("2026-01-02T03:04:05Z"), - duration: Temporal.Duration.from({ hours: 1, minutes: 30 }), - }; - const encoded = await codec.serialize(payload); - const decoded = await codec.deserialize(encoded) as { - instant: Temporal.Instant; - duration: Temporal.Duration; - }; - ok(decoded.instant instanceof Temporal.Instant); - ok(decoded.instant.equals(payload.instant)); - ok(decoded.duration instanceof Temporal.Duration); - strictEqual(decoded.duration.toString(), payload.duration.toString()); + await t.step({ + name: "round-trips Temporal values", + ignore: "Bun" in globalThis, + async fn() { + const payload = { + instant: Temporal.Instant.from("2026-01-02T03:04:05Z"), + duration: Temporal.Duration.from({ hours: 1, minutes: 30 }), + }; + const encoded = await codec.serialize(payload); + const decoded = await codec.deserialize(encoded) as { + instant: Temporal.Instant; + duration: Temporal.Duration; + }; + ok(decoded.instant instanceof Temporal.Instant); + ok(decoded.instant.equals(payload.instant)); + ok(decoded.duration instanceof Temporal.Duration); + strictEqual(decoded.duration.toString(), payload.duration.toString()); + }, }); }); From d8727b7aa8761c27f58d01db9a7fc39707cdb161 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Fri, 26 Jun 2026 18:56:32 +0000 Subject: [PATCH 29/57] Move custom background task changelog to v2.4.0 https://github.com/fedify-dev/fedify/pull/806#discussion_r3483304594 --- CHANGES.md | 69 +++++++++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d858acb37..0f6d419f6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,41 @@ Fedify changelog Version 2.4.0 ------------- -To be released. +### @fedify/fedify + + - Added a custom background task API that generalizes Fedify's + enqueue-and-process-later pattern to arbitrary application-defined jobs: + + - `Federation` and `FederationBuilder` gained a `defineTask()` method + through the new `TaskRegistry` interface, which `Federatable` now + extends. + - `Context` gained `enqueueTask()` and `enqueueTaskMany()` methods, + with `delay` and `orderingKey` options + (new `TaskEnqueueOptions` interface). + - Every task requires a [Standard Schema] + (`schema` option) from which the payload type is inferred; payloads + are validated at enqueue time (fail fast) and again at dequeue time + (protection against schema drift across deployments). + - Payloads are serialized by Fedify with devalue, so `Date`, `Map`, + `Set`, `URL`, `bigint`, circular references, and Activity Vocabulary + objects round-trip faithfully across every message queue backend. + - Failed handlers are retried with exponential backoff by default; + tasks support per-task `retryPolicy` and `onError` options, the new + `FederationOptions.taskRetryPolicy` sets the federation-wide default, + and queues with `nativeRetrial` delegate retries to the backend. + - Tasks can be isolated from activity delivery through the new + `FederationQueueOptions.task` slot or a per-task `queue` option; + without them, tasks fall back to the outbox queue unless the new + `FederationOptions.taskQueueResolution` option is set to `"strict"`. + `Federation.startQueue()` now accepts `queue: "task"` to run + a task-only worker. + + [[#206], [#797], [#803] by ChanHaeng Lee] + +[Standard Schema]: https://standardschema.dev/ +[#206]: https://github.com/fedify-dev/fedify/issues/206 +[#797]: https://github.com/fedify-dev/fedify/issues/797 +[#803]: https://github.com/fedify-dev/fedify/pull/803 ### @fedify/fedify @@ -343,37 +377,6 @@ Released on June 25, 2026. the dispatcher overload hot path; other contributors to `check-all` cost may remain. [[#613], [#800] by ChanHaeng Lee] - - Added a custom background task API that generalizes Fedify's - enqueue-and-process-later pattern to arbitrary application-defined jobs: - - - `Federation` and `FederationBuilder` gained a `defineTask()` method - through the new `TaskRegistry` interface, which `Federatable` now - extends. - - `Context` gained `enqueueTask()` and `enqueueTaskMany()` methods, - with `delay` and `orderingKey` options - (new `TaskEnqueueOptions` interface). - - Every task requires a [Standard Schema] - (`schema` option) from which the payload type is inferred; payloads - are validated at enqueue time (fail fast) and again at dequeue time - (protection against schema drift across deployments). - - Payloads are serialized by Fedify with devalue, so `Date`, `Map`, - `Set`, `URL`, `bigint`, circular references, and Activity Vocabulary - objects round-trip faithfully across every message queue backend. - - Failed handlers are retried with exponential backoff by default; - tasks support per-task `retryPolicy` and `onError` options, the new - `FederationOptions.taskRetryPolicy` sets the federation-wide default, - and queues with `nativeRetrial` delegate retries to the backend. - - Tasks can be isolated from activity delivery through the new - `FederationQueueOptions.task` slot or a per-task `queue` option; - without them, tasks fall back to the outbox queue unless the new - `FederationOptions.taskQueueResolution` option is set to `"strict"`. - `Federation.startQueue()` now accepts `queue: "task"` to run - a task-only worker. - - [[#206], [#797], [#803] by ChanHaeng Lee] - -[Standard Schema]: https://standardschema.dev/ -[#206]: https://github.com/fedify-dev/fedify/issues/206 [#316]: https://github.com/fedify-dev/fedify/issues/316 [#418]: https://github.com/fedify-dev/fedify/issues/418 [#613]: https://github.com/fedify-dev/fedify/issues/613 @@ -403,9 +406,7 @@ Released on June 25, 2026. [#778]: https://github.com/fedify-dev/fedify/pull/778 [#782]: https://github.com/fedify-dev/fedify/issues/782 [#787]: https://github.com/fedify-dev/fedify/pull/787 -[#797]: https://github.com/fedify-dev/fedify/issues/797 [#800]: https://github.com/fedify-dev/fedify/pull/800 -[#803]: https://github.com/fedify-dev/fedify/pull/803 ### @fedify/cli From c12bba6150029a9b8c74b323c6272df4da8a5805 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Sun, 14 Jun 2026 07:07:52 +0000 Subject: [PATCH 30/57] Support deduplicationKey for task enqueue Context.enqueueTask() and enqueueTaskMany() now accept a deduplicationKey requesting at-most-once enqueue for tasks that share it (new TaskEnqueueOptions.deduplicationKey). Resolution follows the queue and key-value store capabilities: - A queue declaring the new MessageQueue.nativeDeduplication owns the check; the key is forwarded through the new MessageQueueEnqueueOptions.deduplicationKey. - Otherwise Fedify applies a best-effort guard through the optional KvStore.cas primitive under a new taskDeduplication key prefix, tunable with the new FederationOptions.taskDeduplicationTtl and taskDeduplicationFallback options. For enqueueTaskMany(), a single key governs the whole batch. A native queue that does not implement enqueueMany() cannot express batch-level at-most-once with a per-message key, so such a multi-item enqueue is rejected with a TypeError instead of silently leaking duplicates. Configuration errors that are decidable without a payload (a native queue lacking enqueueMany, or a closed fallback without cas) are checked before payloads are validated and encoded, so they reject before any user schema runs or any key is reserved. https://github.com/fedify-dev/fedify/issues/798 Assisted-by: Claude Code:claude-opus-4-8 --- CHANGES.md | 16 +- docs/manual/tasks.md | 84 ++- packages/fedify/src/federation/federation.ts | 26 + packages/fedify/src/federation/middleware.ts | 89 ++- packages/fedify/src/federation/mq.test.ts | 23 + packages/fedify/src/federation/mq.ts | 39 ++ packages/fedify/src/federation/tasks/task.ts | 16 + .../fedify/src/federation/tasks/tasks.test.ts | 598 +++++++++++++++++- 8 files changed, 874 insertions(+), 17 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0f6d419f6..85cdf1598 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,13 +34,25 @@ Version 2.4.0 `FederationOptions.taskQueueResolution` option is set to `"strict"`. `Federation.startQueue()` now accepts `queue: "task"` to run a task-only worker. - - [[#206], [#797], [#803] by ChanHaeng Lee] + - Tasks can request at-most-once enqueue with a `deduplicationKey` + (new `TaskEnqueueOptions.deduplicationKey`). A queue declaring the new + `MessageQueue.nativeDeduplication` capability owns the check and + receives the key through the new + `MessageQueueEnqueueOptions.deduplicationKey`; otherwise Fedify + performs a best-effort key–value guard through the optional + `KvStore.cas` primitive, under a new `taskDeduplication` key prefix. + The marker TTL and the no-`cas` fallback are tunable with the new + `FederationOptions.taskDeduplicationTtl` and + `FederationOptions.taskDeduplicationFallback` options. + + [[#206], [#797], [#798], [#803], [#806] by ChanHaeng Lee] [Standard Schema]: https://standardschema.dev/ [#206]: https://github.com/fedify-dev/fedify/issues/206 [#797]: https://github.com/fedify-dev/fedify/issues/797 +[#798]: https://github.com/fedify-dev/fedify/issues/798 [#803]: https://github.com/fedify-dev/fedify/pull/803 +[#806]: https://github.com/fedify-dev/fedify/pull/806 ### @fedify/fedify diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index 55adef8dc..710486d90 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -139,6 +139,10 @@ Both methods accept options: : Tasks with the same ordering key are processed sequentially (one at a time), like the same option on the message queue layer. +`deduplicationKey` +: Requests at-most-once enqueue for tasks that share the key; see + [Deduplication](#deduplication) below. + ~~~~ typescript await ctx.enqueueTask(sendDigest, payload, { delay: { minutes: 30 }, @@ -257,12 +261,84 @@ delivered it. > queue and set `taskQueueResolution: "strict"`. +Deduplication +------------- + +A task often needs *at-most-once-per-key* enqueue: a digest mailer must not +send twice when a request is retried, and a cleanup job should coalesce +duplicate triggers. Passing a `deduplicationKey` requests this—a second +enqueue with the same key is dropped while the first is still within the +deduplication window: + +~~~~ typescript +await ctx.enqueueTask(sendDigest, payload, { + deduplicationKey: `digest:${payload.userId}`, // [!code highlight] +}); +~~~~ + +How the key is resolved depends on the queue and the key–value store: + +1. **Native backend.** When the task's queue declares + `~MessageQueue.nativeDeduplication`, Fedify forwards the key in the + message queue's `~MessageQueueEnqueueOptions.deduplicationKey` and the + backend owns the check. Fedify does not touch the key–value store. + +2. **Key–value fallback.** Otherwise, if the configured `~KvStore` exposes + the optional compare-and-swap (`~KvStore.cas`) primitive, Fedify records + the key under a dedicated `taskDeduplication` prefix with a TTL and skips + the enqueue while a marker is present. The TTL defaults to one hour and is + configurable with `~FederationOptions.taskDeduplicationTtl`: + + ~~~~ typescript + const federation = createFederation({ + // ... + taskDeduplicationTtl: { minutes: 10 }, // [!code highlight] + }); + ~~~~ + +3. **No conditional write.** When neither applies—no native deduplication and + a key–value store without `~KvStore.cas`—the behavior is governed by + `~FederationOptions.taskDeduplicationFallback`. `"open"` (the default) + lets the enqueue proceed without deduplication after a debug-level log; + `"closed"` throws a `TypeError` before enqueuing: + + ~~~~ typescript + const federation = createFederation({ + // ... + taskDeduplicationFallback: "closed", // [!code highlight] + }); + ~~~~ + +Among the first-party adapters, the in-memory, Deno KV, SQLite, and MySQL +key–value stores implement `~KvStore.cas`; PostgreSQL and Redis do not yet, so +those deployments take the `taskDeduplicationFallback` branch until per-adapter +follow-ups add it. + +For `~Context.enqueueTaskMany()`, a single `deduplicationKey` applies to the +whole batch: the batch enqueues as a unit or is skipped as a unit, never +partially. Per-item deduplication means calling `~Context.enqueueTask()` in +a loop, each with its own key. A queue that declares +`~MessageQueue.nativeDeduplication` must also implement +`~MessageQueue.enqueueMany()` to carry a multi-item batch's key as one unit; +fanning the key out across separate `~MessageQueue.enqueue()` calls cannot drop +a whole batch, so Fedify rejects that combination instead of silently leaking +duplicates. + +> [!WARNING] +> The key–value fallback is **best-effort, not transactional**. The marker +> write and the enqueue are separate operations, so a crash between them, the +> `"open"` fallback under concurrency, a non-atomic third-party `~KvStore.cas`, +> or reuse of a key within its TTL window can still admit a duplicate or +> suppress a task. Cleanup is by TTL expiry, not active deletion on handler +> success. Deployments needing strict guarantees use a queue with +> `nativeDeduplication: true`, where the backend owns an atomic check. + + Limitations ----------- -The current API intentionally ships without deduplication, task-specific -OpenTelemetry spans and metrics, cron-style periodic scheduling, result -backends, and per-task priority. Some of these are planned as follow-ups; -see the [tracking issue]. +The current API intentionally ships without task-specific OpenTelemetry spans +and metrics, cron-style periodic scheduling, result backends, and per-task +priority. Some of these are planned as follow-ups; see the [tracking issue]. [tracking issue]: https://github.com/fedify-dev/fedify/issues/206 diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index baa00edf2..5b4177f80 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -1103,6 +1103,32 @@ export interface FederationOptions { */ taskQueueResolution?: "fallback" | "strict"; + /** + * The time-to-live for a {@link TaskEnqueueOptions.deduplicationKey} marker + * stored in the key–value deduplication fallback. A second enqueue with the + * same key within this window is skipped; once it expires, the key may + * enqueue again. Ignored when the task's queue declares + * {@link MessageQueue.nativeDeduplication} (the backend owns the window). + * @default `{ hours: 1 }` + * @since 2.x.x + */ + taskDeduplicationTtl?: Temporal.DurationLike; + + /** + * The behavior when a {@link TaskEnqueueOptions.deduplicationKey} is supplied + * but the task's queue does not declare + * {@link MessageQueue.nativeDeduplication} *and* the configured + * {@link KvStore} exposes no `cas` (compare-and-swap) primitive: + * + * - `"open"` (the default): proceeds without deduplication after logging at + * debug level. + * - `"closed"`: rejects with a `TypeError` before enqueuing. + * + * @default `"open"` + * @since 2.x.x + */ + taskDeduplicationFallback?: "open" | "closed"; + /** * Activity transformers that are applied to outgoing activities. It is * useful for adjusting outgoing activities to satisfy some ActivityPub diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index e4b956dd5..800b864c5 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -547,6 +547,15 @@ export interface FederationKvPrefixes { * @since 2.3.0 */ readonly circuitBreaker: KvKey; + + /** + * The key prefix used for storing custom background task deduplication + * markers. Kept separate from {@link activityIdempotence} so the two key + * spaces never collide. + * @default `["_fedify", "taskDeduplication"]` + * @since 2.x.x + */ + readonly taskDeduplication: KvKey; } /** @@ -612,6 +621,8 @@ export class FederationImpl inboxRetryPolicy: RetryPolicy; taskRetryPolicy: RetryPolicy; taskQueueResolution: "fallback" | "strict"; + taskDeduplicationTtl: Temporal.Duration; + taskDeduplicationFallback: "open" | "closed"; circuitBreaker?: CircuitBreaker; activityTransformers: readonly ActivityTransformer[]; _tracerProvider: TracerProvider | undefined; @@ -673,6 +684,7 @@ export class FederationImpl httpMessageSignaturesSpec: ["_fedify", "httpMessageSignaturesSpec"], acceptSignatureNonce: ["_fedify", "acceptSignatureNonce"], circuitBreaker: ["_fedify", "circuit"], + taskDeduplication: ["_fedify", "taskDeduplication"], } satisfies FederationKvPrefixes), ...(options.kvPrefixes ?? {}), }; @@ -906,6 +918,11 @@ export class FederationImpl this.taskRetryPolicy = options.taskRetryPolicy ?? createExponentialBackoffPolicy(); this.taskQueueResolution = options.taskQueueResolution ?? "fallback"; + this.taskDeduplicationTtl = Temporal.Duration.from( + options.taskDeduplicationTtl ?? { hours: 1 }, + ); + this.taskDeduplicationFallback = options.taskDeduplicationFallback ?? + "open"; this.activityTransformers = options.activityTransformers ?? getDefaultActivityTransformers(); this._tracerProvider = options.tracerProvider; @@ -3734,6 +3751,46 @@ export class ContextImpl implements Context { const delay = options.delay == null ? undefined : Temporal.Duration.from(options.delay); + type DedupPlan = + | { kind: "none" } + | { kind: "native"; key: string } + | { kind: "cas"; key: string; cas: NonNullable } + | { kind: "open"; key: string }; + let dedupPlan: DedupPlan = { kind: "none" }; + if (options.deduplicationKey != null) { + const key = options.deduplicationKey; + if (queue.nativeDeduplication === true) { + if (items.length > 1 && queue.enqueueMany == null) { + throw new TypeError( + `Task ${ + JSON.stringify(task.name) + } was enqueued as a batch with a deduplicationKey, but its ` + + "message queue declares nativeDeduplication without " + + "implementing enqueueMany; a per-message key cannot deduplicate " + + "a whole batch. Implement enqueueMany on the queue, or enqueue " + + "the tasks individually with enqueueTask().", + ); + } + dedupPlan = { kind: "native", key }; + } else if (this.federation.kv.cas != null) { + dedupPlan = { + kind: "cas", + key, + cas: this.federation.kv.cas.bind(this.federation.kv), + }; + } else if (this.federation.taskDeduplicationFallback === "closed") { + // No conditional write, closed: fail fast before any side effect. + throw new TypeError( + "deduplicationKey was set but the message queue does not declare " + + "nativeDeduplication and the key-value store exposes no " + + 'conditional write (cas); set taskDeduplicationFallback to "open" ' + + "to proceed without deduplication, or use a backend that " + + "supports it.", + ); + } else { + dedupPlan = { kind: "open", key }; + } + } // Encode in parallel: `enqueueTaskMany` is the bulk path, and the enqueue // below already parallelizes, so serial encoding would be the bottleneck. // `map` preserves order, and a rejected encode (validation failure) rejects @@ -3741,16 +3798,44 @@ export class ContextImpl implements Context { const messages: TaskMessage[] = await Promise.all( items.map(this.#encodeTaskMessage(task, options)), ); + let forwardedDeduplicationKey: string | undefined; + if (dedupPlan.kind === "native") { + forwardedDeduplicationKey = dedupPlan.key; + } else if (dedupPlan.kind === "cas") { + const cacheKey = [ + ...this.federation.kvPrefixes.taskDeduplication, + dedupPlan.key, + ] satisfies KvKey; + const won = await dedupPlan.cas(cacheKey, undefined, true, { + ttl: this.federation.taskDeduplicationTtl, + }); + if (!won) return; + } else if (dedupPlan.kind === "open") { + getLogger(["fedify", "federation", "task"]).debug( + "deduplicationKey {deduplicationKey} for task {taskName} ignored: " + + "the message queue declares no nativeDeduplication and the " + + "key-value store has no cas; proceeding (taskDeduplicationFallback " + + 'is "open").', + { deduplicationKey: dedupPlan.key, taskName: task.name }, + ); + } if (!this.federation.manuallyStartQueue) { this.federation._startQueueInternal(this.data); } - const enqueueOptions = { delay, orderingKey: options.orderingKey }; + const enqueueOptions = { + delay, + orderingKey: options.orderingKey, + deduplicationKey: forwardedDeduplicationKey, + }; if (messages.length === 1) { await queue.enqueue(messages[0], enqueueOptions); } else if (queue.enqueueMany != null) { await queue.enqueueMany(messages, enqueueOptions); } else { - await Promise.all(messages.map((m) => queue.enqueue(m, enqueueOptions))); + const fanoutOptions = { delay, orderingKey: options.orderingKey }; + await Promise.all( + messages.map((m) => queue.enqueue(m, fanoutOptions)), + ); } } diff --git a/packages/fedify/src/federation/mq.test.ts b/packages/fedify/src/federation/mq.test.ts index e7c402908..b42135fff 100644 --- a/packages/fedify/src/federation/mq.test.ts +++ b/packages/fedify/src/federation/mq.test.ts @@ -34,6 +34,10 @@ test("InProcessMessageQueue", async (t) => { assertFalse(mq.nativeRetrial); }); + await t.step("nativeDeduplication property", () => { + assertFalse(mq.nativeDeduplication); + }); + await t.step("getDepth() [empty]", async () => { assertEquals(await mq.getDepth(), { queued: 0, @@ -419,6 +423,21 @@ test("MessageQueue.nativeRetrial", async (t) => { }); }); +test("ParallelMessageQueue inherits nativeDeduplication", () => { + class NativeDeduplicationQueue implements MessageQueue { + readonly nativeDeduplication = true; + enqueue(): Promise { + return Promise.resolve(); + } + listen(): Promise { + return Promise.resolve(); + } + } + + const workers = new ParallelMessageQueue(new NativeDeduplicationQueue(), 5); + assert(workers.nativeDeduplication); +}); + const queues: Record Promise> = { InProcessMessageQueue: () => Promise.resolve(new InProcessMessageQueue()), }; @@ -450,6 +469,10 @@ for (const mqName in queues) { assertEquals(workers.nativeRetrial, mq.nativeRetrial); }); + await t.step("nativeDeduplication property inheritance", () => { + assertEquals(workers.nativeDeduplication, mq.nativeDeduplication); + }); + await t.step("getDepth() delegation", async () => { if (mq.getDepth == null) { assertEquals(workers.getDepth, undefined); diff --git a/packages/fedify/src/federation/mq.ts b/packages/fedify/src/federation/mq.ts index 36ba9900c..4ee43320e 100644 --- a/packages/fedify/src/federation/mq.ts +++ b/packages/fedify/src/federation/mq.ts @@ -25,6 +25,20 @@ export interface MessageQueueEnqueueOptions { * @since 2.0.0 */ readonly orderingKey?: string; + + /** + * An optional key requesting at-most-once enqueue semantics for messages + * that share it. A backend that declares + * {@link MessageQueue.nativeDeduplication} `true` owns the check: a message + * whose `deduplicationKey` was already seen within the backend's + * deduplication window is dropped instead of enqueued. Backends without + * native deduplication ignore this field; Fedify performs its own + * best-effort deduplication before reaching them on the paths that support + * it. + * + * @since 2.x.x + */ + readonly deduplicationKey?: string; } /** @@ -87,6 +101,18 @@ export interface MessageQueue { */ readonly nativeRetrial?: boolean; + /** + * Whether the message queue backend deduplicates messages that share a + * {@link MessageQueueEnqueueOptions.deduplicationKey} natively. When `true`, + * Fedify forwards the `deduplicationKey` and relies on the backend to drop + * duplicates; when `false` or omitted, Fedify applies its own best-effort + * key–value deduplication on the paths that request it. + * + * @default `false` + * @since 2.x.x + */ + readonly nativeDeduplication?: boolean; + /** * Enqueues a message in the queue. * @param message The message to enqueue. @@ -176,6 +202,12 @@ export class InProcessMessageQueue implements MessageQueue { */ readonly nativeRetrial = false; + /** + * In-process message queue does not deduplicate messages natively. + * @since 2.x.x + */ + readonly nativeDeduplication = false; + /** * Constructs a new {@link InProcessMessageQueue} with the given options. * @param options Additional options for the in-process message queue. @@ -365,6 +397,12 @@ export class ParallelMessageQueue implements MessageQueue { * @since 1.7.0 */ readonly nativeRetrial?: boolean; + + /** + * Inherits the native deduplication capability from the wrapped queue. + * @since 2.x.x + */ + readonly nativeDeduplication?: boolean; readonly getDepth?: () => Promise; /** @@ -398,6 +436,7 @@ export class ParallelMessageQueue implements MessageQueue { this.queue = queue; this.workers = workers; this.nativeRetrial = queue.nativeRetrial; + this.nativeDeduplication = queue.nativeDeduplication; if (queue.getDepth != null) { this.getDepth = () => queue.getDepth!(); } diff --git a/packages/fedify/src/federation/tasks/task.ts b/packages/fedify/src/federation/tasks/task.ts index 0105e8e21..3731f82ca 100644 --- a/packages/fedify/src/federation/tasks/task.ts +++ b/packages/fedify/src/federation/tasks/task.ts @@ -159,6 +159,22 @@ export interface TaskEnqueueOptions { * processed sequentially (one at a time). */ readonly orderingKey?: string; + + /** + * An optional key requesting at-most-once enqueue for tasks that share it. + * + * A queue with {@link MessageQueue.nativeDeduplication} `true` enforces it + * strictly; otherwise deduplication is best-effort via {@link KvStore.cas}, + * and {@link FederationOptions.taskDeduplicationFallback} decides whether a + * missing `cas` proceeds without deduplication or throws. + * + * For {@link Context.enqueueTaskMany}, one key governs the whole batch; a + * native queue must implement {@link MessageQueue.enqueueMany} for a + * multi-item batch, or the call throws a `TypeError`. + * + * @since 2.x.x + */ + readonly deduplicationKey?: string; } /** diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index 6ac05255a..dbfbf4972 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -8,15 +8,6 @@ import { strictEqual, throws, } from "node:assert/strict"; -import { createFederationBuilder } from "../builder.ts"; -import type { Context } from "../context.ts"; -import type { Federatable, FederationOptions } from "../federation.ts"; -import { MemoryKvStore } from "../kv.ts"; -import { createFederation, type FederationImpl } from "../middleware.ts"; -import { InProcessMessageQueue } from "../mq.ts"; -import type { TaskMessage } from "../queue.ts"; -import TaskCodec from "./codec.ts"; -import type { TaskDefinition, TaskRegistry } from "./task.ts"; import { type Envelope, envelopeSchema, @@ -24,9 +15,110 @@ import { numberSchema, stringSchema, } from "../../testing/mod.ts"; +import { createFederationBuilder } from "../builder.ts"; +import type { Context } from "../context.ts"; +import type { Federatable, FederationOptions } from "../federation.ts"; +import { + type KvKey, + type KvStore, + type KvStoreListEntry, + type KvStoreSetOptions, + MemoryKvStore, +} from "../kv.ts"; +import { createFederation, type FederationImpl } from "../middleware.ts"; +import { + InProcessMessageQueue, + type MessageQueue, + type MessageQueueEnqueueOptions, + type MessageQueueListenOptions, +} from "../mq.ts"; +import type { TaskMessage } from "../queue.ts"; +import TaskCodec from "./codec.ts"; +import type { TaskDefinition, TaskRegistry } from "./task.ts"; type Assert = T; +const makeSchema = ( + check: (data: unknown) => data is T, +): StandardSchemaV1 => ({ + "~standard": { + version: 1, + vendor: "fedify-test", + validate: (value: unknown) => + check(value) + ? { value } + : { issues: [{ message: "Invalid task data." }] }, + }, +}); + +interface Envelope { + note: Note; + title: string; +} + +const envelopeSchema = makeSchema( + (data): data is Envelope => + typeof data === "object" && data != null && + (data as Envelope).note instanceof Note && + typeof (data as Envelope).title === "string", +); + +const stringSchema = makeSchema((d): d is string => typeof d === "string"); +const numberSchema = makeSchema((d): d is number => typeof d === "number"); + +class MockQueue implements MessageQueue { + readonly nativeRetrial: boolean; + readonly nativeDeduplication: boolean; + readonly enqueued: { + message: TaskMessage; + options?: MessageQueueEnqueueOptions; + }[] = []; + readonly enqueuedMany: { + messages: readonly TaskMessage[]; + options?: MessageQueueEnqueueOptions; + }[] = []; + listenCount = 0; + enqueueMany?: ( + messages: readonly TaskMessage[], + options?: MessageQueueEnqueueOptions, + ) => Promise; + + constructor( + options: { + nativeRetrial?: boolean; + nativeDeduplication?: boolean; + supportsEnqueueMany?: boolean; + } = {}, + ) { + this.nativeRetrial = options.nativeRetrial ?? false; + this.nativeDeduplication = options.nativeDeduplication ?? false; + if (options.supportsEnqueueMany) { + this.enqueueMany = (messages, opts) => { + this.enqueuedMany.push({ messages, options: opts }); + return Promise.resolve(); + }; + } + } + + enqueue( + message: TaskMessage, + options?: MessageQueueEnqueueOptions, + ): Promise { + this.enqueued.push({ message, options }); + return Promise.resolve(); + } + + listen( + _handler: (message: TaskMessage) => Promise | void, + options?: MessageQueueListenOptions, + ): Promise { + this.listenCount++; + return new Promise((resolve) => { + options?.signal?.addEventListener("abort", () => resolve()); + }); + } +} + const baseOptions: Omit, "queue"> = { kv: new MemoryKvStore(), documentLoaderFactory: () => mockDocumentLoader, @@ -849,3 +941,491 @@ test("processQueuedTask() task dispatch", async (t) => { strictEqual(queue.enqueued.length, 0); }); }); + +/** + * A {@link KvStore} that delegates to an in-memory store but deliberately + * omits `cas`, so that `kv.cas == null`. This drives the deduplication + * fallback branches that fire when no conditional-write primitive exists. + */ +class CaslessKvStore implements KvStore { + readonly inner = new MemoryKvStore(); + get(key: KvKey): Promise { + return this.inner.get(key); + } + set(key: KvKey, value: unknown, options?: KvStoreSetOptions): Promise { + return this.inner.set(key, value, options); + } + delete(key: KvKey): Promise { + return this.inner.delete(key); + } + list(prefix?: KvKey): AsyncIterable { + return this.inner.list(prefix); + } + // No `cas`: the fallback branch is reached precisely when `kv.cas == null`. +} + +async function collectKeys(kv: KvStore, prefix: KvKey): Promise { + const keys: KvKey[] = []; + for await (const { key } of kv.list(prefix)) keys.push(key); + return keys; +} + +const TASK_DEDUP_PREFIX: KvKey = ["_fedify", "taskDeduplication"]; +const ACTIVITY_IDEMPOTENCE_PREFIX: KvKey = ["_fedify", "activityIdempotence"]; + +test("task deduplication", async (t) => { + await t.step( + "forwards the key to a nativeDeduplication queue without writing KV", + async () => { + const queue = new MockQueue({ nativeDeduplication: true }); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("native-dedup", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + strictEqual(queue.enqueued[0].options?.deduplicationKey, "k"); + // The backend owns the check, so Fedify must not write any KV marker. + strictEqual((await collectKeys(kv, TASK_DEDUP_PREFIX)).length, 0); + }, + ); + + await t.step( + "skips a second enqueue with the same key within the TTL", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new MemoryKvStore(), + queue: { task: queue }, + }); + const task = federation.defineTask("kv-dedup", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "first", { deduplicationKey: "k" }); + await ctx.enqueueTask(task, "second", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + strictEqual(queue.enqueued[0].message.taskName, "kv-dedup"); + // A non-native queue never receives a key it would ignore. + strictEqual(queue.enqueued[0].options?.deduplicationKey, undefined); + }, + ); + + await t.step( + "re-enqueues with the same key after the TTL expires", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new MemoryKvStore(), + queue: { task: queue }, + taskDeduplicationTtl: { milliseconds: 100 }, + }); + const task = federation.defineTask("kv-dedup-ttl", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "first", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + // Wait comfortably past the 100 ms TTL so the marker expires. + await delay(300); + await ctx.enqueueTask(task, "second", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 2); + }, + ); + + await t.step( + 'rejects with TypeError when fallback is "closed" and no cas exists', + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new CaslessKvStore(), + queue: { task: queue }, + taskDeduplicationFallback: "closed", + }); + const task = federation.defineTask("closed-fallback", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + () => ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }), + { name: "TypeError" }, + ); + strictEqual(queue.enqueued.length, 0); + }, + ); + + await t.step( + 'proceeds when fallback is "open" and no cas exists', + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new CaslessKvStore(), + queue: { task: queue }, + taskDeduplicationFallback: "open", + }); + const task = federation.defineTask("open-fallback", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + // Best-effort fallback never forwards the key to a non-native queue. + strictEqual(queue.enqueued[0].options?.deduplicationKey, undefined); + }, + ); + + await t.step( + "writes only under taskDeduplication, never activityIdempotence", + async () => { + const queue = new MockQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("prefix-isolation", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); + strictEqual((await collectKeys(kv, TASK_DEDUP_PREFIX)).length, 1); + strictEqual( + (await collectKeys(kv, ACTIVITY_IDEMPOTENCE_PREFIX)).length, + 0, + ); + }, + ); + + await t.step("applies one batch-level key to enqueueTaskMany", async () => { + const queue = new MockQueue({ supportsEnqueueMany: true }); + const federation = createFederation({ + ...baseOptions, + kv: new MemoryKvStore(), + queue: { task: queue }, + }); + const task = federation.defineTask("batch-dedup", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }); + await ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }); + // First batch enqueues all three; the second is skipped entirely. + strictEqual(queue.enqueuedMany.length, 1); + strictEqual(queue.enqueuedMany[0].messages.length, 3); + }); +}); + +test( + "task deduplication validates every payload before reserving the key", + async () => { + const queue = new MockQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("dedup-validation", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + // A rejected payload must neither enqueue nor consume the key. + await rejects(() => + ctx.enqueueTask(task, 123 as unknown as string, { + deduplicationKey: "k", + }) + ); + strictEqual(queue.enqueued.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + + // The same key must remain usable by the first valid enqueue. + await ctx.enqueueTask(task, "valid", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + deepStrictEqual( + await collectKeys(kv, TASK_DEDUP_PREFIX), + [[...TASK_DEDUP_PREFIX, "k"]], + ); + + // Once the valid enqueue reserves it, the same key must deduplicate. + await ctx.enqueueTask(task, "duplicate", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + }, +); + +test( + "native task batch deduplication is one enqueueMany operation per call", + async () => { + class NativeBatchDeduplicatingQueue implements MessageQueue { + readonly nativeDeduplication = true; + readonly #seen = new Set(); + readonly attempts: { + messages: readonly TaskMessage[]; + options?: MessageQueueEnqueueOptions; + }[] = []; + readonly accepted: { + messages: readonly TaskMessage[]; + options?: MessageQueueEnqueueOptions; + }[] = []; + + enqueue(): Promise { + throw new Error("A multi-item native batch must use enqueueMany()."); + } + + enqueueMany( + messages: readonly TaskMessage[], + options?: MessageQueueEnqueueOptions, + ): Promise { + const key = options?.deduplicationKey; + if (key == null) { + throw new TypeError( + "Native batch enqueue requires a deduplication key.", + ); + } + this.attempts.push({ messages, options }); + if (this.#seen.has(key)) return Promise.resolve(); + this.#seen.add(key); + this.accepted.push({ messages, options }); + return Promise.resolve(); + } + + listen(): Promise { + return Promise.resolve(); + } + } + + const queue = new NativeBatchDeduplicatingQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("native-batch-dedup", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + await ctx.enqueueTaskMany(task, ["a1", "a2", "a3"], { + deduplicationKey: "batch-a", + }); + await ctx.enqueueTaskMany( + task, + ["duplicate1", "duplicate2", "duplicate3"], + { + deduplicationKey: "batch-a", + }, + ); + await ctx.enqueueTaskMany(task, ["b1", "b2", "b3"], { + deduplicationKey: "batch-b", + }); + + // Every API call reaches the backend exactly once, with one key governing + // all three messages. The backend accepts complete batches or none. + strictEqual(queue.attempts.length, 3); + deepStrictEqual( + queue.attempts.map(({ messages }) => messages.length), + [3, 3, 3], + ); + deepStrictEqual( + queue.attempts.map(({ options }) => options?.deduplicationKey), + ["batch-a", "batch-a", "batch-b"], + ); + strictEqual(queue.accepted.length, 2); + deepStrictEqual( + queue.accepted.map(({ messages }) => messages.length), + [3, 3], + ); + deepStrictEqual( + queue.accepted.map(({ options }) => options?.deduplicationKey), + ["batch-a", "batch-b"], + ); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + }, +); + +test( + "native task batch deduplication rejects without enqueueMany", + async () => { + const queue = new MockQueue({ nativeDeduplication: true }); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("native-batch-without-enqueue-many", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + await rejects( + () => + ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }), + { name: "TypeError", message: /enqueueMany/ }, + ); + + // Reject before any partial enqueue or fallback KV write. Silently + // dropping the key from items 2..n cannot satisfy these assertions. + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + + // A one-item batch is representable by enqueue() and must remain valid. + await ctx.enqueueTaskMany(task, ["single"], { + deduplicationKey: "single", + }); + strictEqual(queue.enqueued.length, 1); + strictEqual(queue.enqueued[0].options?.deduplicationKey, "single"); + }, +); + +test( + "deduplication - native batch capability errors precede payload validation", + async () => { + let validationCalls = 0; + const schema = makeSchema((data): data is string => { + validationCalls++; + return typeof data === "string"; + }); + const queue = new MockQueue({ nativeDeduplication: true }); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("native-batch-capability-order", { + schema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + let caught: unknown; + try { + await ctx.enqueueTaskMany( + task, + [1, 2, 3] as unknown as readonly string[], + { deduplicationKey: "batch" }, + ); + } catch (error) { + caught = error; + } + + // The queue capability makes this request impossible regardless of the + // payload, so no user-supplied validator may run first. + strictEqual(validationCalls, 0); + ok(caught instanceof TypeError); + ok(caught.message.includes("enqueueMany")); + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + }, +); + +test( + "closed deduplication fallback errors precede payload validation", + async () => { + let validationCalls = 0; + const schema = makeSchema((data): data is string => { + validationCalls++; + return typeof data === "string"; + }); + const queue = new MockQueue(); + const kv = new CaslessKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + taskDeduplicationFallback: "closed", + }); + const task = federation.defineTask("closed-fallback-order", { + schema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + let caught: unknown; + try { + await ctx.enqueueTask( + task, + 1 as unknown as string, + { deduplicationKey: "k" }, + ); + } catch (error) { + caught = error; + } + + // Closed fallback is a configuration-level rejection. It must be + // deterministic and independent of user payload validation. + strictEqual(validationCalls, 0); + ok(caught instanceof TypeError); + ok(caught.message.includes("conditional write")); + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + }, +); From 6ebba539ecb677475e12093f5933506bc21e6614 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Sun, 14 Jun 2026 10:12:52 +0000 Subject: [PATCH 31/57] Extract task enqueue pipeline out of ContextImpl The #enqueueTasks and #encodeTaskMessage methods made ContextImpl oversized, so move the handle validation, deduplication planning, payload encoding, and queue dispatch into a new tasks/enqueue.ts module. ContextImpl now delegates to enqueueTasks(), passing only the small slice of itself (federation, codec, origin, data) the pipeline needs. Pull the shared task-test helpers (the schema factory, stock schemas, base federation options, and the recording MockQueue) into a new testing/mq-tasks.ts module, and split the enqueue-specific cases out of tasks.test.ts into enqueue.test.ts. Teach the fixture-usage check to expand glob patterns in its allowlist so the whole testing/ directory is covered by a single entry instead of one path per file. Assisted-by: Claude Code:claude-opus-4-8 --- packages/fedify/src/federation/middleware.ts | 145 +--- .../src/federation/tasks/enqueue.test.ts | 781 +++++++++++++++++ .../fedify/src/federation/tasks/enqueue.ts | 254 ++++++ packages/fedify/src/federation/tasks/mod.ts | 1 + .../fedify/src/federation/tasks/tasks.test.ts | 802 +----------------- packages/fedify/src/testing/mod.ts | 11 +- packages/fedify/src/testing/mq-tasks.ts | 125 +++ scripts/check_fixture_usage.ts | 28 +- 8 files changed, 1191 insertions(+), 956 deletions(-) create mode 100644 packages/fedify/src/federation/tasks/enqueue.test.ts create mode 100644 packages/fedify/src/federation/tasks/enqueue.ts create mode 100644 packages/fedify/src/testing/mq-tasks.ts diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 800b864c5..3e7498a9c 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -168,6 +168,7 @@ import { type SenderKeyPair, } from "./send.ts"; import { + enqueueTasks, TaskCodec, type TaskDefinition, type TaskEnqueueOptions, @@ -3168,6 +3169,10 @@ export class ContextImpl implements Context { return this.#codec ??= new TaskCodec(this); } + get #enqueueTasks(): ReturnType { + return enqueueTasks(this) as ReturnType; + } + clone(data: TContextData): Context { return new ContextImpl({ url: this.url, @@ -3720,146 +3725,6 @@ export class ContextImpl implements Context { await this.#enqueueTasks(task, payloads, options); } - async #enqueueTasks( - task: TaskDefinition, - items: readonly TData[], - options: TaskEnqueueOptions, - ): Promise { - // Fail fast on a handle from another federation instance; without this - // check the message would enqueue fine and be dropped by the worker. - // Compare the registered handle by identity, not just the name: another - // instance may define the same task name with a different schema, and - // its handle would otherwise encode under that foreign schema here - // while the worker decodes under the local one. - const def = this.federation.taskDefinitions.get(task.name); - if (def == null || def.handle !== task) { - throw new TypeError( - `Task ${ - JSON.stringify(task.name) - } is not defined on this federation; ` + - "pass a handle returned by its defineTask().", - ); - } - const queue = this.federation.resolveTaskQueue(task.name); - if (queue == null) { - throw new TypeError( - "No message queue is configured for tasks; pass `queue` to " + - "createFederation() or to defineTask().", - ); - } - if (items.length < 1) return; - const delay = options.delay == null - ? undefined - : Temporal.Duration.from(options.delay); - type DedupPlan = - | { kind: "none" } - | { kind: "native"; key: string } - | { kind: "cas"; key: string; cas: NonNullable } - | { kind: "open"; key: string }; - let dedupPlan: DedupPlan = { kind: "none" }; - if (options.deduplicationKey != null) { - const key = options.deduplicationKey; - if (queue.nativeDeduplication === true) { - if (items.length > 1 && queue.enqueueMany == null) { - throw new TypeError( - `Task ${ - JSON.stringify(task.name) - } was enqueued as a batch with a deduplicationKey, but its ` + - "message queue declares nativeDeduplication without " + - "implementing enqueueMany; a per-message key cannot deduplicate " + - "a whole batch. Implement enqueueMany on the queue, or enqueue " + - "the tasks individually with enqueueTask().", - ); - } - dedupPlan = { kind: "native", key }; - } else if (this.federation.kv.cas != null) { - dedupPlan = { - kind: "cas", - key, - cas: this.federation.kv.cas.bind(this.federation.kv), - }; - } else if (this.federation.taskDeduplicationFallback === "closed") { - // No conditional write, closed: fail fast before any side effect. - throw new TypeError( - "deduplicationKey was set but the message queue does not declare " + - "nativeDeduplication and the key-value store exposes no " + - 'conditional write (cas); set taskDeduplicationFallback to "open" ' + - "to proceed without deduplication, or use a backend that " + - "supports it.", - ); - } else { - dedupPlan = { kind: "open", key }; - } - } - // Encode in parallel: `enqueueTaskMany` is the bulk path, and the enqueue - // below already parallelizes, so serial encoding would be the bottleneck. - // `map` preserves order, and a rejected encode (validation failure) rejects - // the whole batch before anything is enqueued, keeping fail-fast intact. - const messages: TaskMessage[] = await Promise.all( - items.map(this.#encodeTaskMessage(task, options)), - ); - let forwardedDeduplicationKey: string | undefined; - if (dedupPlan.kind === "native") { - forwardedDeduplicationKey = dedupPlan.key; - } else if (dedupPlan.kind === "cas") { - const cacheKey = [ - ...this.federation.kvPrefixes.taskDeduplication, - dedupPlan.key, - ] satisfies KvKey; - const won = await dedupPlan.cas(cacheKey, undefined, true, { - ttl: this.federation.taskDeduplicationTtl, - }); - if (!won) return; - } else if (dedupPlan.kind === "open") { - getLogger(["fedify", "federation", "task"]).debug( - "deduplicationKey {deduplicationKey} for task {taskName} ignored: " + - "the message queue declares no nativeDeduplication and the " + - "key-value store has no cas; proceeding (taskDeduplicationFallback " + - 'is "open").', - { deduplicationKey: dedupPlan.key, taskName: task.name }, - ); - } - if (!this.federation.manuallyStartQueue) { - this.federation._startQueueInternal(this.data); - } - const enqueueOptions = { - delay, - orderingKey: options.orderingKey, - deduplicationKey: forwardedDeduplicationKey, - }; - if (messages.length === 1) { - await queue.enqueue(messages[0], enqueueOptions); - } else if (queue.enqueueMany != null) { - await queue.enqueueMany(messages, enqueueOptions); - } else { - const fanoutOptions = { delay, orderingKey: options.orderingKey }; - await Promise.all( - messages.map((m) => queue.enqueue(m, fanoutOptions)), - ); - } - } - - #encodeTaskMessage = ( - task: TaskDefinition, - options: TaskEnqueueOptions, - ) => - async (data: TData): Promise => { - const encoded = await this.codec.encode(task.schema, data); - const carrier: Record = {}; - propagation.inject(context.active(), carrier); - return { - type: "task", - id: crypto.randomUUID(), - baseUrl: this.origin, - taskName: task.name, - data: encoded, - started: Temporal.Now.instant().toString(), - attempt: 0, - orderingKey: options.orderingKey, - traceContext: carrier, - }; - }; - sendActivity( sender: | SenderKeyPair diff --git a/packages/fedify/src/federation/tasks/enqueue.test.ts b/packages/fedify/src/federation/tasks/enqueue.test.ts new file mode 100644 index 000000000..d559cccb5 --- /dev/null +++ b/packages/fedify/src/federation/tasks/enqueue.test.ts @@ -0,0 +1,781 @@ +import { test } from "@fedify/fixture"; +import { delay } from "es-toolkit"; +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; +import { + baseOptions, + makeSchema, + MockQueue, + numberSchema, + stringSchema, +} from "../../testing/mod.ts"; +import { + type KvKey, + type KvStore, + type KvStoreListEntry, + type KvStoreSetOptions, + MemoryKvStore, +} from "../kv.ts"; +import { createFederation } from "../middleware.ts"; +import type { MessageQueue, MessageQueueEnqueueOptions } from "../mq.ts"; +import type { TaskMessage } from "../queue.ts"; + +/** + * A {@link KvStore} that delegates to an in-memory store but deliberately + * omits `cas`, so that `kv.cas == null`. This drives the deduplication + * fallback branches that fire when no conditional-write primitive exists. + */ +class CaslessKvStore implements KvStore { + readonly inner = new MemoryKvStore(); + get(key: KvKey): Promise { + return this.inner.get(key); + } + set(key: KvKey, value: unknown, options?: KvStoreSetOptions): Promise { + return this.inner.set(key, value, options); + } + delete(key: KvKey): Promise { + return this.inner.delete(key); + } + list(prefix?: KvKey): AsyncIterable { + return this.inner.list(prefix); + } + // No `cas`: the fallback branch is reached precisely when `kv.cas == null`. +} + +async function collectKeys(kv: KvStore, prefix: KvKey): Promise { + const keys: KvKey[] = []; + for await (const { key } of kv.list(prefix)) keys.push(key); + return keys; +} + +const TASK_DEDUP_PREFIX: KvKey = ["_fedify", "taskDeduplication"]; +const ACTIVITY_IDEMPOTENCE_PREFIX: KvKey = ["_fedify", "activityIdempotence"]; + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +test("enqueueTasks() validation and dispatch", async (t) => { + await t.step("rejects an invalid payload at enqueue", async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("strictly-typed", { + schema: numberSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + // deno-lint-ignore no-explicit-any + () => ctx.enqueueTask(task, "not a number" as any), + { name: "TypeError", message: /Task data failed schema validation/ }, + ); + strictEqual(queue.enqueued.length, 0); + }); + + await t.step("stamps the message envelope", async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("envelope", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload"); + strictEqual(queue.enqueued.length, 1); + const { message } = queue.enqueued[0]; + strictEqual(message.type, "task"); + strictEqual(message.taskName, "envelope"); + // encodeTaskMessage stamps the context's origin (no trailing slash). + strictEqual(message.baseUrl, "https://example.com"); + strictEqual(message.attempt, 0); + ok(UUID_RE.test(message.id)); + ok(typeof message.data === "string" && message.data.length > 0); + // `started` is a serialized Temporal.Instant. + ok(Temporal.Instant.from(message.started) instanceof Temporal.Instant); + // propagation.inject always populates a (possibly empty) carrier object. + ok( + typeof message.traceContext === "object" && message.traceContext != null, + ); + }); + + await t.step("passes delay and orderingKey through", async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("delayed", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { + delay: { seconds: 30 }, + orderingKey: "user:alice", + }); + strictEqual(queue.enqueued.length, 1); + const { message, options } = queue.enqueued[0]; + strictEqual(message.taskName, "delayed"); + strictEqual(message.orderingKey, "user:alice"); + strictEqual(message.attempt, 0); + ok(options?.delay instanceof Temporal.Duration); + strictEqual(options.delay.total("second"), 30); + strictEqual(options.orderingKey, "user:alice"); + }); + + await t.step( + "starts the task worker on first enqueue without startQueue()", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + manuallyStartQueue: false, + queue: { task: queue }, + }); + const task = federation.defineTask("auto-start", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + // An app that only uses the custom task API never sends an activity, + // so enqueueTask() itself must start the worker like the other + // enqueue paths do; otherwise tasks pile up unprocessed forever. + await ctx.enqueueTask(task, "first"); + strictEqual(queue.listenCount, 1); + // The started flag keeps a second enqueue from re-listening. + await ctx.enqueueTask(task, "second"); + strictEqual(queue.listenCount, 1); + strictEqual(queue.enqueued.length, 2); + }, + ); + + await t.step("throws when the resolved task queue is null", async () => { + // No queue is configured at all, so resolveTaskQueue() returns null and + // the enqueue pipeline must fail fast before encoding any payload. + const federation = createFederation({ ...baseOptions }); + const task = federation.defineTask("queueless", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + () => ctx.enqueueTask(task, "data"), + { name: "TypeError", message: /No message queue is configured/ }, + ); + }); + + await t.step( + "rejects a handle from another federation at enqueue", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const other = createFederation({ + ...baseOptions, + queue: { task: new MockQueue() }, + }); + const foreignTask = other.defineTask("foreign", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + () => ctx.enqueueTask(foreignTask, "data"), + { name: "TypeError", message: /is not defined on this federation/ }, + ); + strictEqual(queue.enqueued.length, 0); + }, + ); + + await t.step( + "rejects a same-named handle from another federation", + async () => { + // Name lookup alone cannot tell a foreign handle apart once both + // instances define the same task name: the local context would + // encode under the *schema carried by the foreign handle*, so a + // payload the local schema rejects would enqueue anyway, only to be + // dropped by the worker decoding under the local schema. Both + // instances share TContextData = void, so the phantom-brand check + // cannot reject this at compile time; the handle-identity guard is + // the only defense. + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + let called = 0; + federation.defineTask("rename", { + schema: numberSchema, // the local "rename" takes a number… + handler: () => { + called++; + }, + }); + const other = createFederation({ + ...baseOptions, + queue: { task: new MockQueue() }, + }); + // …while the other instance's "rename" takes a string: + const foreignTask = other.defineTask("rename", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + () => ctx.enqueueTask(foreignTask, "not a number"), + { name: "TypeError", message: /is not defined on this federation/ }, + ); + strictEqual(queue.enqueued.length, 0); + strictEqual(called, 0); + }, + ); + + await t.step( + "enqueueTaskMany() uses enqueueMany when available", + async () => { + const queue = new MockQueue({ supportsEnqueueMany: true }); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("bulk", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTaskMany(task, ["a", "b", "c"]); + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 1); + strictEqual(queue.enqueuedMany[0].messages.length, 3); + }, + ); + + await t.step( + "enqueueTaskMany() falls back to parallel enqueues", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("bulk-fallback", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTaskMany(task, ["a", "b"]); + strictEqual(queue.enqueued.length, 2); + }, + ); + + await t.step( + "enqueueTaskMany() with no payloads touches no queue", + async () => { + const queue = new MockQueue({ supportsEnqueueMany: true }); + const federation = createFederation({ + ...baseOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("bulk-empty", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTaskMany(task, []); + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 0); + }, + ); +}); + +test("task deduplication", async (t) => { + await t.step( + "forwards the key to a nativeDeduplication queue without writing KV", + async () => { + const queue = new MockQueue({ nativeDeduplication: true }); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("native-dedup", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + strictEqual(queue.enqueued[0].options?.deduplicationKey, "k"); + // The backend owns the check, so Fedify must not write any KV marker. + strictEqual((await collectKeys(kv, TASK_DEDUP_PREFIX)).length, 0); + }, + ); + + await t.step( + "skips a second enqueue with the same key within the TTL", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new MemoryKvStore(), + queue: { task: queue }, + }); + const task = federation.defineTask("kv-dedup", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "first", { deduplicationKey: "k" }); + await ctx.enqueueTask(task, "second", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + strictEqual(queue.enqueued[0].message.taskName, "kv-dedup"); + // A non-native queue never receives a key it would ignore. + strictEqual(queue.enqueued[0].options?.deduplicationKey, undefined); + }, + ); + + await t.step( + "re-enqueues with the same key after the TTL expires", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new MemoryKvStore(), + queue: { task: queue }, + taskDeduplicationTtl: { milliseconds: 100 }, + }); + const task = federation.defineTask("kv-dedup-ttl", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "first", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + // Wait comfortably past the 100 ms TTL so the marker expires. + await delay(300); + await ctx.enqueueTask(task, "second", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 2); + }, + ); + + await t.step( + 'rejects with TypeError when fallback is "closed" and no cas exists', + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new CaslessKvStore(), + queue: { task: queue }, + taskDeduplicationFallback: "closed", + }); + const task = federation.defineTask("closed-fallback", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + () => ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }), + { name: "TypeError" }, + ); + strictEqual(queue.enqueued.length, 0); + }, + ); + + await t.step( + 'proceeds when fallback is "open" and no cas exists', + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new CaslessKvStore(), + queue: { task: queue }, + taskDeduplicationFallback: "open", + }); + const task = federation.defineTask("open-fallback", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + // Best-effort fallback never forwards the key to a non-native queue. + strictEqual(queue.enqueued[0].options?.deduplicationKey, undefined); + }, + ); + + await t.step( + "writes only under taskDeduplication, never activityIdempotence", + async () => { + const queue = new MockQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("prefix-isolation", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); + strictEqual((await collectKeys(kv, TASK_DEDUP_PREFIX)).length, 1); + strictEqual( + (await collectKeys(kv, ACTIVITY_IDEMPOTENCE_PREFIX)).length, + 0, + ); + }, + ); + + await t.step("applies one batch-level key to enqueueTaskMany", async () => { + const queue = new MockQueue({ supportsEnqueueMany: true }); + const federation = createFederation({ + ...baseOptions, + kv: new MemoryKvStore(), + queue: { task: queue }, + }); + const task = federation.defineTask("batch-dedup", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }); + await ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }); + // First batch enqueues all three; the second is skipped entirely. + strictEqual(queue.enqueuedMany.length, 1); + strictEqual(queue.enqueuedMany[0].messages.length, 3); + }); +}); + +test( + "task deduplication validates every payload before reserving the key", + async () => { + const queue = new MockQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("dedup-validation", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + // A rejected payload must neither enqueue nor consume the key. + await rejects(() => + ctx.enqueueTask(task, 123 as unknown as string, { + deduplicationKey: "k", + }) + ); + strictEqual(queue.enqueued.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + + // The same key must remain usable by the first valid enqueue. + await ctx.enqueueTask(task, "valid", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + deepStrictEqual( + await collectKeys(kv, TASK_DEDUP_PREFIX), + [[...TASK_DEDUP_PREFIX, "k"]], + ); + + // Once the valid enqueue reserves it, the same key must deduplicate. + await ctx.enqueueTask(task, "duplicate", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + }, +); + +test( + "native task batch deduplication is one enqueueMany operation per call", + async () => { + class NativeBatchDeduplicatingQueue implements MessageQueue { + readonly nativeDeduplication = true; + readonly #seen = new Set(); + readonly attempts: { + messages: readonly TaskMessage[]; + options?: MessageQueueEnqueueOptions; + }[] = []; + readonly accepted: { + messages: readonly TaskMessage[]; + options?: MessageQueueEnqueueOptions; + }[] = []; + + enqueue(): Promise { + throw new Error("A multi-item native batch must use enqueueMany()."); + } + + enqueueMany( + messages: readonly TaskMessage[], + options?: MessageQueueEnqueueOptions, + ): Promise { + const key = options?.deduplicationKey; + if (key == null) { + throw new TypeError( + "Native batch enqueue requires a deduplication key.", + ); + } + this.attempts.push({ messages, options }); + if (this.#seen.has(key)) return Promise.resolve(); + this.#seen.add(key); + this.accepted.push({ messages, options }); + return Promise.resolve(); + } + + listen(): Promise { + return Promise.resolve(); + } + } + + const queue = new NativeBatchDeduplicatingQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("native-batch-dedup", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + await ctx.enqueueTaskMany(task, ["a1", "a2", "a3"], { + deduplicationKey: "batch-a", + }); + await ctx.enqueueTaskMany( + task, + ["duplicate1", "duplicate2", "duplicate3"], + { + deduplicationKey: "batch-a", + }, + ); + await ctx.enqueueTaskMany(task, ["b1", "b2", "b3"], { + deduplicationKey: "batch-b", + }); + + // Every API call reaches the backend exactly once, with one key governing + // all three messages. The backend accepts complete batches or none. + strictEqual(queue.attempts.length, 3); + deepStrictEqual( + queue.attempts.map(({ messages }) => messages.length), + [3, 3, 3], + ); + deepStrictEqual( + queue.attempts.map(({ options }) => options?.deduplicationKey), + ["batch-a", "batch-a", "batch-b"], + ); + strictEqual(queue.accepted.length, 2); + deepStrictEqual( + queue.accepted.map(({ messages }) => messages.length), + [3, 3], + ); + deepStrictEqual( + queue.accepted.map(({ options }) => options?.deduplicationKey), + ["batch-a", "batch-b"], + ); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + }, +); + +test( + "native task batch deduplication rejects without enqueueMany", + async () => { + const queue = new MockQueue({ nativeDeduplication: true }); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("native-batch-without-enqueue-many", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + await rejects( + () => + ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }), + { name: "TypeError", message: /enqueueMany/ }, + ); + + // Reject before any partial enqueue or fallback KV write. Silently + // dropping the key from items 2..n cannot satisfy these assertions. + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + + // A one-item batch is representable by enqueue() and must remain valid. + await ctx.enqueueTaskMany(task, ["single"], { + deduplicationKey: "single", + }); + strictEqual(queue.enqueued.length, 1); + strictEqual(queue.enqueued[0].options?.deduplicationKey, "single"); + }, +); + +test( + "deduplication - native batch capability errors precede payload validation", + async () => { + let validationCalls = 0; + const schema = makeSchema((data): data is string => { + validationCalls++; + return typeof data === "string"; + }); + const queue = new MockQueue({ nativeDeduplication: true }); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("native-batch-capability-order", { + schema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + let caught: unknown; + try { + await ctx.enqueueTaskMany( + task, + [1, 2, 3] as unknown as readonly string[], + { deduplicationKey: "batch" }, + ); + } catch (error) { + caught = error; + } + + // The queue capability makes this request impossible regardless of the + // payload, so no user-supplied validator may run first. + strictEqual(validationCalls, 0); + ok(caught instanceof TypeError); + ok(caught.message.includes("enqueueMany")); + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + }, +); + +test( + "closed deduplication fallback errors precede payload validation", + async () => { + let validationCalls = 0; + const schema = makeSchema((data): data is string => { + validationCalls++; + return typeof data === "string"; + }); + const queue = new MockQueue(); + const kv = new CaslessKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + taskDeduplicationFallback: "closed", + }); + const task = federation.defineTask("closed-fallback-order", { + schema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + let caught: unknown; + try { + await ctx.enqueueTask( + task, + 1 as unknown as string, + { deduplicationKey: "k" }, + ); + } catch (error) { + caught = error; + } + + // Closed fallback is a configuration-level rejection. It must be + // deterministic and independent of user payload validation. + strictEqual(validationCalls, 0); + ok(caught instanceof TypeError); + ok(caught.message.includes("conditional write")); + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + }, +); diff --git a/packages/fedify/src/federation/tasks/enqueue.ts b/packages/fedify/src/federation/tasks/enqueue.ts new file mode 100644 index 000000000..e377b9d6e --- /dev/null +++ b/packages/fedify/src/federation/tasks/enqueue.ts @@ -0,0 +1,254 @@ +/** + * The enqueue pipeline for custom background tasks. `ContextImpl.enqueueTask` + * and `ContextImpl.enqueueTaskMany` delegate to {@link enqueueTasks} so the + * handle validation, deduplication planning, payload encoding, and queue + * dispatch live in one cohesive place instead of one oversized method. + * + * @module + */ +import { getLogger } from "@logtape/logtape"; +import { context, propagation } from "@opentelemetry/api"; +import type { KvKey } from "../kv.ts"; +import type { FederationImpl } from "../middleware.ts"; +import type { MessageQueue } from "../mq.ts"; +import type { TaskMessage } from "../queue.ts"; +import type TaskCodec from "./codec.ts"; +import type { TaskDefinition, TaskEnqueueOptions } from "./task.ts"; + +/** + * The slice of an enqueueing {@link Context} that {@link enqueueTasks} needs: + * its federation plus the few values that are the context's own. `ContextImpl` + * assembles it from itself, so the enqueue pipeline stays out of that class. + * @template TContextData The context data to pass to the {@link Context}. + * @internal + */ +interface EnqueueTasksContext { + /** + * The federation that owns the task registry, queue resolution and start, + * the key-value store, and the deduplication configuration. The public + * {@link Federation} interface exposes none of these, so the concrete + * {@link FederationImpl} is required. + */ + readonly federation: FederationImpl; + + /** The codec, bound to this context's loaders, that encodes payloads. */ + readonly codec: TaskCodec; + + /** The context's origin, stamped onto each message as its `baseUrl`. */ + readonly origin: string; + + /** The context data handed to the queue worker when it auto-starts. */ + readonly data: TContextData; +} + +/** + * Validates the task handle, plans deduplication, encodes every payload, then + * dispatches the resulting messages to the queue. A single item flows through + * the same pipeline as a batch, so {@link Context.enqueueTask} and + * {@link Context.enqueueTaskMany} share one implementation. + * @template TContextData The context data to pass to the {@link Context}. + * @template TData The type of the task payload, inferred from the task's schema. + * @param ctx The enqueueing dependencies assembled by `ContextImpl`. + * @param task The handle returned by `defineTask()`. + * @param items The payloads to enqueue, in order. + * @param options The enqueue options governing delay, ordering, and dedup. + * @internal + */ +const enqueueTasks = ( + ctx: EnqueueTasksContext, +) => + async function ( + task: TaskDefinition, + items: readonly TData[], + options: TaskEnqueueOptions, + ): Promise { + const def = ctx.federation.taskDefinitions.get(task.name); + if (def == null || def.handle !== task) { + throw new TypeError( + `Task ${ + JSON.stringify(task.name) + } is not defined on this federation; ` + + "pass a handle returned by its defineTask().", + ); + } + const queue = ctx.federation.resolveTaskQueue(task.name); + if (queue == null) { + throw new TypeError( + "No message queue is configured for tasks; pass `queue` to " + + "createFederation() or to defineTask().", + ); + } + if (items.length < 1) return; + const plan = planDeduplication( + ctx, + queue, + task.name, + options, + items.length, + ); + const messages: TaskMessage[] = await Promise.all( + items.map(encodeTaskMessage(ctx.codec, ctx.origin, task, options)), + ); + const claim = await claimDeduplication(ctx, plan, task.name); + if (!claim.proceed) return; + if (!ctx.federation.manuallyStartQueue) { + ctx.federation._startQueueInternal(ctx.data); + } + await dispatch(queue, messages, { + delay: getDurationIfDefined(options.delay), + orderingKey: options.orderingKey, + deduplicationKey: claim.forwardedDeduplicationKey, + }); + }; + +export default enqueueTasks; + +const getDurationIfDefined = (item: Temporal.DurationLike | undefined) => + item == null ? undefined : Temporal.Duration.from(item); + +/** + * The deduplication strategy chosen for an enqueue, settled before any payload + * is encoded so the fail-fast errors surface first. + */ +type DedupPlan = + | { readonly kind: "none" } + | { readonly kind: "native"; readonly key: string } + | { readonly kind: "cas"; readonly key: string } + | { readonly kind: "open"; readonly key: string }; + +/** + * Decides how a `deduplicationKey` (if any) is honored: forwarded to a native + * queue, claimed via `cas`, or—when neither is available—dropped or rejected + * per the federation's `taskDeduplicationFallback`. Throws the fail-fast + * `TypeError`s so they precede the encode. + */ +function planDeduplication( + ctx: EnqueueTasksContext, + queue: MessageQueue, + taskName: string, + options: TaskEnqueueOptions, + itemCount: number, +): DedupPlan { + if (options.deduplicationKey == null) return { kind: "none" }; + const key = options.deduplicationKey; + if (queue.nativeDeduplication === true) { + if (itemCount > 1 && queue.enqueueMany == null) { + throw new TypeError( + `Task ${ + JSON.stringify(taskName) + } was enqueued as a batch with a deduplicationKey, but its ` + + "message queue declares nativeDeduplication without " + + "implementing enqueueMany; a per-message key cannot deduplicate " + + "a whole batch. Implement enqueueMany on the queue, or enqueue " + + "the tasks individually with enqueueTask().", + ); + } + return { kind: "native", key }; + } + if (ctx.federation.kv.cas != null) return { kind: "cas", key }; + if (ctx.federation.taskDeduplicationFallback === "closed") { + // No conditional write, closed: fail fast before any side effect. + throw new TypeError( + "deduplicationKey was set but the message queue does not declare " + + "nativeDeduplication and the key-value store exposes no " + + 'conditional write (cas); set taskDeduplicationFallback to "open" ' + + "to proceed without deduplication, or use a backend that " + + "supports it.", + ); + } + return { kind: "open", key }; +} + +/** + * Executes the planned deduplication once the payloads are encoded. A native + * plan forwards its key to the queue; a `cas` plan claims the marker and stops + * the enqueue when it loses the race; an `open` plan logs and proceeds. + * @returns Whether to proceed, and the key (if any) to forward to the queue. + */ +async function claimDeduplication( + ctx: EnqueueTasksContext, + plan: DedupPlan, + taskName: string, +): Promise<{ proceed: boolean; forwardedDeduplicationKey?: string }> { + switch (plan.kind) { + case "native": + return { proceed: true, forwardedDeduplicationKey: plan.key }; + case "cas": { + const cacheKey = [ + ...ctx.federation.kvPrefixes.taskDeduplication, + plan.key, + ] satisfies KvKey; + // planDeduplication only picks "cas" when `ctx.federation.kv.cas` exists. + const won = await ctx.federation.kv.cas!(cacheKey, undefined, true, { + ttl: ctx.federation.taskDeduplicationTtl, + }); + return { proceed: won }; + } + case "open": { + getLogger(["fedify", "federation", "task"]).debug( + "deduplicationKey {deduplicationKey} for task {taskName} ignored: " + + "the message queue declares no nativeDeduplication and the " + + "key-value store has no cas; proceeding (taskDeduplicationFallback " + + 'is "open").', + { deduplicationKey: plan.key, taskName }, + ); + } + } + return { proceed: true }; +} + +/** + * Sends the encoded messages to the queue, picking the bulk path when the + * queue implements `enqueueMany` and otherwise fanning out parallel single + * enqueues. The fan-out drops `deduplicationKey`, which is only ever set for a + * native plan that the bulk paths already cover. + */ +async function dispatch( + queue: MessageQueue, + messages: readonly TaskMessage[], + options: { + delay?: Temporal.Duration; + orderingKey?: string; + deduplicationKey?: string; + }, +): Promise { + if (messages.length === 1) { + await queue.enqueue(messages[0], options); + } else if (queue.enqueueMany != null) { + await queue.enqueueMany(messages, options); + } else { + const fanoutOptions = { + delay: options.delay, + orderingKey: options.orderingKey, + }; + await Promise.all(messages.map((m) => queue.enqueue(m, fanoutOptions))); + } +} + +/** + * Builds the per-payload encoder: validates and serializes the payload, then + * stamps the message envelope with a fresh id, the context's origin, and the + * active trace context. Curried so the batch encode reuses one bound encoder. + */ +const encodeTaskMessage = ( + codec: TaskCodec, + origin: string, + task: TaskDefinition, + options: TaskEnqueueOptions, +) => +async (data: TData): Promise => { + const encoded = await codec.encode(task.schema, data); + const carrier: Record = {}; + propagation.inject(context.active(), carrier); + return { + type: "task", + id: crypto.randomUUID(), + baseUrl: origin, + taskName: task.name, + data: encoded, + started: Temporal.Now.instant().toString(), + attempt: 0, + orderingKey: options.orderingKey, + traceContext: carrier, + }; +}; diff --git a/packages/fedify/src/federation/tasks/mod.ts b/packages/fedify/src/federation/tasks/mod.ts index e151fa576..62072ac45 100644 --- a/packages/fedify/src/federation/tasks/mod.ts +++ b/packages/fedify/src/federation/tasks/mod.ts @@ -7,6 +7,7 @@ * @module */ export { default as TaskCodec } from "./codec.ts"; +export { default as enqueueTasks } from "./enqueue.ts"; export type { TaskDefinition, TaskDefinitionInternal, diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index dbfbf4972..8b4983f79 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -9,48 +9,23 @@ import { throws, } from "node:assert/strict"; import { - type Envelope, - envelopeSchema, + baseOptions, + makeSchema, MockQueue, numberSchema, stringSchema, } from "../../testing/mod.ts"; import { createFederationBuilder } from "../builder.ts"; import type { Context } from "../context.ts"; -import type { Federatable, FederationOptions } from "../federation.ts"; -import { - type KvKey, - type KvStore, - type KvStoreListEntry, - type KvStoreSetOptions, - MemoryKvStore, -} from "../kv.ts"; +import type { Federatable } from "../federation.ts"; import { createFederation, type FederationImpl } from "../middleware.ts"; -import { - InProcessMessageQueue, - type MessageQueue, - type MessageQueueEnqueueOptions, - type MessageQueueListenOptions, -} from "../mq.ts"; +import { InProcessMessageQueue } from "../mq.ts"; import type { TaskMessage } from "../queue.ts"; import TaskCodec from "./codec.ts"; import type { TaskDefinition, TaskRegistry } from "./task.ts"; type Assert = T; -const makeSchema = ( - check: (data: unknown) => data is T, -): StandardSchemaV1 => ({ - "~standard": { - version: 1, - vendor: "fedify-test", - validate: (value: unknown) => - check(value) - ? { value } - : { issues: [{ message: "Invalid task data." }] }, - }, -}); - interface Envelope { note: Note; title: string; @@ -63,69 +38,6 @@ const envelopeSchema = makeSchema( typeof (data as Envelope).title === "string", ); -const stringSchema = makeSchema((d): d is string => typeof d === "string"); -const numberSchema = makeSchema((d): d is number => typeof d === "number"); - -class MockQueue implements MessageQueue { - readonly nativeRetrial: boolean; - readonly nativeDeduplication: boolean; - readonly enqueued: { - message: TaskMessage; - options?: MessageQueueEnqueueOptions; - }[] = []; - readonly enqueuedMany: { - messages: readonly TaskMessage[]; - options?: MessageQueueEnqueueOptions; - }[] = []; - listenCount = 0; - enqueueMany?: ( - messages: readonly TaskMessage[], - options?: MessageQueueEnqueueOptions, - ) => Promise; - - constructor( - options: { - nativeRetrial?: boolean; - nativeDeduplication?: boolean; - supportsEnqueueMany?: boolean; - } = {}, - ) { - this.nativeRetrial = options.nativeRetrial ?? false; - this.nativeDeduplication = options.nativeDeduplication ?? false; - if (options.supportsEnqueueMany) { - this.enqueueMany = (messages, opts) => { - this.enqueuedMany.push({ messages, options: opts }); - return Promise.resolve(); - }; - } - } - - enqueue( - message: TaskMessage, - options?: MessageQueueEnqueueOptions, - ): Promise { - this.enqueued.push({ message, options }); - return Promise.resolve(); - } - - listen( - _handler: (message: TaskMessage) => Promise | void, - options?: MessageQueueListenOptions, - ): Promise { - this.listenCount++; - return new Promise((resolve) => { - options?.signal?.addEventListener("abort", () => resolve()); - }); - } -} - -const baseOptions: Omit, "queue"> = { - kv: new MemoryKvStore(), - documentLoaderFactory: () => mockDocumentLoader, - contextLoaderFactory: () => mockDocumentLoader, - manuallyStartQueue: true, -}; - const makeTaskMessage = async ( taskName: string, data: unknown, @@ -303,224 +215,6 @@ test("Context.enqueueTask() end-to-end", async (t) => { strictEqual(data.title, "greeting"); strictEqual(handlerCtx.origin, "https://example.com"); }); - - await t.step("rejects an invalid payload at enqueue", async () => { - const queue = new MockQueue(); - const federation = createFederation({ - ...baseOptions, - queue: { task: queue }, - }); - const task = federation.defineTask("strictly-typed", { - schema: numberSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await rejects( - // deno-lint-ignore no-explicit-any - () => ctx.enqueueTask(task, "not a number" as any), - { name: "TypeError", message: /Task data failed schema validation/ }, - ); - strictEqual(queue.enqueued.length, 0); - }); - - await t.step( - "starts the task worker on first enqueue without startQueue()", - async () => { - const queue = new MockQueue(); - const federation = createFederation({ - ...baseOptions, - manuallyStartQueue: false, - queue: { task: queue }, - }); - const task = federation.defineTask("auto-start", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - // An app that only uses the custom task API never sends an activity, - // so enqueueTask() itself must start the worker like the other - // enqueue paths do; otherwise tasks pile up unprocessed forever. - await ctx.enqueueTask(task, "first"); - strictEqual(queue.listenCount, 1); - // The started flag keeps a second enqueue from re-listening. - await ctx.enqueueTask(task, "second"); - strictEqual(queue.listenCount, 1); - strictEqual(queue.enqueued.length, 2); - }, - ); - - await t.step( - "rejects a handle from another federation at enqueue", - async () => { - const queue = new MockQueue(); - const federation = createFederation({ - ...baseOptions, - queue: { task: queue }, - }); - const other = createFederation({ - ...baseOptions, - queue: { task: new MockQueue() }, - }); - const foreignTask = other.defineTask("foreign", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await rejects( - () => ctx.enqueueTask(foreignTask, "data"), - { name: "TypeError", message: /is not defined on this federation/ }, - ); - strictEqual(queue.enqueued.length, 0); - }, - ); - - await t.step( - "rejects a same-named handle from another federation", - async () => { - // Name lookup alone cannot tell a foreign handle apart once both - // instances define the same task name: the local context would - // encode under the *schema carried by the foreign handle*, so a - // payload the local schema rejects would enqueue anyway, only to be - // dropped by the worker decoding under the local schema. Both - // instances share TContextData = void, so the phantom-brand check - // cannot reject this at compile time; the handle-identity guard is - // the only defense. - const queue = new MockQueue(); - const federation = createFederation({ - ...baseOptions, - queue: { task: queue }, - }); - let called = 0; - federation.defineTask("rename", { - schema: numberSchema, // the local "rename" takes a number… - handler: () => { - called++; - }, - }); - const other = createFederation({ - ...baseOptions, - queue: { task: new MockQueue() }, - }); - // …while the other instance's "rename" takes a string: - const foreignTask = other.defineTask("rename", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await rejects( - () => ctx.enqueueTask(foreignTask, "not a number"), - { name: "TypeError", message: /is not defined on this federation/ }, - ); - strictEqual(queue.enqueued.length, 0); - strictEqual(called, 0); - }, - ); - - await t.step("passes delay and orderingKey through", async () => { - const queue = new MockQueue(); - const federation = createFederation({ - ...baseOptions, - queue: { task: queue }, - }); - const task = federation.defineTask("delayed", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await ctx.enqueueTask(task, "payload", { - delay: { seconds: 30 }, - orderingKey: "user:alice", - }); - strictEqual(queue.enqueued.length, 1); - const { message, options } = queue.enqueued[0]; - strictEqual(message.taskName, "delayed"); - strictEqual(message.orderingKey, "user:alice"); - strictEqual(message.attempt, 0); - ok(options?.delay instanceof Temporal.Duration); - strictEqual(options.delay.total("second"), 30); - strictEqual(options.orderingKey, "user:alice"); - }); - - await t.step( - "enqueueTaskMany() uses enqueueMany when available", - async () => { - const queue = new MockQueue({ supportsEnqueueMany: true }); - const federation = createFederation({ - ...baseOptions, - queue: { task: queue }, - }); - const task = federation.defineTask("bulk", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await ctx.enqueueTaskMany(task, ["a", "b", "c"]); - strictEqual(queue.enqueued.length, 0); - strictEqual(queue.enqueuedMany.length, 1); - strictEqual(queue.enqueuedMany[0].messages.length, 3); - }, - ); - - await t.step( - "enqueueTaskMany() falls back to parallel enqueues", - async () => { - const queue = new MockQueue(); - const federation = createFederation({ - ...baseOptions, - queue: { task: queue }, - }); - const task = federation.defineTask("bulk-fallback", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await ctx.enqueueTaskMany(task, ["a", "b"]); - strictEqual(queue.enqueued.length, 2); - }, - ); - - await t.step( - "enqueueTaskMany() with no payloads touches no queue", - async () => { - const queue = new MockQueue({ supportsEnqueueMany: true }); - const federation = createFederation({ - ...baseOptions, - queue: { task: queue }, - }); - const task = federation.defineTask("bulk-empty", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await ctx.enqueueTaskMany(task, []); - strictEqual(queue.enqueued.length, 0); - strictEqual(queue.enqueuedMany.length, 0); - }, - ); }); test("task queue routing", async (t) => { @@ -941,491 +635,3 @@ test("processQueuedTask() task dispatch", async (t) => { strictEqual(queue.enqueued.length, 0); }); }); - -/** - * A {@link KvStore} that delegates to an in-memory store but deliberately - * omits `cas`, so that `kv.cas == null`. This drives the deduplication - * fallback branches that fire when no conditional-write primitive exists. - */ -class CaslessKvStore implements KvStore { - readonly inner = new MemoryKvStore(); - get(key: KvKey): Promise { - return this.inner.get(key); - } - set(key: KvKey, value: unknown, options?: KvStoreSetOptions): Promise { - return this.inner.set(key, value, options); - } - delete(key: KvKey): Promise { - return this.inner.delete(key); - } - list(prefix?: KvKey): AsyncIterable { - return this.inner.list(prefix); - } - // No `cas`: the fallback branch is reached precisely when `kv.cas == null`. -} - -async function collectKeys(kv: KvStore, prefix: KvKey): Promise { - const keys: KvKey[] = []; - for await (const { key } of kv.list(prefix)) keys.push(key); - return keys; -} - -const TASK_DEDUP_PREFIX: KvKey = ["_fedify", "taskDeduplication"]; -const ACTIVITY_IDEMPOTENCE_PREFIX: KvKey = ["_fedify", "activityIdempotence"]; - -test("task deduplication", async (t) => { - await t.step( - "forwards the key to a nativeDeduplication queue without writing KV", - async () => { - const queue = new MockQueue({ nativeDeduplication: true }); - const kv = new MemoryKvStore(); - const federation = createFederation({ - ...baseOptions, - kv, - queue: { task: queue }, - }); - const task = federation.defineTask("native-dedup", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); - strictEqual(queue.enqueued.length, 1); - strictEqual(queue.enqueued[0].options?.deduplicationKey, "k"); - // The backend owns the check, so Fedify must not write any KV marker. - strictEqual((await collectKeys(kv, TASK_DEDUP_PREFIX)).length, 0); - }, - ); - - await t.step( - "skips a second enqueue with the same key within the TTL", - async () => { - const queue = new MockQueue(); - const federation = createFederation({ - ...baseOptions, - kv: new MemoryKvStore(), - queue: { task: queue }, - }); - const task = federation.defineTask("kv-dedup", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await ctx.enqueueTask(task, "first", { deduplicationKey: "k" }); - await ctx.enqueueTask(task, "second", { deduplicationKey: "k" }); - strictEqual(queue.enqueued.length, 1); - strictEqual(queue.enqueued[0].message.taskName, "kv-dedup"); - // A non-native queue never receives a key it would ignore. - strictEqual(queue.enqueued[0].options?.deduplicationKey, undefined); - }, - ); - - await t.step( - "re-enqueues with the same key after the TTL expires", - async () => { - const queue = new MockQueue(); - const federation = createFederation({ - ...baseOptions, - kv: new MemoryKvStore(), - queue: { task: queue }, - taskDeduplicationTtl: { milliseconds: 100 }, - }); - const task = federation.defineTask("kv-dedup-ttl", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await ctx.enqueueTask(task, "first", { deduplicationKey: "k" }); - strictEqual(queue.enqueued.length, 1); - // Wait comfortably past the 100 ms TTL so the marker expires. - await delay(300); - await ctx.enqueueTask(task, "second", { deduplicationKey: "k" }); - strictEqual(queue.enqueued.length, 2); - }, - ); - - await t.step( - 'rejects with TypeError when fallback is "closed" and no cas exists', - async () => { - const queue = new MockQueue(); - const federation = createFederation({ - ...baseOptions, - kv: new CaslessKvStore(), - queue: { task: queue }, - taskDeduplicationFallback: "closed", - }); - const task = federation.defineTask("closed-fallback", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await rejects( - () => ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }), - { name: "TypeError" }, - ); - strictEqual(queue.enqueued.length, 0); - }, - ); - - await t.step( - 'proceeds when fallback is "open" and no cas exists', - async () => { - const queue = new MockQueue(); - const federation = createFederation({ - ...baseOptions, - kv: new CaslessKvStore(), - queue: { task: queue }, - taskDeduplicationFallback: "open", - }); - const task = federation.defineTask("open-fallback", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); - strictEqual(queue.enqueued.length, 1); - // Best-effort fallback never forwards the key to a non-native queue. - strictEqual(queue.enqueued[0].options?.deduplicationKey, undefined); - }, - ); - - await t.step( - "writes only under taskDeduplication, never activityIdempotence", - async () => { - const queue = new MockQueue(); - const kv = new MemoryKvStore(); - const federation = createFederation({ - ...baseOptions, - kv, - queue: { task: queue }, - }); - const task = federation.defineTask("prefix-isolation", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); - strictEqual((await collectKeys(kv, TASK_DEDUP_PREFIX)).length, 1); - strictEqual( - (await collectKeys(kv, ACTIVITY_IDEMPOTENCE_PREFIX)).length, - 0, - ); - }, - ); - - await t.step("applies one batch-level key to enqueueTaskMany", async () => { - const queue = new MockQueue({ supportsEnqueueMany: true }); - const federation = createFederation({ - ...baseOptions, - kv: new MemoryKvStore(), - queue: { task: queue }, - }); - const task = federation.defineTask("batch-dedup", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - await ctx.enqueueTaskMany(task, ["a", "b", "c"], { - deduplicationKey: "batch", - }); - await ctx.enqueueTaskMany(task, ["a", "b", "c"], { - deduplicationKey: "batch", - }); - // First batch enqueues all three; the second is skipped entirely. - strictEqual(queue.enqueuedMany.length, 1); - strictEqual(queue.enqueuedMany[0].messages.length, 3); - }); -}); - -test( - "task deduplication validates every payload before reserving the key", - async () => { - const queue = new MockQueue(); - const kv = new MemoryKvStore(); - const federation = createFederation({ - ...baseOptions, - kv, - queue: { task: queue }, - }); - const task = federation.defineTask("dedup-validation", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - - // A rejected payload must neither enqueue nor consume the key. - await rejects(() => - ctx.enqueueTask(task, 123 as unknown as string, { - deduplicationKey: "k", - }) - ); - strictEqual(queue.enqueued.length, 0); - deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); - - // The same key must remain usable by the first valid enqueue. - await ctx.enqueueTask(task, "valid", { deduplicationKey: "k" }); - strictEqual(queue.enqueued.length, 1); - deepStrictEqual( - await collectKeys(kv, TASK_DEDUP_PREFIX), - [[...TASK_DEDUP_PREFIX, "k"]], - ); - - // Once the valid enqueue reserves it, the same key must deduplicate. - await ctx.enqueueTask(task, "duplicate", { deduplicationKey: "k" }); - strictEqual(queue.enqueued.length, 1); - }, -); - -test( - "native task batch deduplication is one enqueueMany operation per call", - async () => { - class NativeBatchDeduplicatingQueue implements MessageQueue { - readonly nativeDeduplication = true; - readonly #seen = new Set(); - readonly attempts: { - messages: readonly TaskMessage[]; - options?: MessageQueueEnqueueOptions; - }[] = []; - readonly accepted: { - messages: readonly TaskMessage[]; - options?: MessageQueueEnqueueOptions; - }[] = []; - - enqueue(): Promise { - throw new Error("A multi-item native batch must use enqueueMany()."); - } - - enqueueMany( - messages: readonly TaskMessage[], - options?: MessageQueueEnqueueOptions, - ): Promise { - const key = options?.deduplicationKey; - if (key == null) { - throw new TypeError( - "Native batch enqueue requires a deduplication key.", - ); - } - this.attempts.push({ messages, options }); - if (this.#seen.has(key)) return Promise.resolve(); - this.#seen.add(key); - this.accepted.push({ messages, options }); - return Promise.resolve(); - } - - listen(): Promise { - return Promise.resolve(); - } - } - - const queue = new NativeBatchDeduplicatingQueue(); - const kv = new MemoryKvStore(); - const federation = createFederation({ - ...baseOptions, - kv, - queue: { task: queue }, - }); - const task = federation.defineTask("native-batch-dedup", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - - await ctx.enqueueTaskMany(task, ["a1", "a2", "a3"], { - deduplicationKey: "batch-a", - }); - await ctx.enqueueTaskMany( - task, - ["duplicate1", "duplicate2", "duplicate3"], - { - deduplicationKey: "batch-a", - }, - ); - await ctx.enqueueTaskMany(task, ["b1", "b2", "b3"], { - deduplicationKey: "batch-b", - }); - - // Every API call reaches the backend exactly once, with one key governing - // all three messages. The backend accepts complete batches or none. - strictEqual(queue.attempts.length, 3); - deepStrictEqual( - queue.attempts.map(({ messages }) => messages.length), - [3, 3, 3], - ); - deepStrictEqual( - queue.attempts.map(({ options }) => options?.deduplicationKey), - ["batch-a", "batch-a", "batch-b"], - ); - strictEqual(queue.accepted.length, 2); - deepStrictEqual( - queue.accepted.map(({ messages }) => messages.length), - [3, 3], - ); - deepStrictEqual( - queue.accepted.map(({ options }) => options?.deduplicationKey), - ["batch-a", "batch-b"], - ); - deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); - }, -); - -test( - "native task batch deduplication rejects without enqueueMany", - async () => { - const queue = new MockQueue({ nativeDeduplication: true }); - const kv = new MemoryKvStore(); - const federation = createFederation({ - ...baseOptions, - kv, - queue: { task: queue }, - }); - const task = federation.defineTask("native-batch-without-enqueue-many", { - schema: stringSchema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - - await rejects( - () => - ctx.enqueueTaskMany(task, ["a", "b", "c"], { - deduplicationKey: "batch", - }), - { name: "TypeError", message: /enqueueMany/ }, - ); - - // Reject before any partial enqueue or fallback KV write. Silently - // dropping the key from items 2..n cannot satisfy these assertions. - strictEqual(queue.enqueued.length, 0); - strictEqual(queue.enqueuedMany.length, 0); - deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); - - // A one-item batch is representable by enqueue() and must remain valid. - await ctx.enqueueTaskMany(task, ["single"], { - deduplicationKey: "single", - }); - strictEqual(queue.enqueued.length, 1); - strictEqual(queue.enqueued[0].options?.deduplicationKey, "single"); - }, -); - -test( - "deduplication - native batch capability errors precede payload validation", - async () => { - let validationCalls = 0; - const schema = makeSchema((data): data is string => { - validationCalls++; - return typeof data === "string"; - }); - const queue = new MockQueue({ nativeDeduplication: true }); - const kv = new MemoryKvStore(); - const federation = createFederation({ - ...baseOptions, - kv, - queue: { task: queue }, - }); - const task = federation.defineTask("native-batch-capability-order", { - schema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - - let caught: unknown; - try { - await ctx.enqueueTaskMany( - task, - [1, 2, 3] as unknown as readonly string[], - { deduplicationKey: "batch" }, - ); - } catch (error) { - caught = error; - } - - // The queue capability makes this request impossible regardless of the - // payload, so no user-supplied validator may run first. - strictEqual(validationCalls, 0); - ok(caught instanceof TypeError); - ok(caught.message.includes("enqueueMany")); - strictEqual(queue.enqueued.length, 0); - strictEqual(queue.enqueuedMany.length, 0); - deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); - }, -); - -test( - "closed deduplication fallback errors precede payload validation", - async () => { - let validationCalls = 0; - const schema = makeSchema((data): data is string => { - validationCalls++; - return typeof data === "string"; - }); - const queue = new MockQueue(); - const kv = new CaslessKvStore(); - const federation = createFederation({ - ...baseOptions, - kv, - queue: { task: queue }, - taskDeduplicationFallback: "closed", - }); - const task = federation.defineTask("closed-fallback-order", { - schema, - handler: () => {}, - }); - const ctx = federation.createContext( - new URL("https://example.com/"), - undefined, - ); - - let caught: unknown; - try { - await ctx.enqueueTask( - task, - 1 as unknown as string, - { deduplicationKey: "k" }, - ); - } catch (error) { - caught = error; - } - - // Closed fallback is a configuration-level rejection. It must be - // deterministic and independent of user payload validation. - strictEqual(validationCalls, 0); - ok(caught instanceof TypeError); - ok(caught.message.includes("conditional write")); - strictEqual(queue.enqueued.length, 0); - strictEqual(queue.enqueuedMany.length, 0); - deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); - }, -); diff --git a/packages/fedify/src/testing/mod.ts b/packages/fedify/src/testing/mod.ts index 27cf5bdc9..434ffdf39 100644 --- a/packages/fedify/src/testing/mod.ts +++ b/packages/fedify/src/testing/mod.ts @@ -6,11 +6,14 @@ export { export { type Envelope, envelopeSchema, - makeSchema, - MockQueue, type MockQueueOptions, - numberSchema, - stringSchema, } from "./tasks.ts"; // Without the export below, `test:cfworkers` makes an error. export { testDefinitions } from "@fedify/fixture"; +export { + baseOptions, + makeSchema, + MockQueue, + numberSchema, + stringSchema, +} from "./mq-tasks.ts"; diff --git a/packages/fedify/src/testing/mq-tasks.ts b/packages/fedify/src/testing/mq-tasks.ts new file mode 100644 index 000000000..351d97c79 --- /dev/null +++ b/packages/fedify/src/testing/mq-tasks.ts @@ -0,0 +1,125 @@ +/** + * Test-only utilities shared by the task suites in this directory + * (*tasks.test.ts* and *enqueue.test.ts*): the schema factory and stock + * schemas, the base federation options, and the recording {@link MockQueue}. + * + * These helpers live beside the suites that use them rather than in a shared + * package because {@link MockQueue} needs the package-internal + * {@link TaskMessage} type, and *deno.json*'s `publish.exclude` keeps this + * module out of the published sources. + * + * @module + */ +import { mockDocumentLoader } from "@fedify/fixture"; +import type { StandardSchemaV1 } from "@standard-schema/spec"; +import type { FederationOptions } from "../federation/federation.ts"; +import { MemoryKvStore } from "../federation/kv.ts"; +import type { + MessageQueue, + MessageQueueEnqueueOptions, + MessageQueueListenOptions, +} from "../federation/mq.ts"; +import type { TaskMessage } from "../federation/queue.ts"; + +/** + * Builds a minimal [Standard Schema](https://standardschema.dev/) from a type + * guard: values the guard accepts validate, and the rest fail. + */ +export const makeSchema = ( + check: (data: unknown) => data is T, +): StandardSchemaV1 => ({ + "~standard": { + version: 1, + vendor: "fedify-test", + validate: (value: unknown) => + check(value) + ? { value } + : { issues: [{ message: "Invalid task data." }] }, + }, +}); + +/** A schema that accepts only strings. */ +export const stringSchema: StandardSchemaV1 = makeSchema( + (d): d is string => typeof d === "string", +); + +/** A schema that accepts only numbers. */ +export const numberSchema: StandardSchemaV1 = makeSchema( + (d): d is number => typeof d === "number", +); + +/** Federation options (sans `queue`) shared by the task suites. */ +export const baseOptions: Omit, "queue"> = { + kv: new MemoryKvStore(), + documentLoaderFactory: () => mockDocumentLoader, + contextLoaderFactory: () => mockDocumentLoader, + manuallyStartQueue: true, +}; + +/** + * Options for the {@link MockQueue} constructor. + */ +export interface MockQueueOptions { + /** Sets {@link MessageQueue.nativeRetrial}. Defaults to `false`. */ + nativeRetrial?: boolean; + /** Sets {@link MessageQueue.nativeDeduplication}. Defaults to `false`. */ + nativeDeduplication?: boolean; + /** + * When `true`, the queue exposes {@link MockQueue.enqueueMany} and records + * bulk enqueues; when omitted, the method is absent so callers exercise the + * per-message fan-out path. + */ + supportsEnqueueMany?: boolean; +} + +/** + * An in-memory {@link MessageQueue} that records task enqueues for assertions + * instead of delivering anything. Its {@link listen} resolves only when the + * abort signal fires. + */ +export class MockQueue implements MessageQueue { + readonly nativeRetrial: boolean; + readonly nativeDeduplication: boolean; + readonly enqueued: { + message: TaskMessage; + options?: MessageQueueEnqueueOptions; + }[] = []; + readonly enqueuedMany: { + messages: readonly TaskMessage[]; + options?: MessageQueueEnqueueOptions; + }[] = []; + listenCount = 0; + enqueueMany?: ( + messages: readonly TaskMessage[], + options?: MessageQueueEnqueueOptions, + ) => Promise; + + constructor(options: MockQueueOptions = {}) { + this.nativeRetrial = options.nativeRetrial ?? false; + this.nativeDeduplication = options.nativeDeduplication ?? false; + if (options.supportsEnqueueMany) { + this.enqueueMany = (messages, opts) => { + this.enqueuedMany.push({ messages, options: opts }); + return Promise.resolve(); + }; + } + } + + enqueue( + message: TaskMessage, + options?: MessageQueueEnqueueOptions, + ): Promise { + this.enqueued.push({ message, options }); + return Promise.resolve(); + } + + listen( + _handler: (message: TaskMessage) => Promise | void, + options?: MessageQueueListenOptions, + ): Promise { + this.listenCount++; + return new Promise((resolve) => { + options?.signal?.addEventListener("abort", () => resolve()); + }); + } +} diff --git a/scripts/check_fixture_usage.ts b/scripts/check_fixture_usage.ts index 73ef518d9..ad4c921cc 100644 --- a/scripts/check_fixture_usage.ts +++ b/scripts/check_fixture_usage.ts @@ -10,6 +10,7 @@ * Reviewers must NOT treat a passing run as proof of safety; code * review and the published package contents remain the source of truth. */ +import { expandGlobSync } from "@std/fs/expand-glob"; import { walk } from "@std/fs/walk"; import { dirname, @@ -20,6 +21,15 @@ import { SEPARATOR, } from "@std/path"; +const projectRoot = resolve(dirname(fromFileUrl(import.meta.url)), ".."); +const packagesDir = resolve(projectRoot, "packages"); + +const expandGlobPattern = (pattern: string) => + Array.from( + expandGlobSync(pattern, { root: projectRoot, includeDirs: false }), + (file) => relative(projectRoot, file.path), + ); + /** * Files exempt from the "@fedify/fixture imports must live in *.test.ts" * rule. Every entry MUST be accompanied by an inline comment explaining @@ -27,23 +37,13 @@ import { * necessary or not. */ const ALLOWLIST: readonly string[] = [ - // cfworkers test harness re-exports `mockDocumentLoader`; bundled in via - // tsdown `noExternal` so consumers never resolve `@fedify/fixture` at - // runtime. - "packages/fedify/src/testing/context.ts", - // cfworkers test harness re-exports `testDefinitions`; bundled in via - // tsdown `noExternal` so consumers never resolve `@fedify/fixture` at - // runtime. - "packages/fedify/src/testing/mod.ts", - // Test utils for custom tasks - "packages/fedify/src/testing/tasks.ts", + // Utils for tests. + "packages/fedify/src/testing/*", // JSDoc `@example` block mentions `import { test } from "@fedify/fixture"` // as documentation; not a real runtime import. "packages/testing/src/mq-tester.ts", -].map((path) => join(...path.split("/") as [string, ...string[]])); - -const projectRoot = resolve(dirname(fromFileUrl(import.meta.url)), ".."); -const packagesDir = resolve(projectRoot, "packages"); +].map((path) => join(...path.split("/") as [string, ...string[]])) + .flatMap((path) => path.includes("*") ? expandGlobPattern(path) : path); /** * Statement-level pattern for any `import` or `export ... from` From 8c02ddd53b8e998a3a3063eaf906d6a1ebc64cc9 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Sun, 14 Jun 2026 12:35:29 +0000 Subject: [PATCH 32/57] Merge task test helpers back into tasks.ts Two branches both touched the task testing utilities and diverged: one split MockQueue and the shared schemas/options out into mq-tasks.ts, while the other kept evolving them in tasks.ts. After rebasing the common edits, consolidate everything back into a single tasks.ts and drop the now-redundant mq-tasks.ts. Assisted-by: Claude Code:claude-opus-4-8 --- packages/fedify/src/testing/mod.ts | 12 +-- packages/fedify/src/testing/mq-tasks.ts | 125 ------------------------ packages/fedify/src/testing/tasks.ts | 51 ++++++++-- 3 files changed, 48 insertions(+), 140 deletions(-) delete mode 100644 packages/fedify/src/testing/mq-tasks.ts diff --git a/packages/fedify/src/testing/mod.ts b/packages/fedify/src/testing/mod.ts index 434ffdf39..393b217f1 100644 --- a/packages/fedify/src/testing/mod.ts +++ b/packages/fedify/src/testing/mod.ts @@ -4,16 +4,14 @@ export { createRequestContext, } from "./context.ts"; export { + baseOptions, type Envelope, envelopeSchema, - type MockQueueOptions, -} from "./tasks.ts"; -// Without the export below, `test:cfworkers` makes an error. -export { testDefinitions } from "@fedify/fixture"; -export { - baseOptions, makeSchema, MockQueue, + type MockQueueOptions, numberSchema, stringSchema, -} from "./mq-tasks.ts"; +} from "./tasks.ts"; +// Without the export below, `test:cfworkers` makes an error. +export { testDefinitions } from "@fedify/fixture"; diff --git a/packages/fedify/src/testing/mq-tasks.ts b/packages/fedify/src/testing/mq-tasks.ts deleted file mode 100644 index 351d97c79..000000000 --- a/packages/fedify/src/testing/mq-tasks.ts +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Test-only utilities shared by the task suites in this directory - * (*tasks.test.ts* and *enqueue.test.ts*): the schema factory and stock - * schemas, the base federation options, and the recording {@link MockQueue}. - * - * These helpers live beside the suites that use them rather than in a shared - * package because {@link MockQueue} needs the package-internal - * {@link TaskMessage} type, and *deno.json*'s `publish.exclude` keeps this - * module out of the published sources. - * - * @module - */ -import { mockDocumentLoader } from "@fedify/fixture"; -import type { StandardSchemaV1 } from "@standard-schema/spec"; -import type { FederationOptions } from "../federation/federation.ts"; -import { MemoryKvStore } from "../federation/kv.ts"; -import type { - MessageQueue, - MessageQueueEnqueueOptions, - MessageQueueListenOptions, -} from "../federation/mq.ts"; -import type { TaskMessage } from "../federation/queue.ts"; - -/** - * Builds a minimal [Standard Schema](https://standardschema.dev/) from a type - * guard: values the guard accepts validate, and the rest fail. - */ -export const makeSchema = ( - check: (data: unknown) => data is T, -): StandardSchemaV1 => ({ - "~standard": { - version: 1, - vendor: "fedify-test", - validate: (value: unknown) => - check(value) - ? { value } - : { issues: [{ message: "Invalid task data." }] }, - }, -}); - -/** A schema that accepts only strings. */ -export const stringSchema: StandardSchemaV1 = makeSchema( - (d): d is string => typeof d === "string", -); - -/** A schema that accepts only numbers. */ -export const numberSchema: StandardSchemaV1 = makeSchema( - (d): d is number => typeof d === "number", -); - -/** Federation options (sans `queue`) shared by the task suites. */ -export const baseOptions: Omit, "queue"> = { - kv: new MemoryKvStore(), - documentLoaderFactory: () => mockDocumentLoader, - contextLoaderFactory: () => mockDocumentLoader, - manuallyStartQueue: true, -}; - -/** - * Options for the {@link MockQueue} constructor. - */ -export interface MockQueueOptions { - /** Sets {@link MessageQueue.nativeRetrial}. Defaults to `false`. */ - nativeRetrial?: boolean; - /** Sets {@link MessageQueue.nativeDeduplication}. Defaults to `false`. */ - nativeDeduplication?: boolean; - /** - * When `true`, the queue exposes {@link MockQueue.enqueueMany} and records - * bulk enqueues; when omitted, the method is absent so callers exercise the - * per-message fan-out path. - */ - supportsEnqueueMany?: boolean; -} - -/** - * An in-memory {@link MessageQueue} that records task enqueues for assertions - * instead of delivering anything. Its {@link listen} resolves only when the - * abort signal fires. - */ -export class MockQueue implements MessageQueue { - readonly nativeRetrial: boolean; - readonly nativeDeduplication: boolean; - readonly enqueued: { - message: TaskMessage; - options?: MessageQueueEnqueueOptions; - }[] = []; - readonly enqueuedMany: { - messages: readonly TaskMessage[]; - options?: MessageQueueEnqueueOptions; - }[] = []; - listenCount = 0; - enqueueMany?: ( - messages: readonly TaskMessage[], - options?: MessageQueueEnqueueOptions, - ) => Promise; - - constructor(options: MockQueueOptions = {}) { - this.nativeRetrial = options.nativeRetrial ?? false; - this.nativeDeduplication = options.nativeDeduplication ?? false; - if (options.supportsEnqueueMany) { - this.enqueueMany = (messages, opts) => { - this.enqueuedMany.push({ messages, options: opts }); - return Promise.resolve(); - }; - } - } - - enqueue( - message: TaskMessage, - options?: MessageQueueEnqueueOptions, - ): Promise { - this.enqueued.push({ message, options }); - return Promise.resolve(); - } - - listen( - _handler: (message: TaskMessage) => Promise | void, - options?: MessageQueueListenOptions, - ): Promise { - this.listenCount++; - return new Promise((resolve) => { - options?.signal?.addEventListener("abort", () => resolve()); - }); - } -} diff --git a/packages/fedify/src/testing/tasks.ts b/packages/fedify/src/testing/tasks.ts index 2dc50c83d..d32e24217 100644 --- a/packages/fedify/src/testing/tasks.ts +++ b/packages/fedify/src/testing/tasks.ts @@ -1,5 +1,19 @@ +/** + * Test-only utilities shared by the task suites the schema factory and stock + * schemas, the base federation options, and the recording {@link MockQueue}. + * + * These helpers live beside the suites that use them rather than in a shared + * package because {@link MockQueue} needs the package-internal + * {@link TaskMessage} type, and *deno.json*'s `publish.exclude` keeps this + * module out of the published sources. + * + * @module + */ +import { mockDocumentLoader } from "@fedify/fixture"; import { Note } from "@fedify/vocab"; import type { StandardSchemaV1 } from "@standard-schema/spec"; +import type { FederationOptions } from "../federation/federation.ts"; +import { MemoryKvStore } from "../federation/kv.ts"; import type { MessageQueue, MessageQueueEnqueueOptions, @@ -7,6 +21,14 @@ import type { } from "../federation/mq.ts"; import type { TaskMessage } from "../federation/queue.ts"; +/** Federation options (sans `queue`) shared by the task suites. */ +export const baseOptions: Omit, "queue"> = { + kv: new MemoryKvStore(), + documentLoaderFactory: () => mockDocumentLoader, + contextLoaderFactory: () => mockDocumentLoader, + manuallyStartQueue: true, +}; + /** * Builds a minimal [Standard Schema](https://standardschema.dev/) from a type * guard, for use as a task payload schema in tests. @@ -45,19 +67,30 @@ export const envelopeSchema = makeSchema( typeof (data as Envelope).title === "string", ); -/** Options for {@link MockQueue}. */ +/** + * Options for the {@link MockQueue} constructor. + */ export interface MockQueueOptions { + /** Sets {@link MessageQueue.nativeRetrial}. Defaults to `false`. */ readonly nativeRetrial?: boolean; + /** Sets {@link MessageQueue.nativeDeduplication}. Defaults to `false`. */ + readonly nativeDeduplication?: boolean; + /** + * When `true`, the queue exposes {@link MockQueue.enqueueMany} and records + * bulk enqueues; when omitted, the method is absent so callers exercise the + * per-message fan-out path. + */ readonly supportsEnqueueMany?: boolean; } /** - * A {@link MessageQueue} that records what it was asked to enqueue and resolves - * its `listen()` when the abort signal fires, so tests can inspect dispatch - * without a real backend. + * An in-memory {@link MessageQueue} that records task enqueues for assertions + * instead of delivering anything. Its {@link listen} resolves only when the + * abort signal fires. */ export class MockQueue implements MessageQueue { readonly nativeRetrial: boolean; + readonly nativeDeduplication: boolean; readonly enqueued: { message: TaskMessage; options?: MessageQueueEnqueueOptions; @@ -74,6 +107,7 @@ export class MockQueue implements MessageQueue { constructor(options: MockQueueOptions = {}) { this.nativeRetrial = options.nativeRetrial ?? false; + this.nativeDeduplication = options.nativeDeduplication ?? false; if (options.supportsEnqueueMany) { this.enqueueMany = (messages, opts) => { this.enqueuedMany.push({ messages, options: opts }); @@ -82,15 +116,16 @@ export class MockQueue implements MessageQueue { } } - // deno-lint-ignore no-explicit-any - enqueue(message: any, options?: MessageQueueEnqueueOptions): Promise { + enqueue( + message: TaskMessage, + options?: MessageQueueEnqueueOptions, + ): Promise { this.enqueued.push({ message, options }); return Promise.resolve(); } listen( - // deno-lint-ignore no-explicit-any - _handler: (message: any) => Promise | void, + _handler: (message: TaskMessage) => Promise | void, options?: MessageQueueListenOptions, ): Promise { this.listenCount++; From 5e3944f3886af959d112714bc68e74de49b6d8e7 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Mon, 15 Jun 2026 11:36:04 +0000 Subject: [PATCH 33/57] Roll back dedup marker when a task enqueue fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The key-value deduplication path reserved a marker before dispatching to the queue but never undid it when the dispatch failed. A transient backend failure therefore left the marker behind, so the retry was silently deduplicated against a task that had never reached the queue. The cas claim now stores a unique token instead of a bare `true`, and a failed dispatch conditionally clears it (cas succeeds only while the stored value is still our token). The conditional clear keeps a stale rollback from deleting a marker that another concurrent enqueue has already re-claimed. A rollback that itself fails is logged and swallowed so the original enqueue error still reaches the caller. The enqueueMany requirement for deduplicated multi-item batches now keys on whether deduplication is actually applied—a native queue or the cas fallback—rather than on nativeDeduplication alone. Under the "open" fallback (no native dedup, no cas) no marker is taken, so the batch fans out without deduplication instead of throwing. ParallelMessageQueue likewise rejects a deduplicated batch when the wrapped queue lacks enqueueMany, since fanning out cannot carry one key atomically. https://github.com/fedify-dev/fedify/issues/798 Assisted-by: Claude Code:claude-opus-4-8 --- docs/manual/tasks.md | 36 +- packages/fedify/src/federation/middleware.ts | 4 +- packages/fedify/src/federation/mq.test.ts | 88 +++ packages/fedify/src/federation/mq.ts | 9 + .../src/federation/tasks/enqueue.test.ts | 593 +++++++++++++++++- .../fedify/src/federation/tasks/enqueue.ts | 92 ++- packages/fedify/src/federation/tasks/task.ts | 10 +- packages/fedify/src/testing/tasks.ts | 2 +- 8 files changed, 789 insertions(+), 45 deletions(-) diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index 710486d90..9e3863f22 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -317,21 +317,33 @@ follow-ups add it. For `~Context.enqueueTaskMany()`, a single `deduplicationKey` applies to the whole batch: the batch enqueues as a unit or is skipped as a unit, never partially. Per-item deduplication means calling `~Context.enqueueTask()` in -a loop, each with its own key. A queue that declares -`~MessageQueue.nativeDeduplication` must also implement -`~MessageQueue.enqueueMany()` to carry a multi-item batch's key as one unit; -fanning the key out across separate `~MessageQueue.enqueue()` calls cannot drop -a whole batch, so Fedify rejects that combination instead of silently leaking -duplicates. +a loop, each with its own key. Deduplicating a multi-item batch requires the +queue to implement `~MessageQueue.enqueueMany()` so the batch enqueues +atomically—whether the check is native or the key–value fallback. Fanning the +key out across separate `~MessageQueue.enqueue()` calls cannot enqueue a whole +batch as one unit: a native per-message key cannot cover it, and a key–value +marker could not be rolled back cleanly if only some of the fanned-out enqueues +failed. So when deduplication is actually applied—a native queue, or a +key–value store with `~KvStore.cas`—Fedify rejects a multi-item batch with a +`deduplicationKey` on a queue without `~MessageQueue.enqueueMany()` instead of +risking duplicates. Under the `"open"` fallback (no native deduplication and no +`cas`), no marker is taken, so the batch simply fans out without deduplication. + +This applies through `~ParallelMessageQueue` as well: wrapping a queue that +lacks `~MessageQueue.enqueueMany()` does not make batch enqueue atomic, so a +deduplicated multi-item batch on such a wrapper is likewise rejected rather than +collapsed onto one message. > [!WARNING] > The key–value fallback is **best-effort, not transactional**. The marker -> write and the enqueue are separate operations, so a crash between them, the -> `"open"` fallback under concurrency, a non-atomic third-party `~KvStore.cas`, -> or reuse of a key within its TTL window can still admit a duplicate or -> suppress a task. Cleanup is by TTL expiry, not active deletion on handler -> success. Deployments needing strict guarantees use a queue with -> `nativeDeduplication: true`, where the backend owns an atomic check. +> write and the enqueue are separate operations. Fedify rolls the marker back +> when an enqueue fails, so a transient failure does not suppress the retry, but +> a crash before that rollback, the `"open"` fallback under concurrency, a +> non-atomic third-party `~KvStore.cas`, or reuse of a key within its TTL window +> can still admit a duplicate or suppress a task. Cleanup is otherwise by TTL +> expiry, not active deletion on handler success. Deployments needing strict +> guarantees use a queue with `nativeDeduplication: true`, where the backend +> owns an atomic check. Limitations diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 3e7498a9c..5a06fb12e 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -3169,8 +3169,8 @@ export class ContextImpl implements Context { return this.#codec ??= new TaskCodec(this); } - get #enqueueTasks(): ReturnType { - return enqueueTasks(this) as ReturnType; + get #enqueueTasks() { + return enqueueTasks(this); } clone(data: TContextData): Context { diff --git a/packages/fedify/src/federation/mq.test.ts b/packages/fedify/src/federation/mq.test.ts index b42135fff..3bb5c0d88 100644 --- a/packages/fedify/src/federation/mq.test.ts +++ b/packages/fedify/src/federation/mq.test.ts @@ -5,11 +5,13 @@ import { assertFalse, assertGreater, assertGreaterOrEqual, + assertRejects, } from "@std/assert"; import { delay } from "es-toolkit"; import { InProcessMessageQueue, type MessageQueue, + type MessageQueueEnqueueOptions, ParallelMessageQueue, } from "./mq.ts"; @@ -438,6 +440,92 @@ test("ParallelMessageQueue inherits nativeDeduplication", () => { assert(workers.nativeDeduplication); }); +test( + "ParallelMessageQueue forwards deduplicationKey to the wrapped queue", + async () => { + class RecordingQueue implements MessageQueue { + readonly nativeDeduplication = true; + readonly singles: (MessageQueueEnqueueOptions | undefined)[] = []; + readonly batches: (MessageQueueEnqueueOptions | undefined)[] = []; + enqueue( + _message: unknown, + options?: MessageQueueEnqueueOptions, + ): Promise { + this.singles.push(options); + return Promise.resolve(); + } + enqueueMany( + _messages: readonly unknown[], + options?: MessageQueueEnqueueOptions, + ): Promise { + this.batches.push(options); + return Promise.resolve(); + } + listen(): Promise { + return Promise.resolve(); + } + } + + const inner = new RecordingQueue(); + const workers = new ParallelMessageQueue(inner, 5); + await workers.enqueue({ x: 1 }, { deduplicationKey: "k1" }); + await workers.enqueueMany([{ x: 1 }, { x: 2 }], { deduplicationKey: "k2" }); + assertEquals(inner.singles[0]?.deduplicationKey, "k1"); + assertEquals(inner.batches[0]?.deduplicationKey, "k2"); + }, +); + +test( + "ParallelMessageQueue rejects a deduplicated batch when the wrapped queue " + + "lacks enqueueMany", + async () => { + class NoBulkQueue implements MessageQueue { + readonly nativeDeduplication = true; + readonly enqueued: unknown[] = []; + enqueue(message: unknown): Promise { + this.enqueued.push(message); + return Promise.resolve(); + } + listen(): Promise { + return Promise.resolve(); + } + } + + const inner = new NoBulkQueue(); + const workers = new ParallelMessageQueue(inner, 5); + await assertRejects( + () => + workers.enqueueMany([{ x: 1 }, { x: 2 }], { deduplicationKey: "k" }), + TypeError, + "enqueueMany", + ); + // It threw before enqueuing anything. + assertEquals(inner.enqueued.length, 0); + }, +); + +test( + "ParallelMessageQueue still fans out a non-deduplicated batch when the " + + "wrapped queue lacks enqueueMany", + async () => { + class NoBulkQueue implements MessageQueue { + readonly enqueued: unknown[] = []; + enqueue(message: unknown): Promise { + this.enqueued.push(message); + return Promise.resolve(); + } + listen(): Promise { + return Promise.resolve(); + } + } + + const inner = new NoBulkQueue(); + const workers = new ParallelMessageQueue(inner, 5); + await workers.enqueueMany([{ x: 1 }, { x: 2 }, { x: 3 }]); + assertEquals(inner.enqueued.length, 3); + }, +); + const queues: Record Promise> = { InProcessMessageQueue: () => Promise.resolve(new InProcessMessageQueue()), }; diff --git a/packages/fedify/src/federation/mq.ts b/packages/fedify/src/federation/mq.ts index 4ee43320e..c99cc63e7 100644 --- a/packages/fedify/src/federation/mq.ts +++ b/packages/fedify/src/federation/mq.ts @@ -451,6 +451,15 @@ export class ParallelMessageQueue implements MessageQueue { options?: MessageQueueEnqueueOptions, ): Promise { if (this.queue.enqueueMany == null) { + if (options?.deduplicationKey != null) { + throw new TypeError( + "Cannot enqueue a batch with a deduplicationKey: the wrapped queue " + + "does not implement enqueueMany, so ParallelMessageQueue would " + + "have to fan out to individual enqueue() calls that cannot share " + + "one deduplicationKey atomically. Wrap a queue that implements " + + "enqueueMany instead.", + ); + } const results = await Promise.allSettled( messages.map((message) => this.queue.enqueue(message, options)), ); diff --git a/packages/fedify/src/federation/tasks/enqueue.test.ts b/packages/fedify/src/federation/tasks/enqueue.test.ts index d559cccb5..ee7db8014 100644 --- a/packages/fedify/src/federation/tasks/enqueue.test.ts +++ b/packages/fedify/src/federation/tasks/enqueue.test.ts @@ -1,4 +1,5 @@ import { test } from "@fedify/fixture"; +import { configure, type LogRecord, reset } from "@logtape/logtape"; import { delay } from "es-toolkit"; import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; import { @@ -16,7 +17,11 @@ import { MemoryKvStore, } from "../kv.ts"; import { createFederation } from "../middleware.ts"; -import type { MessageQueue, MessageQueueEnqueueOptions } from "../mq.ts"; +import { + type MessageQueue, + type MessageQueueEnqueueOptions, + ParallelMessageQueue, +} from "../mq.ts"; import type { TaskMessage } from "../queue.ts"; /** @@ -779,3 +784,589 @@ test( deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); }, ); + +/** + * A {@link MessageQueue} that fails its first enqueue—single or batch—with a + * transient error, then records every later enqueue. One class covers both the + * `enqueue()` and `enqueueMany()` rollback paths; each test instantiates its own + * copy, so the one-shot `#failNext` flag never leaks between them. + */ +class FlakyQueue implements MessageQueue { + readonly nativeDeduplication = false; + #failNext = true; + readonly enqueued: TaskMessage[] = []; + readonly enqueuedMany: TaskMessage[][] = []; + + #failOnce(): boolean { + if (!this.#failNext) return false; + this.#failNext = false; + return true; + } + + enqueue(message: TaskMessage): Promise { + if (this.#failOnce()) { + return Promise.reject(new Error("transient backend failure")); + } + this.enqueued.push(message); + return Promise.resolve(); + } + + enqueueMany(messages: readonly TaskMessage[]): Promise { + if (this.#failOnce()) { + return Promise.reject(new Error("transient backend failure")); + } + this.enqueuedMany.push([...messages]); + return Promise.resolve(); + } + + listen(): Promise { + return Promise.resolve(); + } +} + +test( + "a failed enqueue rolls back its dedup marker so the retry is not dropped", + async () => { + const queue = new FlakyQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("flaky-enqueue", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + const markerKey: KvKey = [...TASK_DEDUP_PREFIX, "k"]; + + // First enqueue: the marker is claimed, then dispatch rejects. + await rejects( + () => ctx.enqueueTask(task, "first", { deduplicationKey: "k" }), + { message: /transient backend failure/ }, + ); + strictEqual(queue.enqueued.length, 0); + strictEqual(await kv.get(markerKey), undefined); + + // The retry (queue healthy again) must enqueue the task, not be dropped. + await ctx.enqueueTask(task, "first-retry", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + + // A successful retry must keep its marker so later duplicates are dropped. + ok(await kv.get(markerKey) != null); + await ctx.enqueueTask(task, "duplicate", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + }, +); + +test( + "a failed batch enqueue rolls back its dedup marker so the retry is not " + + "dropped", + async () => { + const queue = new FlakyQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("flaky-batch-enqueue", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + const markerKey: KvKey = [...TASK_DEDUP_PREFIX, "batch"]; + + await rejects( + () => + ctx.enqueueTaskMany(task, ["first", "second"], { + deduplicationKey: "batch", + }), + { message: /transient backend failure/ }, + ); + strictEqual(queue.enqueuedMany.length, 0); + // Asserted via get() for the same reason as the single-item rollback test + // above (MemoryKvStore.cas leaves a `value: undefined` entry). + strictEqual(await kv.get(markerKey), undefined); + + await ctx.enqueueTaskMany(task, ["first-retry", "second-retry"], { + deduplicationKey: "batch", + }); + strictEqual(queue.enqueuedMany.length, 1); + strictEqual(queue.enqueuedMany[0].length, 2); + ok(await kv.get(markerKey) != null); + + await ctx.enqueueTaskMany(task, ["duplicate-first", "duplicate-second"], { + deduplicationKey: "batch", + }); + strictEqual(queue.enqueuedMany.length, 1); + }, +); + +test( + "a stale rollback does not clear a marker another enqueue re-claimed", + async () => { + const kv = new MemoryKvStore(); + const markerKey: KvKey = [...TASK_DEDUP_PREFIX, "k"]; + let signalFirstEntered!: () => void; + const firstEntered = new Promise((resolve) => { + signalFirstEntered = resolve; + }); + let releaseFirst!: () => void; + const firstReleased = new Promise((resolve) => { + releaseFirst = resolve; + }); + class BlockingThenFailingQueue implements MessageQueue { + readonly nativeDeduplication = false; + #calls = 0; + async enqueue(): Promise { + this.#calls++; + if (this.#calls === 1) { + signalFirstEntered(); + await firstReleased; + throw new Error("transient backend failure"); + } + } + listen(): Promise { + return Promise.resolve(); + } + } + const queue = new BlockingThenFailingQueue(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + taskDeduplicationTtl: { milliseconds: 1 }, + }); + const task = federation.defineTask("stale-rollback", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + const first = ctx.enqueueTask(task, "first", { deduplicationKey: "k" }); + await firstEntered; + await delay(20); + await ctx.enqueueTask(task, "second", { deduplicationKey: "k" }); + const secondToken = await kv.get(markerKey); + ok(secondToken != null); + releaseFirst(); + await rejects(() => first, { message: /transient backend failure/ }); + strictEqual(await kv.get(markerKey), secondToken); + }, +); + +test( + "a multi-item batch dedup without enqueueMany is rejected on the cas path", + async () => { + const queue = new MockQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("cas-batch-without-enqueue-many", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + await rejects( + () => + ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }), + { name: "TypeError", message: /enqueueMany/ }, + ); + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + + await ctx.enqueueTaskMany(task, ["single"], { deduplicationKey: "single" }); + strictEqual(queue.enqueued.length, 1); + }, +); + +test( + "a failed rollback is swallowed; the original enqueue error reaches the caller", + async () => { + class ClearFailingKvStore implements KvStore { + readonly inner = new MemoryKvStore(); + clearAttempts = 0; + get(key: KvKey): Promise { + return this.inner.get(key); + } + set( + key: KvKey, + value: unknown, + options?: KvStoreSetOptions, + ): Promise { + return this.inner.set(key, value, options); + } + delete(key: KvKey): Promise { + return this.inner.delete(key); + } + list(prefix?: KvKey): AsyncIterable { + return this.inner.list(prefix); + } + cas( + key: KvKey, + expectedValue: unknown, + newValue: unknown, + options?: KvStoreSetOptions, + ): Promise { + if (newValue === undefined) { + this.clearAttempts++; + return Promise.reject(new Error("kv cas clear failed")); + } + return this.inner.cas(key, expectedValue, newValue, options); + } + } + + const queue = new FlakyQueue(); + const kv = new ClearFailingKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("rollback-failure", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + await rejects( + () => ctx.enqueueTask(task, "first", { deduplicationKey: "k" }), + { message: /transient backend failure/ }, + ); + strictEqual(queue.enqueued.length, 0); + strictEqual(kv.clearAttempts, 1); + }, +); + +/** + * A native-deduplication backend that drops repeat-key single enqueues and + * does **not** implement `enqueueMany`. Wrapping it in + * {@link ParallelMessageQueue} used to fan a batch out to one `enqueue()` per + * message, all carrying the same `deduplicationKey`, so the backend collapsed + * the whole batch onto its first message. + */ +class NativeDedupNoBulkQueue implements MessageQueue { + readonly nativeDeduplication = true; + readonly #seen = new Set(); + readonly enqueued: { + message: TaskMessage; + options?: MessageQueueEnqueueOptions; + }[] = []; + + enqueue( + message: TaskMessage, + options?: MessageQueueEnqueueOptions, + ): Promise { + const key = options?.deduplicationKey; + if (key != null) { + if (this.#seen.has(key)) return Promise.resolve(); + this.#seen.add(key); + } + this.enqueued.push({ message, options }); + return Promise.resolve(); + } + + listen(): Promise { + return Promise.resolve(); + } +} + +test( + "a deduplicated batch over a ParallelMessageQueue wrapping a native, " + + "no-enqueueMany backend is rejected, not collapsed", + async () => { + const backend = new NativeDedupNoBulkQueue(); + const queue = new ParallelMessageQueue(backend, 5); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("parallel-native-no-bulk", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + // The wrapper cannot enqueue the batch atomically under one key, so the + // multi-item batch must be rejected rather than silently collapsed to one. + await rejects( + () => + ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }), + { name: "TypeError", message: /enqueueMany/ }, + ); + strictEqual(backend.enqueued.length, 0); + // A native plan never touches KV, even when it rejects. + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + + // A single-item batch needs no bulk path, so the key is still forwarded. + await ctx.enqueueTaskMany(task, ["solo"], { deduplicationKey: "solo" }); + strictEqual(backend.enqueued.length, 1); + strictEqual(backend.enqueued[0].options?.deduplicationKey, "solo"); + }, +); + +test( + "a deduplicated batch over a ParallelMessageQueue wrapping a native " + + "enqueueMany backend forwards the key atomically", + async () => { + class NativeBatchQueue implements MessageQueue { + readonly nativeDeduplication = true; + readonly #seen = new Set(); + readonly batches: { + messages: readonly TaskMessage[]; + options?: MessageQueueEnqueueOptions; + }[] = []; + + enqueue(): Promise { + throw new Error("A multi-item native batch must use enqueueMany()."); + } + + enqueueMany( + messages: readonly TaskMessage[], + options?: MessageQueueEnqueueOptions, + ): Promise { + const key = options?.deduplicationKey; + if (key != null && this.#seen.has(key)) return Promise.resolve(); + if (key != null) this.#seen.add(key); + this.batches.push({ messages, options }); + return Promise.resolve(); + } + + listen(): Promise { + return Promise.resolve(); + } + } + + const backend = new NativeBatchQueue(); + const queue = new ParallelMessageQueue(backend, 5); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("parallel-native-bulk", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + await ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }); + // The duplicate batch is dropped by the backend's native check. + await ctx.enqueueTaskMany(task, ["x", "y", "z"], { + deduplicationKey: "batch", + }); + + strictEqual(backend.batches.length, 1); + strictEqual(backend.batches[0].messages.length, 3); + strictEqual(backend.batches[0].options?.deduplicationKey, "batch"); + // The native path never writes KV, even through the wrapper. + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + }, +); + +test( + 'an "open" fallback fans out a multi-item batch without enqueueMany ' + + "instead of rejecting it", + async () => { + const queue = new MockQueue(); // no enqueueMany, not native + const kv = new CaslessKvStore(); // no cas + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + taskDeduplicationFallback: "open", + }); + const task = federation.defineTask("open-batch-fanout", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + // With neither native dedup nor cas under "open", the batch proceeds by + // fanning out every item; it must not throw the enqueueMany requirement. + await ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }); + strictEqual(queue.enqueued.length, 3); + for (const { options } of queue.enqueued) { + strictEqual(options?.deduplicationKey, undefined); + } + // The open path records nothing in the key–value store. + deepStrictEqual(await collectKeys(kv.inner, TASK_DEDUP_PREFIX), []); + }, +); + +test( + 'an "open" fallback logs a debug record when it ignores the key', + async () => { + const records: LogRecord[] = []; + await reset(); + try { + await configure({ + sinks: { + buffer(record: LogRecord): void { + records.push(record); + }, + }, + filters: {}, + loggers: [ + { category: [], lowestLevel: "debug", sinks: ["buffer"] }, + ], + }); + + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new CaslessKvStore(), + queue: { task: queue }, + taskDeduplicationFallback: "open", + }); + const task = federation.defineTask("open-debug-log", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); + + const matched = records.filter((record) => + record.level === "debug" && + record.properties.deduplicationKey === "k" && + record.properties.taskName === "open-debug-log" + ); + strictEqual(matched.length, 1); + } finally { + await reset(); + } + }, +); + +test( + "two concurrent enqueues sharing a key: exactly one wins the cas claim", + async () => { + let signalEntered!: () => void; + const entered = new Promise((resolve) => { + signalEntered = resolve; + }); + let release!: () => void; + const released = new Promise((resolve) => { + release = resolve; + }); + class BlockingQueue implements MessageQueue { + readonly nativeDeduplication = false; + readonly enqueued: TaskMessage[] = []; + #first = true; + async enqueue(message: TaskMessage): Promise { + if (this.#first) { + this.#first = false; + signalEntered(); + await released; + } + this.enqueued.push(message); + } + listen(): Promise { + return Promise.resolve(); + } + } + const queue = new BlockingQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("concurrent-claim", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + // The first enqueue claims the marker, then blocks inside the queue. + const first = ctx.enqueueTask(task, "first", { deduplicationKey: "k" }); + await entered; + // With the first still in flight, the second must lose the cas claim and + // skip the queue entirely. + await ctx.enqueueTask(task, "second", { deduplicationKey: "k" }); + release(); + await first; + strictEqual(queue.enqueued.length, 1); + + // The winner kept its marker, so a later duplicate is still dropped. + await ctx.enqueueTask(task, "third", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + }, +); + +test( + "a native enqueue forwards orderingKey and deduplicationKey together", + async () => { + const queue = new MockQueue({ nativeDeduplication: true }); + const federation = createFederation({ + ...baseOptions, + kv: new MemoryKvStore(), + queue: { task: queue }, + }); + const task = federation.defineTask("native-both-keys", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { + orderingKey: "user:alice", + deduplicationKey: "dedup:alice", + }); + strictEqual(queue.enqueued.length, 1); + const { message, options } = queue.enqueued[0]; + strictEqual(message.orderingKey, "user:alice"); + strictEqual(options?.orderingKey, "user:alice"); + strictEqual(options?.deduplicationKey, "dedup:alice"); + }, +); diff --git a/packages/fedify/src/federation/tasks/enqueue.ts b/packages/fedify/src/federation/tasks/enqueue.ts index e377b9d6e..6500301af 100644 --- a/packages/fedify/src/federation/tasks/enqueue.ts +++ b/packages/fedify/src/federation/tasks/enqueue.ts @@ -94,11 +94,27 @@ const enqueueTasks = ( if (!ctx.federation.manuallyStartQueue) { ctx.federation._startQueueInternal(ctx.data); } - await dispatch(queue, messages, { - delay: getDurationIfDefined(options.delay), - orderingKey: options.orderingKey, - deduplicationKey: claim.forwardedDeduplicationKey, - }); + try { + await dispatch(queue, messages, { + delay: getDurationIfDefined(options.delay), + orderingKey: options.orderingKey, + deduplicationKey: claim.forwardedDeduplicationKey, + }); + } catch (error) { + if (claim.rollback != null) { + try { + await claim.rollback(); + } catch (rollbackError) { + logger.warn( + "Failed to roll back the deduplication marker for task " + + "{taskName} after a failed enqueue; it will expire by TTL. " + + "{rollbackError}", + { taskName: task.name, rollbackError }, + ); + } + } + throw error; + } }; export default enqueueTasks; @@ -131,21 +147,20 @@ function planDeduplication( ): DedupPlan { if (options.deduplicationKey == null) return { kind: "none" }; const key = options.deduplicationKey; - if (queue.nativeDeduplication === true) { - if (itemCount > 1 && queue.enqueueMany == null) { - throw new TypeError( - `Task ${ - JSON.stringify(taskName) - } was enqueued as a batch with a deduplicationKey, but its ` + - "message queue declares nativeDeduplication without " + - "implementing enqueueMany; a per-message key cannot deduplicate " + - "a whole batch. Implement enqueueMany on the queue, or enqueue " + - "the tasks individually with enqueueTask().", - ); - } - return { kind: "native", key }; + const native = queue.nativeDeduplication === true; + const canCas = ctx.federation.kv.cas != null; + if (itemCount > 1 && queue.enqueueMany == null && (native || canCas)) { + throw new TypeError( + `Task ${ + JSON.stringify(taskName) + } was enqueued as a batch with a deduplicationKey, but its message ` + + "queue does not implement enqueueMany; a multi-item batch cannot be " + + "deduplicated atomically without it. Implement enqueueMany on the " + + "queue, or enqueue the tasks individually with enqueueTask().", + ); } - if (ctx.federation.kv.cas != null) return { kind: "cas", key }; + if (native) return { kind: "native", key }; + if (canCas) return { kind: "cas", key }; if (ctx.federation.taskDeduplicationFallback === "closed") { // No conditional write, closed: fail fast before any side effect. throw new TypeError( @@ -169,7 +184,16 @@ async function claimDeduplication( ctx: EnqueueTasksContext, plan: DedupPlan, taskName: string, -): Promise<{ proceed: boolean; forwardedDeduplicationKey?: string }> { +): Promise<{ + proceed: boolean; + forwardedDeduplicationKey?: string; + /** + * Undoes a reserved marker when dispatch fails. Present only for a `cas` + * plan that won its claim; a failed enqueue calls it so the retry is not + * deduplicated against a task that never reached the queue. + */ + rollback?: () => Promise; +}> { switch (plan.kind) { case "native": return { proceed: true, forwardedDeduplicationKey: plan.key }; @@ -178,23 +202,37 @@ async function claimDeduplication( ...ctx.federation.kvPrefixes.taskDeduplication, plan.key, ] satisfies KvKey; - // planDeduplication only picks "cas" when `ctx.federation.kv.cas` exists. - const won = await ctx.federation.kv.cas!(cacheKey, undefined, true, { + const token = crypto.randomUUID(); + const won = await ctx.federation.kv.cas!(cacheKey, undefined, token, { ttl: ctx.federation.taskDeduplicationTtl, }); - return { proceed: won }; + if (!won) return { proceed: false }; + return { + proceed: true, + // Conditional clear: cas succeeds only while the stored value is still + // our token, so we never delete a marker another enqueue now owns. + rollback: async () => { + await ctx.federation.kv.cas!(cacheKey, token, undefined); + }, + }; } - case "open": { - getLogger(["fedify", "federation", "task"]).debug( + case "open": + logger.debug( "deduplicationKey {deduplicationKey} for task {taskName} ignored: " + "the message queue declares no nativeDeduplication and the " + "key-value store has no cas; proceeding (taskDeduplicationFallback " + 'is "open").', { deduplicationKey: plan.key, taskName }, + ); /* falls through */ + case "none": + return { proceed: true }; + default: { + const _exhaustive: never = plan; + throw new TypeError( + `Unknown deduplication plan: ${JSON.stringify(_exhaustive)}`, ); } } - return { proceed: true }; } /** @@ -252,3 +290,5 @@ async (data: TData): Promise => { traceContext: carrier, }; }; + +const logger = getLogger(["fedify", "federation", "task"]); diff --git a/packages/fedify/src/federation/tasks/task.ts b/packages/fedify/src/federation/tasks/task.ts index 3731f82ca..a095e713d 100644 --- a/packages/fedify/src/federation/tasks/task.ts +++ b/packages/fedify/src/federation/tasks/task.ts @@ -168,9 +168,13 @@ export interface TaskEnqueueOptions { * and {@link FederationOptions.taskDeduplicationFallback} decides whether a * missing `cas` proceeds without deduplication or throws. * - * For {@link Context.enqueueTaskMany}, one key governs the whole batch; a - * native queue must implement {@link MessageQueue.enqueueMany} for a - * multi-item batch, or the call throws a `TypeError`. + * For {@link Context.enqueueTaskMany}, one key governs the whole batch. When + * deduplication is actually applied—a native queue, or the key–value + * fallback through {@link KvStore.cas}—a multi-item batch with a + * `deduplicationKey` requires the queue to implement + * {@link MessageQueue.enqueueMany} so it enqueues atomically, or the call + * throws a `TypeError`. Under the `"open"` fallback with no `cas`, no marker + * is taken, so such a batch instead fans out without deduplication. * * @since 2.x.x */ diff --git a/packages/fedify/src/testing/tasks.ts b/packages/fedify/src/testing/tasks.ts index d32e24217..7ccf727fc 100644 --- a/packages/fedify/src/testing/tasks.ts +++ b/packages/fedify/src/testing/tasks.ts @@ -1,5 +1,5 @@ /** - * Test-only utilities shared by the task suites the schema factory and stock + * Test-only utilities shared by the task suites: the schema factory and stock * schemas, the base federation options, and the recording {@link MockQueue}. * * These helpers live beside the suites that use them rather than in a shared From afdaa6136c8da9099f5faffd58a536f3e9d5c9e2 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Mon, 22 Jun 2026 11:14:23 +0000 Subject: [PATCH 34/57] Fix and simplify task queue worker startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit startQueue({ queue: "task" }) started no worker when the effective task queue was the outbox fallback (no dedicated task queue) or an instance shared with inbox/outbox/fanout: the standard-queue branches were gated on their own selector, and the task branch skipped a queue that equalled a standard one. Enqueued tasks then had no listener. Rather than patch the branch conditions, _startQueueInternal now lists every (role, queue) target—including the outbox fallback and each per-task queue—and starts each instance at most once through a single identity Set, which also subsumes the four per-role started flags and the dedicated-task-queue Set into one field. https://github.com/fedify-dev/fedify/pull/806#discussion_r3418490072 https://github.com/fedify-dev/fedify/pull/806#discussion_r3418490090 Assisted-by: Claude Code:claude-opus-4-8 --- packages/fedify/src/federation/middleware.ts | 130 ++++-------------- .../fedify/src/federation/tasks/tasks.test.ts | 52 +++++++ 2 files changed, 77 insertions(+), 105 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 5a06fb12e..dfd40361c 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -602,11 +602,7 @@ export class FederationImpl outboxQueue?: MessageQueue; fanoutQueue?: MessageQueue; taskQueue?: MessageQueue; - inboxQueueStarted: boolean; - outboxQueueStarted: boolean; - fanoutQueueStarted: boolean; - taskQueueStarted: boolean; - startedTaskQueues: Set; + startedQueues: Set; manuallyStartQueue: boolean; origin?: FederationOrigin; documentLoaderFactory: DocumentLoaderFactory; @@ -733,11 +729,7 @@ export class FederationImpl ); } } - this.inboxQueueStarted = false; - this.outboxQueueStarted = false; - this.fanoutQueueStarted = false; - this.taskQueueStarted = false; - this.startedTaskQueues = new Set(); + this.startedQueues = new Set(); this.manuallyStartQueue = options.manuallyStartQueue ?? false; if (options.origin != null) { if (typeof options.origin === "string") { @@ -988,109 +980,37 @@ export class FederationImpl signal?: AbortSignal, queue?: keyof FederationQueueOptions, ): Promise { - // Tasks may route to a dedicated queue of their own (defineTask({ queue })) - // even when no federation-wide queue is configured, so a deployment with - // only per-task queues still has work to start. - const hasDedicatedTaskQueue = [...this.taskDefinitions.values()].some( - (def) => def.queue != null, - ); - if ( - this.inboxQueue == null && this.outboxQueue == null && - this.fanoutQueue == null && this.taskQueue == null && - !hasDedicatedTaskQueue - ) { - return; - } + // Tasks fall back to the outbox queue and may add per-task queues; the + // identity Set then starts each instance once even when roles share one. + type QueueNameMessage = [ + keyof FederationQueueOptions, + MessageQueue | undefined, + ]; + const taskQueue = this.taskQueue ?? + (this.taskQueueResolution === "fallback" ? this.outboxQueue : undefined); + const customQueues = this.taskDefinitions.values() + .map((def): QueueNameMessage => ["task", def.queue]); + const targets: QueueNameMessage[] = [ + ["inbox", this.inboxQueue], + ["outbox", this.outboxQueue], + ["fanout", this.fanoutQueue], + ["task", taskQueue], + ...customQueues, + ]; const logger = getLogger(["fedify", "federation", "queue"]); const promises: Promise[] = []; - if ( - this.inboxQueue != null && (queue == null || queue === "inbox") && - !this.inboxQueueStarted - ) { - logger.debug("Starting an inbox task worker."); - this.inboxQueueStarted = true; - promises.push( - this.inboxQueue.listen( - (msg) => this.processQueuedTask(ctxData, msg), - { signal }, - ), - ); - } - if ( - this.outboxQueue != null && - this.outboxQueue !== this.inboxQueue && - (queue == null || queue === "outbox") && - !this.outboxQueueStarted - ) { - logger.debug("Starting an outbox task worker."); - this.outboxQueueStarted = true; - promises.push( - this.outboxQueue.listen( - (msg) => this.processQueuedTask(ctxData, msg), - { signal }, - ), - ); - } - if ( - this.fanoutQueue != null && - this.fanoutQueue !== this.inboxQueue && - this.fanoutQueue !== this.outboxQueue && - (queue == null || queue === "fanout") && - !this.fanoutQueueStarted - ) { - logger.debug("Starting a fanout task worker."); - this.fanoutQueueStarted = true; + for (const [role, target] of targets) { + if (target == null || !(queue == null || queue === role)) continue; + if (this.startedQueues.has(target)) continue; + this.startedQueues.add(target); + logger.debug("Starting a {role} queue worker.", { role }); promises.push( - this.fanoutQueue.listen( + target.listen( (msg) => this.processQueuedTask(ctxData, msg), { signal }, ), ); } - if ( - this.taskQueue != null && - this.taskQueue !== this.inboxQueue && - this.taskQueue !== this.outboxQueue && - this.taskQueue !== this.fanoutQueue && - (queue == null || queue === "task") && - !this.taskQueueStarted - ) { - logger.debug("Starting a task worker."); - this.taskQueueStarted = true; - promises.push( - this.taskQueue.listen( - (msg) => this.processQueuedTask(ctxData, msg), - { signal }, - ), - ); - } - // Dedicated per-task queues belong to the "task" selector. Each distinct - // instance needs its own worker; dedupe against the standard queues and - // against task queues already started on an earlier call so no instance is - // listened on twice. - if (queue == null || queue === "task") { - const standardQueues = new Set( - [this.inboxQueue, this.outboxQueue, this.fanoutQueue, this.taskQueue] - .filter((q): q is MessageQueue => q != null), - ); - for (const def of this.taskDefinitions.values()) { - const taskQueue = def.queue; - if ( - taskQueue == null || standardQueues.has(taskQueue) || - this.startedTaskQueues.has(taskQueue) - ) { - continue; - } - logger.debug("Starting a worker for a dedicated per-task queue."); - this.startedTaskQueues.add(taskQueue); - promises.push( - taskQueue.listen( - (msg) => this.processQueuedTask(ctxData, msg), - { signal }, - ), - ); - } - } await Promise.all(promises); } diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index 8b4983f79..84041c5aa 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -443,6 +443,58 @@ test("startQueue() task worker", async (t) => { strictEqual(received, "payload"); }, ); + + await t.step( + 'queue: "task" starts the outbox fallback when no task queue is set', + async () => { + const inbox = new MockQueue(); + const outbox = new MockQueue(); + const fanout = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { inbox, outbox, fanout }, + }); + federation.defineTask("fallback", { + schema: stringSchema, + handler: () => {}, + }); + const controller = new AbortController(); + const listening = federation.startQueue(undefined, { + signal: controller.signal, + queue: "task", + }); + // Tasks route to the outbox fallback, so the "task" selector must start + // the outbox worker—otherwise enqueued tasks have no listener. + strictEqual(outbox.listenCount, 1); + strictEqual(inbox.listenCount, 0); + strictEqual(fanout.listenCount, 0); + controller.abort(); + await listening; + }, + ); + + await t.step( + 'queue: "task" starts a task queue shared with the outbox once', + async () => { + const inbox = new MockQueue(); + const shared = new MockQueue(); + const fanout = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + queue: { inbox, outbox: shared, fanout, task: shared }, + }); + const controller = new AbortController(); + const listening = federation.startQueue(undefined, { + signal: controller.signal, + queue: "task", + }); + strictEqual(shared.listenCount, 1); + strictEqual(inbox.listenCount, 0); + strictEqual(fanout.listenCount, 0); + controller.abort(); + await listening; + }, + ); }); test("processQueuedTask() task dispatch", async (t) => { From 01c3fabcb548d7406cf181589b00a83ffd5807ee Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Mon, 22 Jun 2026 11:22:28 +0000 Subject: [PATCH 35/57] Resolve MockQueue.listen() on a pre-aborted signal MockQueue.listen() only resolved from an "abort" event listener, so a signal already aborted before the call left the promise pending forever and could hang queue-listener teardown in tests. Return a resolved promise when the signal is already aborted, and register the listener with { once: true }. https://github.com/fedify-dev/fedify/pull/806#discussion_r3418490094 Assisted-by: Claude Code:claude-opus-4-8 --- .../fedify/src/federation/tasks/tasks.test.ts | 17 +++++++++++++++++ packages/fedify/src/testing/tasks.ts | 5 ++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index 84041c5aa..c42c7bb94 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -497,6 +497,23 @@ test("startQueue() task worker", async (t) => { ); }); +test("MockQueue.listen() resolves on a pre-aborted signal", async () => { + const queue = new MockQueue(); + const controller = new AbortController(); + controller.abort(); + let timer: ReturnType | undefined; + const settled = await Promise.race([ + queue.listen(() => {}, { signal: controller.signal }).then(() => + "resolved" + ), + new Promise((resolve) => { + timer = setTimeout(() => resolve("pending"), 50); + }), + ]); + clearTimeout(timer); + strictEqual(settled, "resolved"); +}); + test("processQueuedTask() task dispatch", async (t) => { await t.step("drops an unknown task with a warning", async () => { const queue = new MockQueue(); diff --git a/packages/fedify/src/testing/tasks.ts b/packages/fedify/src/testing/tasks.ts index 7ccf727fc..134281fbf 100644 --- a/packages/fedify/src/testing/tasks.ts +++ b/packages/fedify/src/testing/tasks.ts @@ -129,8 +129,11 @@ export class MockQueue implements MessageQueue { options?: MessageQueueListenOptions, ): Promise { this.listenCount++; + if (options?.signal?.aborted) return Promise.resolve(); return new Promise((resolve) => { - options?.signal?.addEventListener("abort", () => resolve()); + options?.signal?.addEventListener("abort", () => resolve(), { + once: true, + }); }); } } From ed64469e92dfd59440539142251add5608751057 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Mon, 22 Jun 2026 13:28:20 +0000 Subject: [PATCH 36/57] Fall back to enqueue() for single deduplicated batch ParallelMessageQueue.enqueueMany threw whenever the wrapped queue lacked enqueueMany and a deduplicationKey was set, regardless of how many messages the batch held. The atomicity limitation it guards against only applies to multi-item fan-out: a single-item batch can carry its deduplicationKey straight to the wrapped queue's enqueue() without splitting it across calls. Narrow the guard to multi-item batches and fall back to enqueue() for a single item, so direct ParallelMessageQueue callers are no longer rejected for a case that is perfectly safe. The task enqueue path already routes single items through enqueue() before reaching this branch, so its behavior is unchanged. https://github.com/fedify-dev/fedify/pull/806#discussion_r3418490075 Assisted-by: Claude Code:claude-opus-4-8 --- packages/fedify/src/federation/mq.test.ts | 31 +++++++++++++++++++++++ packages/fedify/src/federation/mq.ts | 4 +++ 2 files changed, 35 insertions(+) diff --git a/packages/fedify/src/federation/mq.test.ts b/packages/fedify/src/federation/mq.test.ts index 3bb5c0d88..7c6497616 100644 --- a/packages/fedify/src/federation/mq.test.ts +++ b/packages/fedify/src/federation/mq.test.ts @@ -504,6 +504,37 @@ test( }, ); +test( + "ParallelMessageQueue falls back to enqueue() for a single deduplicated " + + "item when the wrapped queue lacks enqueueMany", + async () => { + class NoBulkQueue implements MessageQueue { + readonly nativeDeduplication = true; + readonly enqueued: unknown[] = []; + readonly options: (MessageQueueEnqueueOptions | undefined)[] = []; + enqueue( + message: unknown, + options?: MessageQueueEnqueueOptions, + ): Promise { + this.enqueued.push(message); + this.options.push(options); + return Promise.resolve(); + } + listen(): Promise { + return Promise.resolve(); + } + } + + const inner = new NoBulkQueue(); + const workers = new ParallelMessageQueue(inner, 5); + // The atomicity limitation only applies to multi-item fan-out, so a + // single-item batch forwards its deduplicationKey to enqueue() unchanged. + await workers.enqueueMany([{ x: 1 }], { deduplicationKey: "k" }); + assertEquals(inner.enqueued, [{ x: 1 }]); + assertEquals(inner.options[0]?.deduplicationKey, "k"); + }, +); + test( "ParallelMessageQueue still fans out a non-deduplicated batch when the " + "wrapped queue lacks enqueueMany", diff --git a/packages/fedify/src/federation/mq.ts b/packages/fedify/src/federation/mq.ts index c99cc63e7..5fb33ea0c 100644 --- a/packages/fedify/src/federation/mq.ts +++ b/packages/fedify/src/federation/mq.ts @@ -451,6 +451,10 @@ export class ParallelMessageQueue implements MessageQueue { options?: MessageQueueEnqueueOptions, ): Promise { if (this.queue.enqueueMany == null) { + if (messages.length === 1) { + await this.queue.enqueue(messages[0], options); + return; + } if (options?.deduplicationKey != null) { throw new TypeError( "Cannot enqueue a batch with a deduplicationKey: the wrapped queue " + From 5319c1e4af0d2ddd237bcf23c602365f4f318d69 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Mon, 22 Jun 2026 13:29:43 +0000 Subject: [PATCH 37/57] Give each task test its own MemoryKvStore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The task suites in middleware.test.ts spread a module-scope taskFederationOptions object that held a single shared MemoryKvStore, so every createFederation() call reused the same key-value store. As more task cases are added, a deduplication marker written by one test could leak into another, making the suite order-dependent and flaky. Replace the shared constant with a mockOptions() factory that returns fresh options—including a fresh MemoryKvStore—on every call, and spread mockOptions() at each use site. https://github.com/fedify-dev/fedify/pull/806#discussion_r3418490056 Assisted-by: Claude Code:claude-opus-4-8 --- .../fedify/src/federation/middleware.test.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/fedify/src/federation/middleware.test.ts b/packages/fedify/src/federation/middleware.test.ts index 40b67f189..6ff54ca91 100644 --- a/packages/fedify/src/federation/middleware.test.ts +++ b/packages/fedify/src/federation/middleware.test.ts @@ -10565,12 +10565,15 @@ const withTimeout = ( return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); }; -const taskFederationOptions = { +// A factory, not a shared constant: each task test gets its own +// MemoryKvStore so deduplication markers never leak across tests and the +// suite stays order-independent as more cases are added. +const mockOptions = () => ({ kv: new MemoryKvStore(), documentLoaderFactory: () => mockDocumentLoader, contextLoaderFactory: () => mockDocumentLoader, manuallyStartQueue: true, -}; +}); test("ContextImpl.enqueueTask()", async (t) => { await t.step( @@ -10578,7 +10581,7 @@ test("ContextImpl.enqueueTask()", async (t) => { async () => { const queue = new MockQueue({ supportsEnqueueMany: true }); const federation = createFederation({ - ...taskFederationOptions, + ...mockOptions(), queue: { task: queue }, }); const task = federation.defineTask("greet", { @@ -10619,7 +10622,7 @@ test("ContextImpl.enqueueTaskMany()", async (t) => { async () => { const queue = new MockQueue({ supportsEnqueueMany: true }); const federation = createFederation({ - ...taskFederationOptions, + ...mockOptions(), queue: { task: queue }, }); const task = federation.defineTask("bulk", { @@ -10652,7 +10655,7 @@ test("ContextImpl.enqueueTaskMany()", async (t) => { async () => { const queue = new MockQueue({ supportsEnqueueMany: true }); const federation = createFederation({ - ...taskFederationOptions, + ...mockOptions(), queue: { task: queue }, }); const task = federation.defineTask("bulk-single", { @@ -10674,7 +10677,7 @@ test("ContextImpl.enqueueTaskMany()", async (t) => { async () => { const queue = new RendezvousQueue(2); const federation = createFederation({ - ...taskFederationOptions, + ...mockOptions(), queue: { task: queue }, }); const task = federation.defineTask("bulk-fallback", { @@ -10716,7 +10719,7 @@ test("ContextImpl.enqueueTaskMany()", async (t) => { async () => { const queue = new MockQueue(); const federation = createFederation({ - ...taskFederationOptions, + ...mockOptions(), queue: { task: queue }, }); const task = federation.defineTask("bulk-typed", { From ae0d9c76cd453be3f98d0fe875568fe8f82867e0 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Mon, 22 Jun 2026 14:09:31 +0000 Subject: [PATCH 38/57] Clarify task docs and add missing JSDoc Address review feedback on the custom background task API: - The enqueueTaskMany JSDoc makes its fallback wording conditional and now documents the TypeError thrown when a deduplicated multi-item batch cannot be enqueued atomically because the queue does not implement bulk enqueue. - TaskCodec is marked @internal, and serialize(), deserialize(), and the static validate() helper each gain a one-line JSDoc. - The deduplication section of the manual lists Cloudflare Workers KV among the key-value stores that do not yet implement cas. https://github.com/fedify-dev/fedify/pull/806 Assisted-by: Claude Code:claude-opus-4-8 --- docs/manual/tasks.md | 8 +++++--- packages/fedify/src/federation/context.ts | 12 ++++++++---- packages/fedify/src/federation/tasks/codec.ts | 9 +++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index 9e3863f22..3f51cdfe7 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -1,3 +1,5 @@ + + Background tasks ================ @@ -310,9 +312,9 @@ How the key is resolved depends on the queue and the key–value store: ~~~~ Among the first-party adapters, the in-memory, Deno KV, SQLite, and MySQL -key–value stores implement `~KvStore.cas`; PostgreSQL and Redis do not yet, so -those deployments take the `taskDeduplicationFallback` branch until per-adapter -follow-ups add it. +key–value stores implement `~KvStore.cas`; PostgreSQL, Redis, and +Cloudflare Workers KV do not yet, so those deployments take the +`taskDeduplicationFallback` branch until per-adapter follow-ups add it. For `~Context.enqueueTaskMany()`, a single `deduplicationKey` applies to the whole batch: the batch enqueues as a unit or is skipped as a unit, never diff --git a/packages/fedify/src/federation/context.ts b/packages/fedify/src/federation/context.ts index fda194530..c381e7d06 100644 --- a/packages/fedify/src/federation/context.ts +++ b/packages/fedify/src/federation/context.ts @@ -459,8 +459,9 @@ export interface Context { /** * Enqueues multiple payloads for a custom background task at once. - * Uses the queue's bulk enqueue operation when available, falling back - * to parallel single enqueues. + * Uses the queue's bulk enqueue operation when available. Without + * deduplication, it may fall back to parallel single enqueues when the + * queue does not implement bulk enqueue. * @template TData The type of the task payload, inferred from the task's * schema. * @param task The handle returned by {@link TaskRegistry.defineTask}. @@ -468,8 +469,11 @@ export interface Context { * task's schema before being enqueued. * @param options Options for enqueuing the tasks. * @throws {TypeError} If the task is not defined on this federation, - * if no message queue is configured for tasks, or if - * a payload fails schema validation. + * if no message queue is configured for tasks, if + * a payload fails schema validation, or if a + * deduplicated multi-item batch cannot be enqueued + * atomically because the queue does not implement + * bulk enqueue. * @since 2.x.x */ enqueueTaskMany( diff --git a/packages/fedify/src/federation/tasks/codec.ts b/packages/fedify/src/federation/tasks/codec.ts index f13d85581..e649ce061 100644 --- a/packages/fedify/src/federation/tasks/codec.ts +++ b/packages/fedify/src/federation/tasks/codec.ts @@ -4,12 +4,20 @@ import type { TracerProvider } from "@opentelemetry/api"; import type { StandardSchemaV1 } from "@standard-schema/spec"; import { parse, stringifyAsync } from "devalue"; +/** + * Serializes and deserializes task payloads for the queue, preserving + * `@fedify/vocab` objects across the wire by reducing them to JSON-LD and + * rebuilding them on the worker with the bound {@link TaskCodecLoaders}. + * @internal + */ export default class TaskCodec { constructor(readonly options: TaskCodecLoaders) {} + /** Serializes `data`, encoding any vocabulary object as its JSON-LD. */ serialize = (data: unknown): Promise => stringifyAsync(data, { Vocab: this.#stringifyVocab }); + /** Deserializes `raw`, rebuilding any encoded vocabulary object. */ deserialize = (raw: string): Promise => this.#revive(new Map())(parse(raw, { Vocab: VocabHolder.from })); @@ -26,6 +34,7 @@ export default class TaskCodec { ): Promise> => TaskCodec.validate(schema, await this.deserialize(raw)); + /** Validates `data` against `schema`, returning its parsed output. */ static validate = async ( schema: S, data: unknown, From b100954814ecf0e27b4e4ba7c52537f657003f23 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Sat, 27 Jun 2026 04:31:26 +0000 Subject: [PATCH 39/57] Finalized `@since` tag related to custom tasks from 2.x.x to 2.4.0 --- docs/manual/tasks.md | 2 +- packages/fedify/src/federation/context.ts | 4 ++-- packages/fedify/src/federation/federation.ts | 10 +++++----- packages/fedify/src/federation/middleware.ts | 4 ++-- packages/fedify/src/federation/mq.ts | 8 ++++---- packages/fedify/src/federation/queue.ts | 2 +- packages/fedify/src/federation/tasks/task.ts | 12 ++++++------ 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index 3f51cdfe7..db7a45994 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -3,7 +3,7 @@ Background tasks ================ -*This API is available since Fedify 2.x.x.* +*This API is available since Fedify 2.4.0.* Fedify already processes outgoing and incoming activities on background workers through its [message queue](./mq.md). The custom background task API diff --git a/packages/fedify/src/federation/context.ts b/packages/fedify/src/federation/context.ts index c381e7d06..99d3eb2a5 100644 --- a/packages/fedify/src/federation/context.ts +++ b/packages/fedify/src/federation/context.ts @@ -449,7 +449,7 @@ export interface Context { * @throws {TypeError} If the task is not defined on this federation, * if no message queue is configured for tasks, or if * the payload fails schema validation. - * @since 2.x.x + * @since 2.4.0 */ enqueueTask( task: TaskDefinition, @@ -474,7 +474,7 @@ export interface Context { * deduplicated multi-item batch cannot be enqueued * atomically because the queue does not implement * bulk enqueue. - * @since 2.x.x + * @since 2.4.0 */ enqueueTaskMany( task: TaskDefinition, diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index 5b4177f80..1b6072e2d 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -13,7 +13,6 @@ import type { import type { MeterProvider, TracerProvider } from "@opentelemetry/api"; import type { ActivityTransformer } from "../compat/types.ts"; import type { HttpMessageSignaturesSpec } from "../sig/http.ts"; -import type { CircuitBreakerOptions } from "./circuit-breaker.ts"; import type { ActorAliasMapper, ActorDispatcher, @@ -39,6 +38,7 @@ import type { UnverifiedActivityHandler, WebFingerLinksDispatcher, } from "./callback.ts"; +import type { CircuitBreakerOptions } from "./circuit-breaker.ts"; import type { Context, InboxContext, RequestContext } from "./context.ts"; import type { KvStore } from "./kv.ts"; import type { @@ -1086,7 +1086,7 @@ export interface FederationOptions { * this uses an exponential backoff strategy with a maximum of 10 attempts * and a maximum delay of 12 hours. A per-task retry policy * ({@link TaskDefinitionOptions.retryPolicy}) overrides this. - * @since 2.x.x + * @since 2.4.0 */ taskRetryPolicy?: RetryPolicy; @@ -1099,7 +1099,7 @@ export interface FederationOptions { * - `"strict"`: no fallback; enqueuing the task throws instead of * silently sharing the outbox queue. * @default `"fallback"` - * @since 2.x.x + * @since 2.4.0 */ taskQueueResolution?: "fallback" | "strict"; @@ -1110,7 +1110,7 @@ export interface FederationOptions { * enqueue again. Ignored when the task's queue declares * {@link MessageQueue.nativeDeduplication} (the backend owns the window). * @default `{ hours: 1 }` - * @since 2.x.x + * @since 2.4.0 */ taskDeduplicationTtl?: Temporal.DurationLike; @@ -1125,7 +1125,7 @@ export interface FederationOptions { * - `"closed"`: rejects with a `TypeError` before enqueuing. * * @default `"open"` - * @since 2.x.x + * @since 2.4.0 */ taskDeduplicationFallback?: "open" | "closed"; diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index dfd40361c..0f93c900c 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -496,7 +496,7 @@ export interface FederationQueueOptions { * The message queue for custom background tasks. If not provided, * tasks are routed to the outbox queue (unless * {@link FederationOptions.taskQueueResolution} is `"strict"`). - * @since 2.x.x + * @since 2.4.0 */ readonly task?: MessageQueue; } @@ -554,7 +554,7 @@ export interface FederationKvPrefixes { * markers. Kept separate from {@link activityIdempotence} so the two key * spaces never collide. * @default `["_fedify", "taskDeduplication"]` - * @since 2.x.x + * @since 2.4.0 */ readonly taskDeduplication: KvKey; } diff --git a/packages/fedify/src/federation/mq.ts b/packages/fedify/src/federation/mq.ts index 5fb33ea0c..d725f0567 100644 --- a/packages/fedify/src/federation/mq.ts +++ b/packages/fedify/src/federation/mq.ts @@ -36,7 +36,7 @@ export interface MessageQueueEnqueueOptions { * best-effort deduplication before reaching them on the paths that support * it. * - * @since 2.x.x + * @since 2.4.0 */ readonly deduplicationKey?: string; } @@ -109,7 +109,7 @@ export interface MessageQueue { * key–value deduplication on the paths that request it. * * @default `false` - * @since 2.x.x + * @since 2.4.0 */ readonly nativeDeduplication?: boolean; @@ -204,7 +204,7 @@ export class InProcessMessageQueue implements MessageQueue { /** * In-process message queue does not deduplicate messages natively. - * @since 2.x.x + * @since 2.4.0 */ readonly nativeDeduplication = false; @@ -400,7 +400,7 @@ export class ParallelMessageQueue implements MessageQueue { /** * Inherits the native deduplication capability from the wrapped queue. - * @since 2.x.x + * @since 2.4.0 */ readonly nativeDeduplication?: boolean; readonly getDepth?: () => Promise; diff --git a/packages/fedify/src/federation/queue.ts b/packages/fedify/src/federation/queue.ts index a8e862a1e..8e2cac0bd 100644 --- a/packages/fedify/src/federation/queue.ts +++ b/packages/fedify/src/federation/queue.ts @@ -79,7 +79,7 @@ export interface OutboxMessage { * A message that carries a custom background task. Every field is * a string, number, or plain record so that the message survives both * JSON serialization and structured clone on every queue backend. - * @since 2.x.x + * @since 2.4.0 */ export interface TaskMessage { readonly type: "task"; diff --git a/packages/fedify/src/federation/tasks/task.ts b/packages/fedify/src/federation/tasks/task.ts index a095e713d..4a310c466 100644 --- a/packages/fedify/src/federation/tasks/task.ts +++ b/packages/fedify/src/federation/tasks/task.ts @@ -10,7 +10,7 @@ import type { RetryPolicy } from "../retry.ts"; * schema. * @param ctx The context for the worker processing the task. * @param data The decoded and validated task payload. - * @since 2.x.x + * @since 2.4.0 */ export type TaskHandler = ( ctx: Context, @@ -22,7 +22,7 @@ export type TaskHandler = ( * @template TContextData The context data to pass to the {@link Context}. * @template TSchema The [Standard Schema](https://standardschema.dev/) that * validates the task payload. - * @since 2.x.x + * @since 2.4.0 */ export interface TaskDefinitionOptions< TContextData, @@ -91,7 +91,7 @@ declare const contextDataBrand: unique symbol; * @template TContextData The context data to pass to the {@link Context}. * @template TData The type of the task payload, inferred from the task's * schema. - * @since 2.x.x + * @since 2.4.0 */ export interface TaskDefinition { /** @@ -115,7 +115,7 @@ export interface TaskDefinition { * Registration of custom background tasks. Both {@link Federation} and * {@link FederationBuilder} implement this interface. * @template TContextData The context data to pass to the {@link Context}. - * @since 2.x.x + * @since 2.4.0 */ export interface TaskRegistry { /** @@ -146,7 +146,7 @@ export interface TaskRegistry { /** * Options for {@link Context.enqueueTask} and {@link Context.enqueueTaskMany}. - * @since 2.x.x + * @since 2.4.0 */ export interface TaskEnqueueOptions { /** @@ -176,7 +176,7 @@ export interface TaskEnqueueOptions { * throws a `TypeError`. Under the `"open"` fallback with no `cas`, no marker * is taken, so such a batch instead fans out without deduplication. * - * @since 2.x.x + * @since 2.4.0 */ readonly deduplicationKey?: string; } From ffd842a00db62cd7e25354b6f913053184b9e9f1 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Wed, 1 Jul 2026 09:39:19 +0000 Subject: [PATCH 40/57] Soften emphasis and fix jsr-ref tildes in tasks.md The deduplication section used bold for label and warning phrases where italics carry enough emphasis, and applied the markdown-it-jsr-ref `~` prefix to standalone type names. The `~` prefix is only for member references such as `~KvStore.cas` that hide the owner type; a standalone type name should not use it. Lower the four labels/warnings to italics and drop the `~` from the two standalone `KvStore` and `ParallelMessageQueue` references, leaving member references untouched. https://github.com/fedify-dev/fedify/pull/806#discussion_r3498191335 https://github.com/fedify-dev/fedify/pull/806#discussion_r3498191343 https://github.com/fedify-dev/fedify/pull/806#discussion_r3498191347 https://github.com/fedify-dev/fedify/pull/806#discussion_r3498191353 https://github.com/fedify-dev/fedify/pull/806#discussion_r3498206740 https://github.com/fedify-dev/fedify/pull/806#discussion_r3498206818 Assisted-by: Claude Code:claude-opus-4-8 --- docs/manual/tasks.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index db7a45994..856b7fe60 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -280,12 +280,12 @@ await ctx.enqueueTask(sendDigest, payload, { How the key is resolved depends on the queue and the key–value store: -1. **Native backend.** When the task's queue declares +1. *Native backend.* When the task's queue declares `~MessageQueue.nativeDeduplication`, Fedify forwards the key in the message queue's `~MessageQueueEnqueueOptions.deduplicationKey` and the backend owns the check. Fedify does not touch the key–value store. -2. **Key–value fallback.** Otherwise, if the configured `~KvStore` exposes +2. *Key–value fallback.* Otherwise, if the configured `KvStore` exposes the optional compare-and-swap (`~KvStore.cas`) primitive, Fedify records the key under a dedicated `taskDeduplication` prefix with a TTL and skips the enqueue while a marker is present. The TTL defaults to one hour and is @@ -298,7 +298,7 @@ How the key is resolved depends on the queue and the key–value store: }); ~~~~ -3. **No conditional write.** When neither applies—no native deduplication and +3. *No conditional write.* When neither applies—no native deduplication and a key–value store without `~KvStore.cas`—the behavior is governed by `~FederationOptions.taskDeduplicationFallback`. `"open"` (the default) lets the enqueue proceed without deduplication after a debug-level log; @@ -331,13 +331,13 @@ key–value store with `~KvStore.cas`—Fedify rejects a multi-item batch with a risking duplicates. Under the `"open"` fallback (no native deduplication and no `cas`), no marker is taken, so the batch simply fans out without deduplication. -This applies through `~ParallelMessageQueue` as well: wrapping a queue that +This applies through `ParallelMessageQueue` as well: wrapping a queue that lacks `~MessageQueue.enqueueMany()` does not make batch enqueue atomic, so a deduplicated multi-item batch on such a wrapper is likewise rejected rather than collapsed onto one message. > [!WARNING] -> The key–value fallback is **best-effort, not transactional**. The marker +> The key–value fallback is *best-effort, not transactional*. The marker > write and the enqueue are separate operations. Fedify rolls the marker back > when an enqueue fails, so a transient failure does not suppress the retry, but > a crash before that rollback, the `"open"` fallback under concurrency, a From 7ccdc0d99a670ad05757853324f537bf4b82b7e9 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Wed, 1 Jul 2026 09:46:37 +0000 Subject: [PATCH 41/57] Clarify that deduplicationKey only requests dedup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The introduction to the deduplication section read as if a `deduplicationKey` always drops a duplicate enqueue. That is not true under the default "open" fallback, where a queue without native deduplication and a key–value store without `cas` proceed without deduplication after a debug log. Reword the sentence so the key is described as requesting at-most-once-per-key enqueue, with whether the drop actually happens deferred to the fallback rules that follow. https://github.com/fedify-dev/fedify/pull/806#discussion_r3498265777 Assisted-by: Claude Code:claude-opus-4-8 --- docs/manual/tasks.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index 856b7fe60..599bed2f7 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -268,9 +268,10 @@ Deduplication A task often needs *at-most-once-per-key* enqueue: a digest mailer must not send twice when a request is retried, and a cleanup job should coalesce -duplicate triggers. Passing a `deduplicationKey` requests this—a second -enqueue with the same key is dropped while the first is still within the -deduplication window: +duplicate triggers. Passing a `deduplicationKey` requests this—while the +first enqueue is still within the deduplication window, a second enqueue +with the same key is dropped. Whether that drop actually happens depends +on the queue and key–value store, as the fallback rules below decide: ~~~~ typescript await ctx.enqueueTask(sendDigest, payload, { From 77109fec27b0aa5356c49c9c84a03916ce588b6e Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Wed, 1 Jul 2026 09:52:48 +0000 Subject: [PATCH 42/57] Reject dedup batch on cas path through wrapper On the cas fallback path, planDeduplication only rejected a multi-item deduplicated batch when `queue.enqueueMany` was absent. A `ParallelMessageQueue` always exposes `enqueueMany()`, so the guard was skipped even when the wrapped queue lacked it; the cas plan then claimed one marker and `ParallelMessageQueue.enqueueMany()` fanned the batch out to individual `enqueue()` calls (the wrapper only throws when a `deduplicationKey` is forwarded, which the cas path never does). That silently fanned out a batch the docs promise to reject and left it non-atomic, so a partial failure could duplicate the messages that already succeeded. Look through the wrapper: a `ParallelMessageQueue` whose wrapped queue has no `enqueueMany()` cannot enqueue a batch atomically, so the batch is now rejected before any marker is claimed. The native path was already covered because it forwards the key and the wrapper rejects it there. Add a regression test for a deduplicated multi-item batch over a `ParallelMessageQueue` wrapping a non-native, no-`enqueueMany` backend on the cas path. https://github.com/fedify-dev/fedify/pull/806#discussion_r3498292117 Assisted-by: Claude Code:claude-opus-4-8 --- .../src/federation/tasks/enqueue.test.ts | 37 +++++++++++++++++++ .../fedify/src/federation/tasks/enqueue.ts | 6 ++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/packages/fedify/src/federation/tasks/enqueue.test.ts b/packages/fedify/src/federation/tasks/enqueue.test.ts index ee7db8014..83bfdc70a 100644 --- a/packages/fedify/src/federation/tasks/enqueue.test.ts +++ b/packages/fedify/src/federation/tasks/enqueue.test.ts @@ -1136,6 +1136,43 @@ test( }, ); +test( + "a deduplicated batch over a ParallelMessageQueue wrapping a non-native, " + + "no-enqueueMany backend is rejected on the cas path", + async () => { + const backend = new MockQueue(); + const queue = new ParallelMessageQueue(backend, 5); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("parallel-cas-no-bulk", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + await rejects( + () => + ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }), + { name: "TypeError", message: /enqueueMany/ }, + ); + strictEqual(backend.enqueued.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + + // A single-item batch needs no bulk path, so it still enqueues. + await ctx.enqueueTaskMany(task, ["solo"], { deduplicationKey: "solo" }); + strictEqual(backend.enqueued.length, 1); + }, +); + test( "a deduplicated batch over a ParallelMessageQueue wrapping a native " + "enqueueMany backend forwards the key atomically", diff --git a/packages/fedify/src/federation/tasks/enqueue.ts b/packages/fedify/src/federation/tasks/enqueue.ts index 6500301af..a4009aa1a 100644 --- a/packages/fedify/src/federation/tasks/enqueue.ts +++ b/packages/fedify/src/federation/tasks/enqueue.ts @@ -10,7 +10,7 @@ import { getLogger } from "@logtape/logtape"; import { context, propagation } from "@opentelemetry/api"; import type { KvKey } from "../kv.ts"; import type { FederationImpl } from "../middleware.ts"; -import type { MessageQueue } from "../mq.ts"; +import { type MessageQueue, ParallelMessageQueue } from "../mq.ts"; import type { TaskMessage } from "../queue.ts"; import type TaskCodec from "./codec.ts"; import type { TaskDefinition, TaskEnqueueOptions } from "./task.ts"; @@ -149,7 +149,9 @@ function planDeduplication( const key = options.deduplicationKey; const native = queue.nativeDeduplication === true; const canCas = ctx.federation.kv.cas != null; - if (itemCount > 1 && queue.enqueueMany == null && (native || canCas)) { + const canBatchAtomically = queue.enqueueMany != null && + !(queue instanceof ParallelMessageQueue && queue.queue.enqueueMany == null); + if (itemCount > 1 && !canBatchAtomically && (native || canCas)) { throw new TypeError( `Task ${ JSON.stringify(taskName) From 270344599b05df098cda3b0e0ea90079a8c2c7dc Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Sun, 14 Jun 2026 07:07:52 +0000 Subject: [PATCH 43/57] Support deduplicationKey for task enqueue Context.enqueueTask() and enqueueTaskMany() now accept a deduplicationKey requesting at-most-once enqueue for tasks that share it (new TaskEnqueueOptions.deduplicationKey). Resolution follows the queue and key-value store capabilities: - A queue declaring the new MessageQueue.nativeDeduplication owns the check; the key is forwarded through the new MessageQueueEnqueueOptions.deduplicationKey. - Otherwise Fedify applies a best-effort guard through the optional KvStore.cas primitive under a new taskDeduplication key prefix, tunable with the new FederationOptions.taskDeduplicationTtl and taskDeduplicationFallback options. For enqueueTaskMany(), a single key governs the whole batch. A native queue that does not implement enqueueMany() cannot express batch-level at-most-once with a per-message key, so such a multi-item enqueue is rejected with a TypeError instead of silently leaking duplicates. Configuration errors that are decidable without a payload (a native queue lacking enqueueMany, or a closed fallback without cas) are checked before payloads are validated and encoded, so they reject before any user schema runs or any key is reserved. https://github.com/fedify-dev/fedify/issues/798 Assisted-by: Claude Code:claude-opus-4-8 --- .../fedify/src/federation/tasks/tasks.test.ts | 501 +++++++++++++++++- 1 file changed, 500 insertions(+), 1 deletion(-) diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index c42c7bb94..e54ad8ff6 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -18,8 +18,19 @@ import { import { createFederationBuilder } from "../builder.ts"; import type { Context } from "../context.ts"; import type { Federatable } from "../federation.ts"; +import { + type KvKey, + type KvStore, + type KvStoreListEntry, + type KvStoreSetOptions, + MemoryKvStore, +} from "../kv.ts"; import { createFederation, type FederationImpl } from "../middleware.ts"; -import { InProcessMessageQueue } from "../mq.ts"; +import { + InProcessMessageQueue, + type MessageQueue, + type MessageQueueEnqueueOptions, +} from "../mq.ts"; import type { TaskMessage } from "../queue.ts"; import TaskCodec from "./codec.ts"; import type { TaskDefinition, TaskRegistry } from "./task.ts"; @@ -704,3 +715,491 @@ test("processQueuedTask() task dispatch", async (t) => { strictEqual(queue.enqueued.length, 0); }); }); + +/** + * A {@link KvStore} that delegates to an in-memory store but deliberately + * omits `cas`, so that `kv.cas == null`. This drives the deduplication + * fallback branches that fire when no conditional-write primitive exists. + */ +class CaslessKvStore implements KvStore { + readonly inner = new MemoryKvStore(); + get(key: KvKey): Promise { + return this.inner.get(key); + } + set(key: KvKey, value: unknown, options?: KvStoreSetOptions): Promise { + return this.inner.set(key, value, options); + } + delete(key: KvKey): Promise { + return this.inner.delete(key); + } + list(prefix?: KvKey): AsyncIterable { + return this.inner.list(prefix); + } + // No `cas`: the fallback branch is reached precisely when `kv.cas == null`. +} + +async function collectKeys(kv: KvStore, prefix: KvKey): Promise { + const keys: KvKey[] = []; + for await (const { key } of kv.list(prefix)) keys.push(key); + return keys; +} + +const TASK_DEDUP_PREFIX: KvKey = ["_fedify", "taskDeduplication"]; +const ACTIVITY_IDEMPOTENCE_PREFIX: KvKey = ["_fedify", "activityIdempotence"]; + +test("task deduplication", async (t) => { + await t.step( + "forwards the key to a nativeDeduplication queue without writing KV", + async () => { + const queue = new MockQueue({ nativeDeduplication: true }); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("native-dedup", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + strictEqual(queue.enqueued[0].options?.deduplicationKey, "k"); + // The backend owns the check, so Fedify must not write any KV marker. + strictEqual((await collectKeys(kv, TASK_DEDUP_PREFIX)).length, 0); + }, + ); + + await t.step( + "skips a second enqueue with the same key within the TTL", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new MemoryKvStore(), + queue: { task: queue }, + }); + const task = federation.defineTask("kv-dedup", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "first", { deduplicationKey: "k" }); + await ctx.enqueueTask(task, "second", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + strictEqual(queue.enqueued[0].message.taskName, "kv-dedup"); + // A non-native queue never receives a key it would ignore. + strictEqual(queue.enqueued[0].options?.deduplicationKey, undefined); + }, + ); + + await t.step( + "re-enqueues with the same key after the TTL expires", + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new MemoryKvStore(), + queue: { task: queue }, + taskDeduplicationTtl: { milliseconds: 100 }, + }); + const task = federation.defineTask("kv-dedup-ttl", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "first", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + // Wait comfortably past the 100 ms TTL so the marker expires. + await delay(300); + await ctx.enqueueTask(task, "second", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 2); + }, + ); + + await t.step( + 'rejects with TypeError when fallback is "closed" and no cas exists', + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new CaslessKvStore(), + queue: { task: queue }, + taskDeduplicationFallback: "closed", + }); + const task = federation.defineTask("closed-fallback", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await rejects( + () => ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }), + { name: "TypeError" }, + ); + strictEqual(queue.enqueued.length, 0); + }, + ); + + await t.step( + 'proceeds when fallback is "open" and no cas exists', + async () => { + const queue = new MockQueue(); + const federation = createFederation({ + ...baseOptions, + kv: new CaslessKvStore(), + queue: { task: queue }, + taskDeduplicationFallback: "open", + }); + const task = federation.defineTask("open-fallback", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + // Best-effort fallback never forwards the key to a non-native queue. + strictEqual(queue.enqueued[0].options?.deduplicationKey, undefined); + }, + ); + + await t.step( + "writes only under taskDeduplication, never activityIdempotence", + async () => { + const queue = new MockQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("prefix-isolation", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }); + strictEqual((await collectKeys(kv, TASK_DEDUP_PREFIX)).length, 1); + strictEqual( + (await collectKeys(kv, ACTIVITY_IDEMPOTENCE_PREFIX)).length, + 0, + ); + }, + ); + + await t.step("applies one batch-level key to enqueueTaskMany", async () => { + const queue = new MockQueue({ supportsEnqueueMany: true }); + const federation = createFederation({ + ...baseOptions, + kv: new MemoryKvStore(), + queue: { task: queue }, + }); + const task = federation.defineTask("batch-dedup", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }); + await ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }); + // First batch enqueues all three; the second is skipped entirely. + strictEqual(queue.enqueuedMany.length, 1); + strictEqual(queue.enqueuedMany[0].messages.length, 3); + }); +}); + +test( + "task deduplication validates every payload before reserving the key", + async () => { + const queue = new MockQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("dedup-validation", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + // A rejected payload must neither enqueue nor consume the key. + await rejects(() => + ctx.enqueueTask(task, 123 as unknown as string, { + deduplicationKey: "k", + }) + ); + strictEqual(queue.enqueued.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + + // The same key must remain usable by the first valid enqueue. + await ctx.enqueueTask(task, "valid", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + deepStrictEqual( + await collectKeys(kv, TASK_DEDUP_PREFIX), + [[...TASK_DEDUP_PREFIX, "k"]], + ); + + // Once the valid enqueue reserves it, the same key must deduplicate. + await ctx.enqueueTask(task, "duplicate", { deduplicationKey: "k" }); + strictEqual(queue.enqueued.length, 1); + }, +); + +test( + "native task batch deduplication is one enqueueMany operation per call", + async () => { + class NativeBatchDeduplicatingQueue implements MessageQueue { + readonly nativeDeduplication = true; + readonly #seen = new Set(); + readonly attempts: { + messages: readonly TaskMessage[]; + options?: MessageQueueEnqueueOptions; + }[] = []; + readonly accepted: { + messages: readonly TaskMessage[]; + options?: MessageQueueEnqueueOptions; + }[] = []; + + enqueue(): Promise { + throw new Error("A multi-item native batch must use enqueueMany()."); + } + + enqueueMany( + messages: readonly TaskMessage[], + options?: MessageQueueEnqueueOptions, + ): Promise { + const key = options?.deduplicationKey; + if (key == null) { + throw new TypeError( + "Native batch enqueue requires a deduplication key.", + ); + } + this.attempts.push({ messages, options }); + if (this.#seen.has(key)) return Promise.resolve(); + this.#seen.add(key); + this.accepted.push({ messages, options }); + return Promise.resolve(); + } + + listen(): Promise { + return Promise.resolve(); + } + } + + const queue = new NativeBatchDeduplicatingQueue(); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("native-batch-dedup", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + await ctx.enqueueTaskMany(task, ["a1", "a2", "a3"], { + deduplicationKey: "batch-a", + }); + await ctx.enqueueTaskMany( + task, + ["duplicate1", "duplicate2", "duplicate3"], + { + deduplicationKey: "batch-a", + }, + ); + await ctx.enqueueTaskMany(task, ["b1", "b2", "b3"], { + deduplicationKey: "batch-b", + }); + + // Every API call reaches the backend exactly once, with one key governing + // all three messages. The backend accepts complete batches or none. + strictEqual(queue.attempts.length, 3); + deepStrictEqual( + queue.attempts.map(({ messages }) => messages.length), + [3, 3, 3], + ); + deepStrictEqual( + queue.attempts.map(({ options }) => options?.deduplicationKey), + ["batch-a", "batch-a", "batch-b"], + ); + strictEqual(queue.accepted.length, 2); + deepStrictEqual( + queue.accepted.map(({ messages }) => messages.length), + [3, 3], + ); + deepStrictEqual( + queue.accepted.map(({ options }) => options?.deduplicationKey), + ["batch-a", "batch-b"], + ); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + }, +); + +test( + "native task batch deduplication rejects without enqueueMany", + async () => { + const queue = new MockQueue({ nativeDeduplication: true }); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("native-batch-without-enqueue-many", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + await rejects( + () => + ctx.enqueueTaskMany(task, ["a", "b", "c"], { + deduplicationKey: "batch", + }), + { name: "TypeError", message: /enqueueMany/ }, + ); + + // Reject before any partial enqueue or fallback KV write. Silently + // dropping the key from items 2..n cannot satisfy these assertions. + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + + // A one-item batch is representable by enqueue() and must remain valid. + await ctx.enqueueTaskMany(task, ["single"], { + deduplicationKey: "single", + }); + strictEqual(queue.enqueued.length, 1); + strictEqual(queue.enqueued[0].options?.deduplicationKey, "single"); + }, +); + +test( + "deduplication - native batch capability errors precede payload validation", + async () => { + let validationCalls = 0; + const schema = makeSchema((data): data is string => { + validationCalls++; + return typeof data === "string"; + }); + const queue = new MockQueue({ nativeDeduplication: true }); + const kv = new MemoryKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + }); + const task = federation.defineTask("native-batch-capability-order", { + schema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + let caught: unknown; + try { + await ctx.enqueueTaskMany( + task, + [1, 2, 3] as unknown as readonly string[], + { deduplicationKey: "batch" }, + ); + } catch (error) { + caught = error; + } + + // The queue capability makes this request impossible regardless of the + // payload, so no user-supplied validator may run first. + strictEqual(validationCalls, 0); + ok(caught instanceof TypeError); + ok(caught.message.includes("enqueueMany")); + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + }, +); + +test( + "closed deduplication fallback errors precede payload validation", + async () => { + let validationCalls = 0; + const schema = makeSchema((data): data is string => { + validationCalls++; + return typeof data === "string"; + }); + const queue = new MockQueue(); + const kv = new CaslessKvStore(); + const federation = createFederation({ + ...baseOptions, + kv, + queue: { task: queue }, + taskDeduplicationFallback: "closed", + }); + const task = federation.defineTask("closed-fallback-order", { + schema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + let caught: unknown; + try { + await ctx.enqueueTask( + task, + 1 as unknown as string, + { deduplicationKey: "k" }, + ); + } catch (error) { + caught = error; + } + + // Closed fallback is a configuration-level rejection. It must be + // deterministic and independent of user payload validation. + strictEqual(validationCalls, 0); + ok(caught instanceof TypeError); + ok(caught.message.includes("conditional write")); + strictEqual(queue.enqueued.length, 0); + strictEqual(queue.enqueuedMany.length, 0); + deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); + }, +); From 31177724368341ab37017bd9716de0a6b32680ee Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Sat, 20 Jun 2026 15:50:36 +0000 Subject: [PATCH 44/57] Add OpenTelemetry observability to custom tasks Layer task-specific telemetry onto the custom background task dispatch path, reusing the queue-task metric pattern and mirroring the existing `http_signatures.failure_reason` enum in metrics.ts. Each dequeued task now runs in a `fedify.task` span that inherits the enqueue site's trace context and carries `fedify.task.name`, `fedify.task.attempt`, and, on a terminal failure, `fedify.task.failure_reason`. The `fedify.queue.task.*` metrics report task runs under the new `"task"` role with the task name and, on failure, a bounded `fedify.task.failure_reason`. To tell the failure reasons apart, `#listenTaskMessage` splits the former `decode()` call into its deserialize and validate phases and returns the decision point that failed: `deserialization`, `validation`, `unknown_task`, or `handler`. A swallowed abort is reported as a graceful interruption, not a failure. The reported `fedify.queue.backend` reflects the resolved queue so it stays accurate under the outbox fallback. Public surface: `QueueTaskRole` gains `"task"`, `QueueTaskCommonAttributes` gains `taskName`, and a new `QueueTaskFailureReason` type plus an optional trailing `failureReason` parameter on `recordQueueTaskOutcome()` carry the reason. `TaskCodec` exposes an instance `validate()` wrapper so the dispatch site can split decoding without importing the class. https://github.com/fedify-dev/fedify/issues/799 Assisted-by: Claude Code:claude-opus-4-8 --- AGENTS.md | 18 +- CHANGES.md | 4 +- docs/manual/opentelemetry.md | 52 ++- docs/manual/tasks.md | 74 ++- packages/fedify/src/federation/metrics.ts | 33 +- packages/fedify/src/federation/middleware.ts | 114 ++++- packages/fedify/src/federation/tasks/codec.ts | 19 +- .../fedify/src/federation/tasks/enqueue.ts | 10 + .../fedify/src/federation/tasks/tasks.test.ts | 434 +++++++++++++++++- 9 files changed, 703 insertions(+), 55 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 653114446..35c48b786 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -145,23 +145,23 @@ Development workflow the editor/LSP after editing YAML. - **Building Packages**: All packages are built automatically as part of setup. Run `mise run build` to rebuild everything, or - `mise run prepare-each ` to rebuild specific packages (without the + `mise run prepare-each ` to rebuild specific packages (without the `@fedify/` prefix). - **Checking Code**: Run `mise run check` before committing, or run - `mise run check-each ` to check specific packages. If any issues from - `check:fmt`, `check:lint` or `check:md`, are found, refers + `mise run check-each ` to check specific packages. If any issues from + `check:fmt`, `check:lint` or `check:md`, are found, refers **Formatting and Linting** section. - **Formatting and Linting**: Run `mise run fmt` to format all code and docs. - **Running Tests**: - While testing is certainly important, blindly running every test suite every - time is inefficient. Since Deno executes TS source code directly, it doesn't + While testing is certainly important, blindly running every test suite every + time is inefficient. Since Deno executes TS source code directly, it doesn't waste resources on builds. Therefore, during development, run - `mise run test:deno {TEST_PATH} --filter ` for most tests that - are independent of the runtime. If the test is dependent on a specific + `mise run test:deno {TEST_PATH} --filter ` for most tests that + are independent of the runtime. If the test is dependent on a specific runtime other than Deno, replace `test:deno` with `test:node` or `test:bun`. - Once development is complete, run `mise run test-each ` to test the + Once development is complete, run `mise run test-each ` to test the modified packages (without the `@fedify/` prefix). - Finally, when ready for deployment, run `mise run test` to execute the + Finally, when ready for deployment, run `mise run test` to execute the whole codebase-wide tests. - `mise run test`: Executes all the tests in every runtime. - `mise run test:`: diff --git a/CHANGES.md b/CHANGES.md index 85cdf1598..b231e61ee 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,14 +45,16 @@ Version 2.4.0 `FederationOptions.taskDeduplicationTtl` and `FederationOptions.taskDeduplicationFallback` options. - [[#206], [#797], [#798], [#803], [#806] by ChanHaeng Lee] + [[#206], [#797], [#798], [#799] [#803], [#806], [#812] by ChanHaeng Lee] [Standard Schema]: https://standardschema.dev/ [#206]: https://github.com/fedify-dev/fedify/issues/206 [#797]: https://github.com/fedify-dev/fedify/issues/797 [#798]: https://github.com/fedify-dev/fedify/issues/798 +[#799]: https://github.com/fedify-dev/fedify/issues/799 [#803]: https://github.com/fedify-dev/fedify/pull/803 [#806]: https://github.com/fedify-dev/fedify/pull/806 +[#812]: https://github.com/fedify-dev/fedify/pull/812 ### @fedify/fedify diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index 89beafe86..e96dc68ad 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -226,6 +226,7 @@ spans: | `activitypub.fetch_document` | Client | Fetches a remote JSON-LD document. | | `activitypub.send_activity` | Client | Sends the ActivityPub activity. | | `activitypub.verify_key_ownership` | Internal | Verifies actor ownership of a key. | +| `fedify.task` | Consumer | Dequeues a custom background task to process. | | `http_signatures.sign` | Internal | Signs the HTTP request. | | `http_signatures.verify` | Internal | Verifies the HTTP request signature. | | `ld_signatures.sign` | Internal | Makes the Linked Data signature. | @@ -364,7 +365,7 @@ Fedify records the following OpenTelemetry metrics: | `webfinger.handle.duration` | Histogram | `ms` | Measures inbound WebFinger request handling duration. | | `fedify.http.server.request.count` | Counter | `{request}` | Counts inbound HTTP requests handled by `Federation.fetch()`. | | `fedify.http.server.request.duration` | Histogram | `ms` | Measures inbound HTTP request duration in `Federation.fetch()`. | -| `fedify.queue.task.enqueued` | Counter | `{task}` | Counts inbox, outbox, and fanout tasks Fedify enqueued. | +| `fedify.queue.task.enqueued` | Counter | `{task}` | Counts inbox, outbox, fanout, and custom background tasks Fedify enqueued. | | `fedify.queue.task.started` | Counter | `{task}` | Counts queue tasks Fedify began processing as a worker. | | `fedify.queue.task.completed` | Counter | `{task}` | Counts queue tasks Fedify finished processing without throwing. | | `fedify.queue.task.failed` | Counter | `{task}` | Counts queue tasks Fedify abandoned because processing threw. | @@ -803,11 +804,20 @@ Fedify records the following OpenTelemetry metrics: `fedify.queue.task.enqueued`, `fedify.queue.task.started`, `fedify.queue.task.completed`, `fedify.queue.task.failed`, and `fedify.queue.task.duration` -: `fedify.queue.role` (`inbox`, `outbox`, or `fanout`) is always present. +: `fedify.queue.role` (`inbox`, `outbox`, `fanout`, or `task`) is always + present. `fedify.queue.backend` is the queue implementation's constructor name (for example `RedisMessageQueue`) when available; it is omitted for queues whose constructor is the plain `Object` (for example, - `MessageQueue` instances built from an object literal). + `MessageQueue` instances built from an object literal). For a custom + background task (`role=task`) the backend reflects the queue actually used + after routing, including the `outboxQueue` fallback, and `fedify.task.name` + carries the registered task name—it is omitted for an `unknown_task` drop, + whose name is wire-derived, so task-metric cardinality stays bounded to the + registered names. A failed task outcome + (`fedify.queue.task.result=failed`) additionally carries + `fedify.task.failure_reason`, one of `deserialization`, `validation`, + `unknown_task`, or `handler`. `fedify.queue.native_retrial` reflects the queue backend's `nativeRetrial` flag when set on the queue. `activitypub.activity.type` is recorded whenever Fedify knows the activity type for the queued message; for inbox @@ -818,22 +828,26 @@ Fedify records the following OpenTelemetry metrics: from initial enqueues. `fedify.queue.task.completed`, `fedify.queue.task.failed`, and `fedify.queue.task.duration` carry `fedify.queue.task.result`, which is `completed` when processing returned - without throwing, `failed` when the worker re-threw a non-abort error, and - `aborted` when the worker re-threw an `AbortError` (for example, because a - graceful-shutdown `AbortSignal` interrupted processing). When the queue - backend does not declare `nativeRetrial`, Fedify catches inbox listener and - outbox delivery errors itself; if its retry policy still allows another - attempt, it schedules a retry by re-enqueuing the message and returns from - the worker without re-throwing, so the worker boundary records - `result=completed`. When the retry policy gives up, the worker also - returns normally (`result=completed`) without scheduling a retry. - Outbox-side activity failures remain observable through the - `activitypub.delivery.*` metrics and the `activitypub.delivery.failed` - span event, and any retry attempt (inbox or outbox) appears as a - `fedify.queue.task.enqueued` measurement with a non-zero - `fedify.queue.task.attempt`. Inbox listener errors that the retry policy - abandons are visible through error logs and the inbox span's error status, - but not through a dedicated metric. + without throwing, `failed` when processing did not succeed (for inbox and + outbox, the worker re-threw a non-abort error; for a custom task, either + the handler threw or the payload was dropped—`deserialization`, + `validation`, or `unknown_task`—in which case the message is still acked + but the outcome is recorded as `failed` with a + `fedify.task.failure_reason`), and `aborted` when the worker re-threw an + `AbortError` (for example, because a graceful-shutdown `AbortSignal` + interrupted processing). When the queue backend does not declare + `nativeRetrial`, Fedify catches inbox listener and outbox delivery errors + itself; if its retry policy still allows another attempt, it schedules a + retry by re-enqueuing the message and returns from the worker without + re-throwing, so the worker boundary records `result=completed`. When the + retry policy gives up, the worker also returns normally + (`result=completed`) without scheduling a retry. Outbox-side activity + failures remain observable through the `activitypub.delivery.*` metrics and + the `activitypub.delivery.failed` span event, and any retry attempt (inbox + or outbox) appears as a `fedify.queue.task.enqueued` measurement with a + non-zero `fedify.queue.task.attempt`. Inbox listener errors that the retry + policy abandons are visible through error logs and the inbox span's error + status, but not through a dedicated metric. `fedify.queue.task.in_flight` : `fedify.queue.role` and `fedify.queue.backend` (when available), plus diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index 599bed2f7..c95039495 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -349,11 +349,79 @@ collapsed onto one message. > owns an atomic check. +Observability +------------- + +*Task-specific telemetry is available since Fedify 2.3.0.* + +Each task the worker dequeues runs inside a `fedify.task` [OpenTelemetry] span +(a *consumer* span, since tasks are not part of ActivityPub it is namespaced +under `fedify.` rather than `activitypub.`). The span inherits the trace +context captured at the enqueue site, so a task's processing chains to the +request or job that enqueued it—and every retry attempt chains to the same +parent. The span carries: + + - `fedify.task.name` — the registered task name. + - `fedify.task.attempt` — the zero-based attempt number; a retry re-enqueue + increments it. + - `fedify.task.failure_reason` — set only on a terminal failure, one of the + four bounded values below. + +On a terminal failure the span's status is also set to `ERROR`, so trace-based +error views surface dropped and given-up tasks together with their +`fedify.task.failure_reason`. A worker shutdown is the one exception: an +`aborted` attempt leaves the status unset, since an interruption is not a task +failure. + +Tasks also reuse the `fedify.queue.task.*` metric family (`enqueued`, +`started`, `completed`, `failed`, `duration`, `in_flight`) that the inbox, +outbox, and fanout workers already report. On a task run measurement +(`enqueued`, `started`, `completed`, `failed`, `duration`), +`fedify.queue.role` is `task` and `fedify.task.name` names the task; the +process-local `in_flight` UpDownCounter omits `fedify.task.name` so its +increments and decrements pair up cleanly. +`fedify.queue.backend` reflects the queue actually used after routing—so a task +that falls back to the `outboxQueue` (see +[Routing](#queue-routing-and-isolation)) is labeled with the outbox queue's +backend, not a task queue's. A failed outcome +also carries `fedify.task.failure_reason` on `fedify.queue.task.failed` and +`fedify.queue.task.duration`. + +The `fedify.task.failure_reason` attribute takes one of four bounded values, +mapping to the worker's dispatch decision points: + +| Value | Meaning | +| ----------------- | -------------------------------------------------- | +| `deserialization` | The wire payload could not be deserialized. | +| `validation` | The deserialized payload failed schema validation. | +| `unknown_task` | The task name has no registered handler. | +| `handler` | The registered handler threw. | + +The first three are *drops*: the payload cannot succeed by retrying, so the +worker acknowledges the message and does not re-enqueue it. Telemetry still +records these as a failed outcome with the matching reason, while the queue is +left drained—so a drop is observable without being retried. A `handler` +failure follows the configured retry policy (see +[Retries](#retry-and-error-handling)). A worker shutdown is never counted as a +failure: an interrupted attempt carries no `fedify.task.failure_reason`, +recorded as an `aborted` outcome when the abort propagates (on a `nativeRetrial` +queue) and otherwise folded into a retry like any handler error. + +The bounded value set keeps metric cardinality finite: a metric's task name is +a registered, known-at-startup value, never derived from message content—an +`unknown_task` drop carries a wire-supplied name, so that name is kept off the +metrics (it still appears on the span, which does not aggregate into time +series). See the [OpenTelemetry](./opentelemetry.md) manual for the full span, +attribute, and metric reference. + +[OpenTelemetry]: https://opentelemetry.io/ + + Limitations ----------- -The current API intentionally ships without task-specific OpenTelemetry spans -and metrics, cron-style periodic scheduling, result backends, and per-task -priority. Some of these are planned as follow-ups; see the [tracking issue]. +The current API intentionally ships without cron-style periodic scheduling, +result backends, and per-task priority. Some of these are planned as +follow-ups; see the [tracking issue]. [tracking issue]: https://github.com/fedify-dev/fedify/issues/206 diff --git a/packages/fedify/src/federation/metrics.ts b/packages/fedify/src/federation/metrics.ts index 70de51948..0b5d2b0aa 100644 --- a/packages/fedify/src/federation/metrics.ts +++ b/packages/fedify/src/federation/metrics.ts @@ -16,7 +16,7 @@ import type { MessageQueue } from "./mq.ts"; * The role of a queued task, derived from the queued message's `type` field. * @since 2.3.0 */ -export type QueueTaskRole = "fanout" | "outbox" | "inbox"; +export type QueueTaskRole = "fanout" | "outbox" | "inbox" | "task"; /** * The terminal result of a queued task processing attempt. @@ -91,6 +91,13 @@ export interface QueueTaskCommonAttributes { role: QueueTaskRole; queue?: MessageQueue; activityType?: string; + + /** + * The registered name of a custom background task, emitted as the + * `fedify.task.name` attribute. Set only for the `"task"` role. + * @since 2.3.0 + */ + taskName?: string; } /** @@ -209,6 +216,23 @@ export type HttpSignatureMetricFailureReason = | "invalidSignature" | "keyFetchError"; +/** + * The reason a custom background task terminated unsuccessfully, emitted as the + * `fedify.task.failure_reason` attribute. A small bounded set mapping to the + * worker's dispatch decision points; open to later refinement. + * + * - `deserialization`: the wire payload could not be deserialized. + * - `validation`: the deserialized payload failed schema validation. + * - `unknown_task`: the task name has no registered handler. + * - `handler`: the registered handler threw. + * @since 2.3.0 + */ +export type QueueTaskFailureReason = + | "deserialization" + | "validation" + | "unknown_task" + | "handler"; + /** * Bounded values recorded as `ld_signatures.type` on the signature * verification duration histogram. Fedify only signs and verifies @@ -1009,9 +1033,13 @@ class FederationMetrics { common: QueueTaskCommonAttributes, result: QueueTaskResult, durationMs: number, + failureReason?: QueueTaskFailureReason, ): void { const attributes = buildQueueTaskAttributes(common); attributes["fedify.queue.task.result"] = result; + if (failureReason != null && result === "failed") { + attributes["fedify.task.failure_reason"] = failureReason; + } if (result === "completed") { this.queueTaskCompleted.add(1, attributes); } else if (result === "failed") { @@ -1197,6 +1225,9 @@ function buildQueueTaskAttributes( if (common.activityType != null) { attributes["activitypub.activity.type"] = common.activityType; } + if (common.taskName != null) { + attributes["fedify.task.name"] = common.taskName; + } return attributes; } diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 0f93c900c..69002f270 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -141,6 +141,7 @@ import { isAbortError, type QueueDepthGaugeEntry, type QueueTaskCommonAttributes, + type QueueTaskFailureReason, type QueueTaskResult, recordCircuitBreakerStateChange, recordCollectionRequest, @@ -1192,7 +1193,73 @@ export class FederationImpl }, ); } else if (message.type === "task") { - await this.#listenTaskMessage(contextData, message); + const registered = this.taskDefinitions.get(message.taskName) != null; + const common: QueueTaskCommonAttributes = { + role: "task", + queue: this.resolveTaskQueue(message.taskName), + taskName: registered ? message.taskName : undefined, + }; + await tracer.startActiveSpan( + "fedify.task", + { + kind: SpanKind.CONSUMER, + attributes: { + "fedify.task.name": message.taskName, + "fedify.task.attempt": message.attempt, + }, + }, + extractedContext, + async (span) => { + const spanCtx = span.spanContext(); + return await withContext( + { traceId: spanCtx.traceId, spanId: spanCtx.spanId }, + async () => { + meter.recordQueueTaskStarted(common); + meter.incrementQueueTaskInFlight(common); + const startedAt = performance.now(); + const recordOutcome = ( + outcome: QueueTaskResult, + failureReason: QueueTaskFailureReason | undefined, + error?: unknown, + ): void => { + if (failureReason != null) { + span.setAttribute( + "fedify.task.failure_reason", + failureReason, + ); + span.setStatus({ + code: SpanStatusCode.ERROR, + ...(error == null ? {} : { message: String(error) }), + }); + } + meter.recordQueueTaskOutcome( + common, + outcome, + getDurationMs(startedAt), + outcome === "failed" ? failureReason : undefined, + ); + }; + try { + const failureReason = await this.#listenTaskMessage( + contextData, + message, + ); + recordOutcome( + failureReason == null ? "completed" : "failed", + failureReason, + ); + } catch (e) { + if (isAbortError(e)) recordOutcome("aborted", undefined); + else recordOutcome("failed", "handler", e); + throw e; + } finally { + meter.decrementQueueTaskInFlight(common); + span.end(); + } + }, + ); + }, + ); } }); } @@ -2091,7 +2158,7 @@ export class FederationImpl async #listenTaskMessage( contextData: TContextData, message: TaskMessage, - ): Promise { + ): Promise { const logger = getLogger(["fedify", "federation", "task"]); const def = this.taskDefinitions.get(message.taskName); if (def == null) { @@ -2101,30 +2168,39 @@ export class FederationImpl "dropping.", { taskName: message.taskName }, ); - return; + return "unknown_task"; } const context = this.#createContext(new URL(message.baseUrl), contextData); - let data: unknown; - try { - // decode() deserializes then re-validates at the dequeue boundary - // (drift protection): a durable queue can hand a new deploy a payload - // an old deploy enqueued. - data = await context.codec.decode(def.schema, message.data); - } catch (error) { - // A malformed or incompatible payload won't succeed by retrying. + const deserialized = await context.codec.deserialize(message.data) + .then((value) => ({ ok: true, value }) as const) + .catch((error) => ({ ok: false, error }) as const); + if (!deserialized.ok) { logger.error( - "Custom task {taskName} payload could not be decoded or validated; " + + "Custom task {taskName} payload could not be deserialized; " + "dropping:\n{error}", - { taskName: message.taskName, error }, + { taskName: message.taskName, error: deserialized.error }, ); - return; + return "deserialization"; + } + const data = await context.codec.validate(def.schema, deserialized.value) + .then((value) => ({ ok: true, value }) as const) + .catch((error) => ({ ok: false, error }) as const); + if (!data.ok) { + // An incompatible payload won't succeed by retrying. + logger.error( + "Custom task {taskName} payload failed schema validation; " + + "dropping:\n{error}", + { taskName: message.taskName, error: data.error }, + ); + return "validation"; } try { - await def.handler(context, data); + await def.handler(context, data.value); + return undefined; } catch (error) { if (def.onError != null) { try { - await def.onError(context, error, data); + await def.onError(context, error, data.value); } catch (onErrorError) { logger.error( "onError for custom task {taskName} threw:\n{error}", @@ -2169,6 +2245,10 @@ export class FederationImpl delay: clampNegativeDelay(delay), orderingKey: message.orderingKey, }); + getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued( + { role: "task", queue, taskName: message.taskName }, + retryMessage.attempt, + ); } else { logger.error( "Custom task {taskName} failed after {attempt} attempts; giving " + @@ -2176,6 +2256,8 @@ export class FederationImpl { taskName: message.taskName, attempt: message.attempt, error }, ); } + // A swallowed abort is a graceful interruption, not a task failure. + return isAbortError(error) ? undefined : "handler"; } } diff --git a/packages/fedify/src/federation/tasks/codec.ts b/packages/fedify/src/federation/tasks/codec.ts index e649ce061..d29d682c5 100644 --- a/packages/fedify/src/federation/tasks/codec.ts +++ b/packages/fedify/src/federation/tasks/codec.ts @@ -17,9 +17,8 @@ export default class TaskCodec { serialize = (data: unknown): Promise => stringifyAsync(data, { Vocab: this.#stringifyVocab }); - /** Deserializes `raw`, rebuilding any encoded vocabulary object. */ - deserialize = (raw: string): Promise => - this.#revive(new Map())(parse(raw, { Vocab: VocabHolder.from })); + deserialize = async (raw: string): Promise => + await this.#revive(new Map())(parse(raw, { Vocab: VocabHolder.from })); /** Validates `data` against `schema`, then serializes it. */ encode = async ( @@ -34,7 +33,19 @@ export default class TaskCodec { ): Promise> => TaskCodec.validate(schema, await this.deserialize(raw)); - /** Validates `data` against `schema`, returning its parsed output. */ + /** + * Validates an already-deserialized `data` against `schema`. An instance + * wrapper over {@link TaskCodec.validate} so the dispatch site can split + * {@link decode} into its deserialize and validate phases—telling a + * deserialization failure apart from a validation failure—without importing + * the class. + */ + validate = ( + schema: S, + data: unknown, + ): Promise> => + TaskCodec.validate(schema, data); + static validate = async ( schema: S, data: unknown, diff --git a/packages/fedify/src/federation/tasks/enqueue.ts b/packages/fedify/src/federation/tasks/enqueue.ts index a4009aa1a..acdd9dae0 100644 --- a/packages/fedify/src/federation/tasks/enqueue.ts +++ b/packages/fedify/src/federation/tasks/enqueue.ts @@ -9,6 +9,7 @@ import { getLogger } from "@logtape/logtape"; import { context, propagation } from "@opentelemetry/api"; import type { KvKey } from "../kv.ts"; +import { getFederationMetrics } from "../metrics.ts"; import type { FederationImpl } from "../middleware.ts"; import { type MessageQueue, ParallelMessageQueue } from "../mq.ts"; import type { TaskMessage } from "../queue.ts"; @@ -100,6 +101,15 @@ const enqueueTasks = ( orderingKey: options.orderingKey, deduplicationKey: claim.forwardedDeduplicationKey, }); + // Counted only after a genuine dispatch: a dedup skip returns before + // this, and a failed dispatch throws into the rollback below. + const meter = getFederationMetrics(ctx.federation.meterProvider); + for (const message of messages) { + meter.recordQueueTaskEnqueued( + { role: "task", queue, taskName: task.name }, + message.attempt, + ); + } } catch (error) { if (claim.rollback != null) { try { diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index e54ad8ff6..e62bb0945 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -1,5 +1,12 @@ -import { mockDocumentLoader, test } from "@fedify/fixture"; +import { + createTestMeterProvider, + createTestTracerProvider, + mockDocumentLoader, + test, +} from "@fedify/fixture"; import { Note } from "@fedify/vocab"; +import { propagation, SpanStatusCode } from "@opentelemetry/api"; +import { W3CTraceContextPropagator } from "@opentelemetry/core"; import { delay } from "es-toolkit"; import { deepStrictEqual, @@ -17,7 +24,7 @@ import { } from "../../testing/mod.ts"; import { createFederationBuilder } from "../builder.ts"; import type { Context } from "../context.ts"; -import type { Federatable } from "../federation.ts"; +import type { Federatable, FederationOptions } from "../federation.ts"; import { type KvKey, type KvStore, @@ -1203,3 +1210,426 @@ test( deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []); }, ); + +/** Wires test telemetry doubles into a fresh federation for the suite below. */ +const instrument = (options: FederationOptions) => { + const [meterProvider, recorder] = createTestMeterProvider(); + const [tracerProvider, exporter] = createTestTracerProvider(); + const federation = createFederation({ + ...options, + meterProvider, + tracerProvider, + }) as FederationImpl; + return { federation, recorder, exporter }; +}; + +test("task observability", async (t) => { + await t.step( + "opens a fedify.task span carrying name and attempt on success", + async () => { + const queue = new MockQueue(); + const { federation, recorder, exporter } = instrument({ + ...baseOptions, + queue: { task: queue }, + }); + federation.defineTask("sendDigest", { + schema: stringSchema, + handler: () => {}, + }); + await federation.processQueuedTask( + undefined, + await makeTaskMessage("sendDigest", "payload"), + ); + + const spans = exporter.getSpans("fedify.task"); + strictEqual(spans.length, 1); + strictEqual(spans[0].attributes["fedify.task.name"], "sendDigest"); + strictEqual(spans[0].attributes["fedify.task.attempt"], 0); + // A completed task carries no failure reason on its span… + strictEqual(spans[0].attributes["fedify.task.failure_reason"], undefined); + strictEqual(spans[0].status.code, SpanStatusCode.UNSET); + + const started = recorder.getMeasurements("fedify.queue.task.started"); + strictEqual(started.length, 1); + strictEqual(started[0].attributes["fedify.queue.role"], "task"); + strictEqual(started[0].attributes["fedify.task.name"], "sendDigest"); + + const completed = recorder.getMeasurements("fedify.queue.task.completed"); + strictEqual(completed.length, 1); + strictEqual(completed[0].attributes["fedify.queue.role"], "task"); + strictEqual(completed[0].attributes["fedify.task.name"], "sendDigest"); + strictEqual( + completed[0].attributes["fedify.queue.task.result"], + "completed", + ); + // …nor on its outcome metric. + strictEqual( + completed[0].attributes["fedify.task.failure_reason"], + undefined, + ); + strictEqual( + recorder.getMeasurements("fedify.queue.task.failed").length, + 0, + ); + }, + ); + + await t.step( + "inherits the parent trace context from the enqueue site", + async () => { + // The worker extracts the parent through the global propagator; a real + // W3C propagator is required because the default is a no-op. + const traceId = "0af7651916cd43dd8448eb211c80319c"; + const spanId = "b7ad6b7169203331"; + propagation.setGlobalPropagator(new W3CTraceContextPropagator()); + try { + const queue = new MockQueue(); + const { federation, exporter } = instrument({ + ...baseOptions, + queue: { task: queue }, + }); + federation.defineTask("traced", { + schema: stringSchema, + handler: () => {}, + }); + await federation.processQueuedTask( + undefined, + await makeTaskMessage("traced", "payload", { + traceContext: { + traceparent: `00-${traceId}-${spanId}-01`, + }, + }), + ); + + const span = exporter.getSpans("fedify.task")[0]; + ok(span != null); + strictEqual(span.spanContext().traceId, traceId); + strictEqual(span.parentSpanContext?.spanId, spanId); + } finally { + propagation.disable(); + } + }, + ); + + await t.step( + "attributes a deserialization failure and drops without retry", + async () => { + const queue = new MockQueue(); + const { federation, recorder, exporter } = instrument({ + ...baseOptions, + queue: { task: queue }, + }); + let called = 0; + federation.defineTask("decode-me", { + schema: stringSchema, + handler: () => { + called++; + }, + }); + const message = await makeTaskMessage("decode-me", "payload"); + await federation.processQueuedTask(undefined, { + ...message, + data: "garbage that is not devalue", + }); + + strictEqual(called, 0); + strictEqual(queue.enqueued.length, 0); // dropped, not retried + const failed = recorder.getMeasurements("fedify.queue.task.failed"); + strictEqual(failed.length, 1); + strictEqual(failed[0].attributes["fedify.queue.role"], "task"); + strictEqual( + failed[0].attributes["fedify.task.failure_reason"], + "deserialization", + ); + const span = exporter.getSpans("fedify.task")[0]; + strictEqual( + span.attributes["fedify.task.failure_reason"], + "deserialization", + ); + // A dropped payload is a failed outcome, so the span status is ERROR. + strictEqual(span.status.code, SpanStatusCode.ERROR); + }, + ); + + await t.step( + "attributes a validation failure and drops without retry", + async () => { + const queue = new MockQueue(); + const { federation, recorder, exporter } = instrument({ + ...baseOptions, + queue: { task: queue }, + }); + let called = 0; + federation.defineTask("strict-shape", { + schema: numberSchema, // expects a number… + handler: () => { + called++; + }, + }); + // …but a valid devalue payload carries a string. + await federation.processQueuedTask( + undefined, + await makeTaskMessage("strict-shape", "not a number"), + ); + + strictEqual(called, 0); + strictEqual(queue.enqueued.length, 0); // dropped, not retried + const failed = recorder.getMeasurements("fedify.queue.task.failed"); + strictEqual(failed.length, 1); + strictEqual( + failed[0].attributes["fedify.task.failure_reason"], + "validation", + ); + const span = exporter.getSpans("fedify.task")[0]; + strictEqual( + span.attributes["fedify.task.failure_reason"], + "validation", + ); + strictEqual(span.status.code, SpanStatusCode.ERROR); + }, + ); + + await t.step( + "attributes an unknown task and drops without retry", + async () => { + const queue = new MockQueue(); + const { federation, recorder, exporter } = instrument({ + ...baseOptions, + queue: { task: queue }, + }); + await federation.processQueuedTask( + undefined, + await makeTaskMessage("never-registered", "payload"), + ); + + strictEqual(queue.enqueued.length, 0); // dropped, not retried + const failed = recorder.getMeasurements("fedify.queue.task.failed"); + strictEqual(failed.length, 1); + strictEqual( + failed[0].attributes["fedify.task.failure_reason"], + "unknown_task", + ); + // The backend label is still populated on an unknown-task drop. + strictEqual(failed[0].attributes["fedify.queue.backend"], "MockQueue"); + // Cardinality guard: an unregistered, wire-derived task name must NOT + // become a metric attribute—it would spawn unbounded time series… + strictEqual(failed[0].attributes["fedify.task.name"], undefined); + strictEqual( + recorder.getMeasurements("fedify.queue.task.started")[0] + .attributes["fedify.task.name"], + undefined, + ); + const span = exporter.getSpans("fedify.task")[0]; + // …but the span still carries the real name for tracing the drop. + strictEqual(span.attributes["fedify.task.name"], "never-registered"); + strictEqual( + span.attributes["fedify.task.failure_reason"], + "unknown_task", + ); + strictEqual(span.status.code, SpanStatusCode.ERROR); + }, + ); + + await t.step( + "attributes a handler failure on a terminal give-up", + async () => { + const queue = new MockQueue(); + const { federation, recorder, exporter } = instrument({ + ...baseOptions, + queue: { task: queue }, + }); + federation.defineTask("explode", { + schema: stringSchema, + handler: () => { + throw new Error("boom"); + }, + retryPolicy: () => null, // give up immediately + }); + await federation.processQueuedTask( + undefined, + await makeTaskMessage("explode", "payload"), + ); + + strictEqual(queue.enqueued.length, 0); // gave up, no retry + const failed = recorder.getMeasurements("fedify.queue.task.failed"); + strictEqual(failed.length, 1); + strictEqual( + failed[0].attributes["fedify.task.failure_reason"], + "handler", + ); + const span = exporter.getSpans("fedify.task")[0]; + strictEqual( + span.attributes["fedify.task.failure_reason"], + "handler", + ); + // A terminal give-up is a failed outcome, so the span status is ERROR. + strictEqual(span.status.code, SpanStatusCode.ERROR); + }, + ); + + await t.step( + "reports the resolved outbox queue as the backend on fallback", + async () => { + // A distinctly named queue so the backend label is unambiguous. + class FallbackOutboxQueue extends MockQueue {} + const outboxQueue = new FallbackOutboxQueue(); + const { federation, recorder } = instrument({ + ...baseOptions, + queue: { outbox: outboxQueue }, // no dedicated task queue + }); + federation.defineTask("fallback", { + schema: stringSchema, + handler: () => {}, + }); + await federation.processQueuedTask( + undefined, + await makeTaskMessage("fallback", "payload"), + ); + + const completed = recorder.getMeasurements("fedify.queue.task.completed"); + strictEqual(completed.length, 1); + strictEqual( + completed[0].attributes["fedify.queue.backend"], + "FallbackOutboxQueue", + ); + }, + ); + + await t.step( + "records an enqueue measurement with role task", + async () => { + const queue = new MockQueue(); + const { federation, recorder } = instrument({ + ...baseOptions, + queue: { task: queue }, + }); + const task = federation.defineTask("enqueue-me", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + await ctx.enqueueTask(task, "payload"); + + const enqueued = recorder.getMeasurements("fedify.queue.task.enqueued"); + strictEqual(enqueued.length, 1); + strictEqual(enqueued[0].attributes["fedify.queue.role"], "task"); + strictEqual(enqueued[0].attributes["fedify.task.name"], "enqueue-me"); + strictEqual(enqueued[0].attributes["fedify.queue.task.attempt"], 0); + }, + ); + + await t.step( + "records the retry re-enqueue with role task and a bumped attempt", + async () => { + const queue = new MockQueue(); + const { federation, recorder } = instrument({ + ...baseOptions, + queue: { task: queue }, + }); + federation.defineTask("retry-me", { + schema: stringSchema, + handler: () => { + throw new Error("boom"); + }, + retryPolicy: () => Temporal.Duration.from({ milliseconds: 1 }), + }); + await federation.processQueuedTask( + undefined, + await makeTaskMessage("retry-me", "payload"), + ); + + strictEqual(queue.enqueued.length, 1); + strictEqual(queue.enqueued[0].message.attempt, 1); + const enqueued = recorder.getMeasurements("fedify.queue.task.enqueued"); + strictEqual(enqueued.length, 1); + strictEqual(enqueued[0].attributes["fedify.queue.role"], "task"); + strictEqual(enqueued[0].attributes["fedify.task.name"], "retry-me"); + strictEqual(enqueued[0].attributes["fedify.queue.task.attempt"], 1); + }, + ); + + await t.step( + "records an abort as aborted, without a failure reason or error status", + async () => { + const queue = new MockQueue({ nativeRetrial: true }); + const { federation, recorder, exporter } = instrument({ + ...baseOptions, + queue: { task: queue }, + }); + federation.defineTask("aborts", { + schema: stringSchema, + handler: () => { + throw globalThis.Object.assign(new Error("shutting down"), { + name: "AbortError", + }); + }, + }); + const message = await makeTaskMessage("aborts", "payload"); + await rejects( + () => federation.processQueuedTask(undefined, message), + { name: "AbortError" }, + ); + + const span = exporter.getSpans("fedify.task")[0]; + ok(span != null); + strictEqual(span.attributes["fedify.task.failure_reason"], undefined); + strictEqual(span.status.code, SpanStatusCode.UNSET); + strictEqual( + recorder.getMeasurements("fedify.queue.task.failed").length, + 0, + ); + const durations = recorder.getMeasurements("fedify.queue.task.duration"); + strictEqual(durations.length, 1); + strictEqual( + durations[0].attributes["fedify.queue.task.result"], + "aborted", + ); + }, + ); + + await t.step( + "on a non-native queue an aborted handler is retried, not failed", + async () => { + const queue = new MockQueue(); // nativeRetrial: false + const { federation, recorder, exporter } = instrument({ + ...baseOptions, + queue: { task: queue }, + }); + federation.defineTask("aborts-soft", { + schema: stringSchema, + handler: () => { + throw globalThis.Object.assign(new Error("shutting down"), { + name: "AbortError", + }); + }, + retryPolicy: () => Temporal.Duration.from({ milliseconds: 1 }), + }); + await federation.processQueuedTask( + undefined, + await makeTaskMessage("aborts-soft", "payload"), + ); + + strictEqual(queue.enqueued.length, 1); // retried, behavior unchanged + strictEqual(queue.enqueued[0].message.attempt, 1); + // No `handler` failure leaks onto the span or any failure metric… + const span = exporter.getSpans("fedify.task")[0]; + strictEqual(span.attributes["fedify.task.failure_reason"], undefined); + strictEqual(span.status.code, SpanStatusCode.UNSET); + strictEqual( + recorder.getMeasurements("fedify.queue.task.failed").length, + 0, + ); + // …and the swallowed-into-retry attempt records `completed`, matching the + // inbox/outbox internal-retry convention. + const completed = recorder.getMeasurements("fedify.queue.task.completed"); + strictEqual(completed.length, 1); + strictEqual( + completed[0].attributes["fedify.task.failure_reason"], + undefined, + ); + }, + ); +}); From 3fb3d8b888307bdb329038a13873ed9cbfbdfe41 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Thu, 11 Jun 2026 13:57:26 +0000 Subject: [PATCH 45/57] Drop the build step from the Deno test task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deno executes the TypeScript sources directly, so `test:deno` spent most of its time on a `build` it never needed: with no dist/ output present, the whole Deno suite passes except the npm packaging regression tests added for https://github.com/fedify-dev/fedify/issues/655, which assert that the built package.json entry points of `@fedify/cli`, `@fedify/create`, and `@fedify/init` exist. Those checks guard the npm artifacts, not the Deno runtime, and still run under `test:node` and `test:bun`, which build first—so skip them under Deno and drop the `build` dependency from `test:deno`. `@fedify/lint`'s oxlint integration test already skips itself when *dist/oxlint.js* is absent. Update AGENTS.md to match: document `mise run build`/`prepare-each` for building, `check-each` and `test-each` for scoping work to specific packages, recommend the now build-free `test:deno` as the default test loop during development, and add a section directing agents to consult `mise tasks`. Assisted-by: Claude Code:claude-opus-4-8 Assisted-by: Claude Code:claude-fable-5 --- packages/cli/src/startup.test.ts | 5 ++++- packages/create/src/package.test.ts | 1 + packages/init/src/package.test.ts | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/startup.test.ts b/packages/cli/src/startup.test.ts index e73f5098f..f13e1170d 100644 --- a/packages/cli/src/startup.test.ts +++ b/packages/cli/src/startup.test.ts @@ -5,7 +5,10 @@ import test from "node:test"; import { fileURLToPath } from "node:url"; const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -test("CLI build keeps the init command bridge", async () => { + +test("CLI build keeps the init command bridge", { + skip: "Deno" in globalThis, +}, async () => { const entrypoint = resolve(packageDir, "dist/mod.js"); const commandBridge = resolve(packageDir, "dist/commands.js"); await access(entrypoint); diff --git a/packages/create/src/package.test.ts b/packages/create/src/package.test.ts index ba50b3660..4a753d467 100644 --- a/packages/create/src/package.test.ts +++ b/packages/create/src/package.test.ts @@ -8,6 +8,7 @@ const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), ".."); test( "package.json entrypoints match built create CLI", + { skip: "Deno" in globalThis }, async () => { const packageJson = JSON.parse( await readFile(resolve(packageDir, "package.json"), "utf8"), diff --git a/packages/init/src/package.test.ts b/packages/init/src/package.test.ts index 97a33247b..39cf17011 100644 --- a/packages/init/src/package.test.ts +++ b/packages/init/src/package.test.ts @@ -13,6 +13,7 @@ async function assertTargetExists(path: string): Promise { test( "package.json entrypoints match built init files", + { skip: "Deno" in globalThis }, async () => { const packageJson = JSON.parse( await readFile(resolve(packageDir, "package.json"), "utf8"), From 72929a6f7f13b9b8a750518c4098b45730c7d75d Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Sat, 4 Jul 2026 08:46:26 +0000 Subject: [PATCH 46/57] Record only terminal task failures in telemetry The fedify.task span and the fedify.queue.task.* metrics promised that fedify.task.failure_reason is set only on a terminal failure, but the task worker attributed a handler failure to every thrown attempt, even when the retry policy had just scheduled a re-enqueue. A transient error that later succeeded thus produced a failed measurement and an ERROR span, inflating failure alerts and diverging from the inbox/outbox convention where an attempt folded into a retry records result=completed. To fix this at the decision point instead of patching each call site, the task listener now returns a dispatch result that distinguishes the outcome: - A handler error folded into a scheduled retry records completed. - Only a terminal give-up records failed with the handler reason. - An aborted attempt that the retry policy abandons records aborted rather than completed, so tasks dropped during a graceful shutdown are no longer invisible to telemetry. TaskCodec.decode() now reports which phase failed (deserialization or validation) instead of throwing, so the worker no longer re-composes the codec's deserialize/validate pipeline inline to tell the two apart; the redundant instance validate() wrapper is removed. The retry-path test now pins the decided semantics (no failed measurement, span status UNSET), a new test covers the abandoned-abort case, and the manual's failed/aborted definitions are updated to match. https://github.com/fedify-dev/fedify/pull/812#discussion_r3517778514 https://github.com/fedify-dev/fedify/pull/812#discussion_r3517806156 Assisted-by: Claude Code:claude-fable-5 Assisted-by: Claude Code:claude-opus-4-8 Assisted-by: Codex:gpt-5.5 --- docs/manual/opentelemetry.md | 13 ++-- docs/manual/tasks.md | 11 ++-- .../fedify/src/federation/middleware.test.ts | 7 ++- packages/fedify/src/federation/middleware.ts | 62 ++++++++++--------- .../fedify/src/federation/tasks/codec.test.ts | 35 +++++++---- packages/fedify/src/federation/tasks/codec.ts | 36 ++++++----- .../fedify/src/federation/tasks/tasks.test.ts | 58 ++++++++++++++++- 7 files changed, 155 insertions(+), 67 deletions(-) diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index e96dc68ad..78fff942c 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -830,12 +830,15 @@ Fedify records the following OpenTelemetry metrics: `fedify.queue.task.result`, which is `completed` when processing returned without throwing, `failed` when processing did not succeed (for inbox and outbox, the worker re-threw a non-abort error; for a custom task, either - the handler threw or the payload was dropped—`deserialization`, - `validation`, or `unknown_task`—in which case the message is still acked - but the outcome is recorded as `failed` with a - `fedify.task.failure_reason`), and `aborted` when the worker re-threw an + the payload was dropped—`deserialization`, `validation`, or + `unknown_task`—in which case the message is still acked but the outcome + is recorded as `failed` with a `fedify.task.failure_reason`, or the + handler threw and the retry policy declined another attempt, recorded + with the `handler` reason—a handler error folded into a scheduled retry + records `completed` instead), and `aborted` when the worker re-threw an `AbortError` (for example, because a graceful-shutdown `AbortSignal` - interrupted processing). When the queue backend does not declare + interrupted processing) or when an aborted custom task attempt was + abandoned without a retry. When the queue backend does not declare `nativeRetrial`, Fedify catches inbox listener and outbox delivery errors itself; if its retry policy still allows another attempt, it schedules a retry by re-enqueuing the message and returns from the worker without diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index c95039495..96c90fabe 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -402,10 +402,13 @@ worker acknowledges the message and does not re-enqueue it. Telemetry still records these as a failed outcome with the matching reason, while the queue is left drained—so a drop is observable without being retried. A `handler` failure follows the configured retry policy (see -[Retries](#retry-and-error-handling)). A worker shutdown is never counted as a -failure: an interrupted attempt carries no `fedify.task.failure_reason`, -recorded as an `aborted` outcome when the abort propagates (on a `nativeRetrial` -queue) and otherwise folded into a retry like any handler error. +[Retries](#retry-and-error-handling)): an attempt folded into a scheduled +retry records a `completed` outcome, and only the terminal give-up records +`failed` with the `handler` reason. A worker shutdown is never counted as a +failure: an interrupted attempt carries no `fedify.task.failure_reason`—it is +recorded as an `aborted` outcome when the abort propagates (on a +`nativeRetrial` queue) or when the retry policy declines another attempt, and +otherwise folded into a scheduled retry like any handler error. The bounded value set keeps metric cardinality finite: a metric's task name is a registered, known-at-startup value, never derived from message content—an diff --git a/packages/fedify/src/federation/middleware.test.ts b/packages/fedify/src/federation/middleware.test.ts index 6ff54ca91..d1e3c3882 100644 --- a/packages/fedify/src/federation/middleware.test.ts +++ b/packages/fedify/src/federation/middleware.test.ts @@ -10506,8 +10506,11 @@ test("createFederation() omits instrumentation when no meterProvider is set", () }); const taskCodec = new TaskCodec({ contextLoader: mockDocumentLoader }); -const decodeEnvelope = (message: TaskMessage): Promise => - taskCodec.decode(envelopeSchema, message.data); +const decodeEnvelope = async (message: TaskMessage): Promise => { + const decoded = await taskCodec.decode(envelopeSchema, message.data); + if (!decoded.ok) throw decoded.error; + return decoded.value; +}; const envelope = (title: string): Envelope => ({ note: new Note({ content: title }), title, diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 69002f270..433362eb5 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -177,6 +177,13 @@ import { import { hasMalformedKnownTemporalLiteral } from "./temporal.ts"; import { handleWebFinger } from "./webfinger.ts"; +type QueueTaskDispatchResult = + | { readonly outcome: "completed" | "aborted" } + | { + readonly outcome: "failed"; + readonly failureReason: QueueTaskFailureReason; + }; + const circuitBreakerCasWarningKvStores = new WeakSet(); let nextQueueDepthGaugeSourceId = 0; const retryAfterHttpDate = new RegExp( @@ -1240,13 +1247,15 @@ export class FederationImpl ); }; try { - const failureReason = await this.#listenTaskMessage( + const result = await this.#listenTaskMessage( contextData, message, ); recordOutcome( - failureReason == null ? "completed" : "failed", - failureReason, + result.outcome, + result.outcome === "failed" + ? result.failureReason + : undefined, ); } catch (e) { if (isAbortError(e)) recordOutcome("aborted", undefined); @@ -2158,7 +2167,7 @@ export class FederationImpl async #listenTaskMessage( contextData: TContextData, message: TaskMessage, - ): Promise { + ): Promise { const logger = getLogger(["fedify", "federation", "task"]); const def = this.taskDefinitions.get(message.taskName); if (def == null) { @@ -2168,35 +2177,29 @@ export class FederationImpl "dropping.", { taskName: message.taskName }, ); - return "unknown_task"; + return { outcome: "failed", failureReason: "unknown_task" }; } const context = this.#createContext(new URL(message.baseUrl), contextData); - const deserialized = await context.codec.deserialize(message.data) - .then((value) => ({ ok: true, value }) as const) - .catch((error) => ({ ok: false, error }) as const); - if (!deserialized.ok) { - logger.error( - "Custom task {taskName} payload could not be deserialized; " + - "dropping:\n{error}", - { taskName: message.taskName, error: deserialized.error }, - ); - return "deserialization"; - } - const data = await context.codec.validate(def.schema, deserialized.value) - .then((value) => ({ ok: true, value }) as const) - .catch((error) => ({ ok: false, error }) as const); + const data = await context.codec.decode(def.schema, message.data); if (!data.ok) { - // An incompatible payload won't succeed by retrying. - logger.error( - "Custom task {taskName} payload failed schema validation; " + - "dropping:\n{error}", - { taskName: message.taskName, error: data.error }, - ); - return "validation"; + if (data.phase === "deserialization") { + logger.error( + "Custom task {taskName} payload could not be deserialized; " + + "dropping:\n{error}", + { taskName: message.taskName, error: data.error }, + ); + } else { + logger.error( + "Custom task {taskName} payload failed schema validation; " + + "dropping:\n{error}", + { taskName: message.taskName, error: data.error }, + ); + } + return { outcome: "failed", failureReason: data.phase }; } try { await def.handler(context, data.value); - return undefined; + return { outcome: "completed" }; } catch (error) { if (def.onError != null) { try { @@ -2249,6 +2252,7 @@ export class FederationImpl { role: "task", queue, taskName: message.taskName }, retryMessage.attempt, ); + return { outcome: "completed" }; } else { logger.error( "Custom task {taskName} failed after {attempt} attempts; giving " + @@ -2257,7 +2261,9 @@ export class FederationImpl ); } // A swallowed abort is a graceful interruption, not a task failure. - return isAbortError(error) ? undefined : "handler"; + return isAbortError(error) + ? { outcome: "aborted" } + : { outcome: "failed", failureReason: "handler" }; } } diff --git a/packages/fedify/src/federation/tasks/codec.test.ts b/packages/fedify/src/federation/tasks/codec.test.ts index fcc84de0e..d4fae0463 100644 --- a/packages/fedify/src/federation/tasks/codec.test.ts +++ b/packages/fedify/src/federation/tasks/codec.test.ts @@ -328,9 +328,10 @@ test("TaskCodec.encode() / decode()", async (t) => { const wire = await codec.encode(schema, payload); strictEqual(typeof wire, "string"); const back = await codec.decode(schema, wire); - ok(back.note instanceof Note); - strictEqual(back.note.content?.toString(), "Hi"); - strictEqual(back.title, "greeting"); + if (!back.ok) throw back.error; + ok(back.value.note instanceof Note); + strictEqual(back.value.note.content?.toString(), "Hi"); + strictEqual(back.value.title, "greeting"); }, ); @@ -342,18 +343,27 @@ test("TaskCodec.encode() / decode()", async (t) => { }); await t.step( - "decode() re-validates and rejects a drifted payload", + "decode() reports a validation phase for a drifted payload", async () => { // Encode under a permissive schema, decode under the strict one. const loose = makeSchema((_data): _data is unknown => true); const wire = await codec.encode(loose, { note: "not a note" }); - await rejects( - () => codec.decode(schema, wire), - { name: "TypeError", message: /Task data failed schema validation/ }, - ); + const result = await codec.decode(schema, wire); + strictEqual(result.ok, false); + if (result.ok) throw new Error("Expected decode to fail."); + strictEqual(result.phase, "validation"); + ok(result.error instanceof TypeError); }, ); + await t.step("decode() reports a deserialization phase", async () => { + const result = await codec.decode(schema, "not devalue"); + strictEqual(result.ok, false); + if (result.ok) throw new Error("Expected decode to fail."); + strictEqual(result.phase, "deserialization"); + ok(result.error instanceof Error); + }); + await t.step( "a non-idempotent (transforming) schema fails to round-trip", async () => { @@ -370,10 +380,11 @@ test("TaskCodec.encode() / decode()", async (t) => { }, }; const wire = await codec.encode(transforming, "hello"); - await rejects( - () => codec.decode(transforming, wire), - { name: "TypeError", message: /Task data failed schema validation/ }, - ); + const result = await codec.decode(transforming, wire); + strictEqual(result.ok, false); + if (result.ok) throw new Error("Expected decode to fail."); + strictEqual(result.phase, "validation"); + ok(result.error instanceof TypeError); }, ); }); diff --git a/packages/fedify/src/federation/tasks/codec.ts b/packages/fedify/src/federation/tasks/codec.ts index d29d682c5..39ce04fd8 100644 --- a/packages/fedify/src/federation/tasks/codec.ts +++ b/packages/fedify/src/federation/tasks/codec.ts @@ -4,6 +4,14 @@ import type { TracerProvider } from "@opentelemetry/api"; import type { StandardSchemaV1 } from "@standard-schema/spec"; import { parse, stringifyAsync } from "devalue"; +type TaskCodecDecodeResult = + | { readonly ok: true; readonly value: StandardSchemaV1.InferOutput } + | { + readonly ok: false; + readonly phase: "deserialization" | "validation"; + readonly error: unknown; + }; + /** * Serializes and deserializes task payloads for the queue, preserving * `@fedify/vocab` objects across the wire by reducing them to JSON-LD and @@ -30,21 +38,19 @@ export default class TaskCodec { decode = async ( schema: S, raw: string, - ): Promise> => - TaskCodec.validate(schema, await this.deserialize(raw)); - - /** - * Validates an already-deserialized `data` against `schema`. An instance - * wrapper over {@link TaskCodec.validate} so the dispatch site can split - * {@link decode} into its deserialize and validate phases—telling a - * deserialization failure apart from a validation failure—without importing - * the class. - */ - validate = ( - schema: S, - data: unknown, - ): Promise> => - TaskCodec.validate(schema, data); + ): Promise> => { + let data: unknown; + try { + data = await this.deserialize(raw); + } catch (error) { + return { ok: false, phase: "deserialization", error }; + } + try { + return { ok: true, value: await TaskCodec.validate(schema, data) }; + } catch (error) { + return { ok: false, phase: "validation", error }; + } + }; static validate = async ( schema: S, diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index e62bb0945..28fce13d1 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -1525,7 +1525,7 @@ test("task observability", async (t) => { "records the retry re-enqueue with role task and a bumped attempt", async () => { const queue = new MockQueue(); - const { federation, recorder } = instrument({ + const { federation, recorder, exporter } = instrument({ ...baseOptions, queue: { task: queue }, }); @@ -1548,6 +1548,19 @@ test("task observability", async (t) => { strictEqual(enqueued[0].attributes["fedify.queue.role"], "task"); strictEqual(enqueued[0].attributes["fedify.task.name"], "retry-me"); strictEqual(enqueued[0].attributes["fedify.queue.task.attempt"], 1); + const span = exporter.getSpans("fedify.task")[0]; + strictEqual(span.attributes["fedify.task.failure_reason"], undefined); + strictEqual(span.status.code, SpanStatusCode.UNSET); + strictEqual( + recorder.getMeasurements("fedify.queue.task.failed").length, + 0, + ); + const completed = recorder.getMeasurements("fedify.queue.task.completed"); + strictEqual(completed.length, 1); + strictEqual( + completed[0].attributes["fedify.task.failure_reason"], + undefined, + ); }, ); @@ -1590,6 +1603,49 @@ test("task observability", async (t) => { }, ); + await t.step( + "on a non-native queue an aborted handler that gives up is aborted", + async () => { + const queue = new MockQueue(); + const { federation, recorder, exporter } = instrument({ + ...baseOptions, + queue: { task: queue }, + }); + federation.defineTask("aborts-give-up", { + schema: stringSchema, + handler: () => { + throw globalThis.Object.assign(new Error("shutting down"), { + name: "AbortError", + }); + }, + retryPolicy: () => null, + }); + await federation.processQueuedTask( + undefined, + await makeTaskMessage("aborts-give-up", "payload"), + ); + + strictEqual(queue.enqueued.length, 0); + const span = exporter.getSpans("fedify.task")[0]; + strictEqual(span.attributes["fedify.task.failure_reason"], undefined); + strictEqual(span.status.code, SpanStatusCode.UNSET); + strictEqual( + recorder.getMeasurements("fedify.queue.task.failed").length, + 0, + ); + strictEqual( + recorder.getMeasurements("fedify.queue.task.completed").length, + 0, + ); + const durations = recorder.getMeasurements("fedify.queue.task.duration"); + strictEqual(durations.length, 1); + strictEqual( + durations[0].attributes["fedify.queue.task.result"], + "aborted", + ); + }, + ); + await t.step( "on a non-native queue an aborted handler is retried, not failed", async () => { From c7c512d962738b48b976ca992258cae82cf51da6 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Sat, 4 Jul 2026 10:35:49 +0000 Subject: [PATCH 47/57] Make task enqueue metrics survive partial fan-out The fedify.queue.task.enqueued counter was recorded only after the whole dispatch resolved, so on a queue without enqueueMany a batch fanned out through Promise.all could lose measurements: one rejected enqueue aborted the batch before any recording, leaving messages that had already reached the backend uncounted and skewing the enqueued-versus-completed reconciliation. The fan-out path now uses Promise.allSettled and records each message right after its individual enqueue succeeds, mirroring how the outbox delivery path counts partial successes, and rethrows the first rejection afterwards. The single-message and enqueueMany paths record the whole batch with one counter add carrying the batch size, so recordQueueTaskEnqueued() gains an optional count parameter instead of being called once per identical-attribute message. FederationImpl also gains a metrics getter wrapping the memoized getFederationMetrics() lookup, replacing the scattered per-call-site invocations. https://github.com/fedify-dev/fedify/pull/812#pullrequestreview-4623203224 Assisted-by: Claude Code:claude-fable-5 Assisted-by: Codex:gpt-5.5 --- packages/fedify/src/federation/metrics.ts | 3 +- packages/fedify/src/federation/middleware.ts | 53 +++++------ .../src/federation/tasks/enqueue.test.ts | 89 ++++++++++++++++++- .../fedify/src/federation/tasks/enqueue.ts | 52 +++++++---- 4 files changed, 154 insertions(+), 43 deletions(-) diff --git a/packages/fedify/src/federation/metrics.ts b/packages/fedify/src/federation/metrics.ts index 0b5d2b0aa..838b0bf23 100644 --- a/packages/fedify/src/federation/metrics.ts +++ b/packages/fedify/src/federation/metrics.ts @@ -1011,10 +1011,11 @@ class FederationMetrics { recordQueueTaskEnqueued( common: QueueTaskCommonAttributes, attempt: number, + count = 1, ): void { const attributes = buildQueueTaskAttributes(common); attributes["fedify.queue.task.attempt"] = attempt; - this.queueTaskEnqueued.add(1, attributes); + this.queueTaskEnqueued.add(count, attributes); } recordQueueTaskStarted(common: QueueTaskCommonAttributes): void { diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 433362eb5..35e998441 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -955,6 +955,10 @@ export class FederationImpl return meterProvider; } + get metrics() { + return getFederationMetrics(this.meterProvider); + } + #registerQueueDepthGauge(meterProvider: MeterProvider): void { if (meterProvider === this.#queueDepthGaugeMeterProvider) return; registerQueueDepthGauge(meterProvider, this.#queueDepthGaugeEntries, { @@ -1031,7 +1035,6 @@ export class FederationImpl context.active(), message.traceContext, ); - const meter = getFederationMetrics(this.meterProvider); return withContext({ messageId: message.id }, async () => { if (message.type === "fanout") { const common: QueueTaskCommonAttributes = { @@ -1059,8 +1062,8 @@ export class FederationImpl message.activityId, ); } - meter.recordQueueTaskStarted(common); - meter.incrementQueueTaskInFlight(common); + this.metrics.recordQueueTaskStarted(common); + this.metrics.incrementQueueTaskInFlight(common); const startedAt = performance.now(); let outcome: QueueTaskResult = "completed"; try { @@ -1076,12 +1079,12 @@ export class FederationImpl } throw e; } finally { - meter.recordQueueTaskOutcome( + this.metrics.recordQueueTaskOutcome( common, outcome, getDurationMs(startedAt), ); - meter.decrementQueueTaskInFlight(common); + this.metrics.decrementQueueTaskInFlight(common); span.end(); } }, @@ -1115,8 +1118,8 @@ export class FederationImpl message.activityId, ); } - meter.recordQueueTaskStarted(common); - meter.incrementQueueTaskInFlight(common); + this.metrics.recordQueueTaskStarted(common); + this.metrics.incrementQueueTaskInFlight(common); const startedAt = performance.now(); let outcome: QueueTaskResult = "completed"; try { @@ -1132,12 +1135,12 @@ export class FederationImpl } throw e; } finally { - meter.recordQueueTaskOutcome( + this.metrics.recordQueueTaskOutcome( common, outcome, getDurationMs(startedAt), ); - meter.decrementQueueTaskInFlight(common); + this.metrics.decrementQueueTaskInFlight(common); span.end(); } }, @@ -1163,8 +1166,8 @@ export class FederationImpl return await withContext( { traceId: spanCtx.traceId, spanId: spanCtx.spanId }, async () => { - meter.recordQueueTaskStarted(common); - meter.incrementQueueTaskInFlight(common); + this.metrics.recordQueueTaskStarted(common); + this.metrics.incrementQueueTaskInFlight(common); const startedAt = performance.now(); let outcome: QueueTaskResult = "completed"; try { @@ -1187,12 +1190,12 @@ export class FederationImpl } throw e; } finally { - meter.recordQueueTaskOutcome( + this.metrics.recordQueueTaskOutcome( common, outcome, getDurationMs(startedAt), ); - meter.decrementQueueTaskInFlight(common); + this.metrics.decrementQueueTaskInFlight(common); span.end(); } }, @@ -1221,8 +1224,8 @@ export class FederationImpl return await withContext( { traceId: spanCtx.traceId, spanId: spanCtx.spanId }, async () => { - meter.recordQueueTaskStarted(common); - meter.incrementQueueTaskInFlight(common); + this.metrics.recordQueueTaskStarted(common); + this.metrics.incrementQueueTaskInFlight(common); const startedAt = performance.now(); const recordOutcome = ( outcome: QueueTaskResult, @@ -1239,7 +1242,7 @@ export class FederationImpl ...(error == null ? {} : { message: String(error) }), }); } - meter.recordQueueTaskOutcome( + this.metrics.recordQueueTaskOutcome( common, outcome, getDurationMs(startedAt), @@ -1262,7 +1265,7 @@ export class FederationImpl else recordOutcome("failed", "handler", e); throw e; } finally { - meter.decrementQueueTaskInFlight(common); + this.metrics.decrementQueueTaskInFlight(common); span.end(); } }, @@ -1388,7 +1391,7 @@ export class FederationImpl delay: clampNegativeDelay(delay), orderingKey: message.orderingKey, }); - getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued( + this.metrics.recordQueueTaskEnqueued( { role: "outbox", queue: outboxQueue, @@ -1667,7 +1670,7 @@ export class FederationImpl if ( isPermanentFailure ) { - getFederationMetrics(this.meterProvider).recordPermanentFailure( + this.metrics.recordPermanentFailure( error.inbox, error.statusCode, ); @@ -1767,7 +1770,7 @@ export class FederationImpl orderingKey: message.orderingKey, }, ); - getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued( + this.metrics.recordQueueTaskEnqueued( { role: "outbox", queue: outboxQueue, @@ -1911,7 +1914,7 @@ export class FederationImpl }, ); if (activityType != null) { - getFederationMetrics(this.meterProvider) + this.metrics .recordQueueTaskEnqueued( { role: "inbox", @@ -2135,7 +2138,7 @@ export class FederationImpl activity, ); } finally { - getFederationMetrics(this.meterProvider) + this.metrics .recordInboxProcessingDuration( activityType, getDurationMs(started), @@ -2248,7 +2251,7 @@ export class FederationImpl delay: clampNegativeDelay(delay), orderingKey: message.orderingKey, }); - getFederationMetrics(this.meterProvider).recordQueueTaskEnqueued( + this.metrics.recordQueueTaskEnqueued( { role: "task", queue, taskName: message.taskName }, retryMessage.attempt, ); @@ -2636,7 +2639,7 @@ export class FederationImpl response.headers.set("Vary", "Accept"); } } catch (error) { - getFederationMetrics(this.meterProvider) + this.metrics .recordHttpServerRequest( request.method, metricState.endpoint ?? "error", @@ -2655,7 +2658,7 @@ export class FederationImpl ); throw error; } - getFederationMetrics(this.meterProvider).recordHttpServerRequest( + this.metrics.recordHttpServerRequest( request.method, metricState.endpoint ?? "error", getDurationMs(metricStart), diff --git a/packages/fedify/src/federation/tasks/enqueue.test.ts b/packages/fedify/src/federation/tasks/enqueue.test.ts index 83bfdc70a..4ec9112da 100644 --- a/packages/fedify/src/federation/tasks/enqueue.test.ts +++ b/packages/fedify/src/federation/tasks/enqueue.test.ts @@ -1,4 +1,4 @@ -import { test } from "@fedify/fixture"; +import { createTestMeterProvider, test } from "@fedify/fixture"; import { configure, type LogRecord, reset } from "@logtape/logtape"; import { delay } from "es-toolkit"; import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; @@ -284,6 +284,36 @@ test("enqueueTasks() validation and dispatch", async (t) => { }, ); + await t.step( + "enqueueTaskMany() records one enqueue metric for an enqueueMany batch", + async () => { + const [meterProvider, recorder] = createTestMeterProvider(); + const queue = new MockQueue({ supportsEnqueueMany: true }); + const federation = createFederation({ + ...baseOptions, + meterProvider, + queue: { task: queue }, + }); + const task = federation.defineTask("bulk-metric", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + await ctx.enqueueTaskMany(task, ["a", "b", "c"]); + + const enqueued = recorder.getMeasurements("fedify.queue.task.enqueued"); + strictEqual(enqueued.length, 1); + strictEqual(enqueued[0].value, 3); + strictEqual(enqueued[0].attributes["fedify.queue.role"], "task"); + strictEqual(enqueued[0].attributes["fedify.task.name"], "bulk-metric"); + strictEqual(enqueued[0].attributes["fedify.queue.task.attempt"], 0); + }, + ); + await t.step( "enqueueTaskMany() falls back to parallel enqueues", async () => { @@ -305,6 +335,63 @@ test("enqueueTasks() validation and dispatch", async (t) => { }, ); + await t.step( + "enqueueTaskMany() records fan-out successes before a partial failure", + async () => { + class PartiallyFailingQueue implements MessageQueue { + readonly enqueued: TaskMessage[] = []; + #calls = 0; + + enqueue(message: TaskMessage): Promise { + this.#calls++; + if (this.#calls === 2) { + return Promise.reject(new Error("second enqueue failed")); + } + this.enqueued.push(message); + return Promise.resolve(); + } + + listen(): Promise { + return Promise.resolve(); + } + } + + const [meterProvider, recorder] = createTestMeterProvider(); + const queue = new PartiallyFailingQueue(); + const federation = createFederation({ + ...baseOptions, + meterProvider, + queue: { task: queue }, + }); + const task = federation.defineTask("partial-fanout", { + schema: stringSchema, + handler: () => {}, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + undefined, + ); + + await rejects( + () => ctx.enqueueTaskMany(task, ["a", "b", "c"]), + { message: /second enqueue failed/ }, + ); + + strictEqual(queue.enqueued.length, 2); + const enqueued = recorder.getMeasurements("fedify.queue.task.enqueued"); + strictEqual(enqueued.length, 2); + for (const measurement of enqueued) { + strictEqual(measurement.value, 1); + strictEqual(measurement.attributes["fedify.queue.role"], "task"); + strictEqual( + measurement.attributes["fedify.task.name"], + "partial-fanout", + ); + strictEqual(measurement.attributes["fedify.queue.task.attempt"], 0); + } + }, + ); + await t.step( "enqueueTaskMany() with no payloads touches no queue", async () => { diff --git a/packages/fedify/src/federation/tasks/enqueue.ts b/packages/fedify/src/federation/tasks/enqueue.ts index acdd9dae0..5531f2e2f 100644 --- a/packages/fedify/src/federation/tasks/enqueue.ts +++ b/packages/fedify/src/federation/tasks/enqueue.ts @@ -7,7 +7,7 @@ * @module */ import { getLogger } from "@logtape/logtape"; -import { context, propagation } from "@opentelemetry/api"; +import { context, type MeterProvider, propagation } from "@opentelemetry/api"; import type { KvKey } from "../kv.ts"; import { getFederationMetrics } from "../metrics.ts"; import type { FederationImpl } from "../middleware.ts"; @@ -96,20 +96,19 @@ const enqueueTasks = ( ctx.federation._startQueueInternal(ctx.data); } try { - await dispatch(queue, messages, { - delay: getDurationIfDefined(options.delay), - orderingKey: options.orderingKey, - deduplicationKey: claim.forwardedDeduplicationKey, - }); - // Counted only after a genuine dispatch: a dedup skip returns before - // this, and a failed dispatch throws into the rollback below. - const meter = getFederationMetrics(ctx.federation.meterProvider); - for (const message of messages) { - meter.recordQueueTaskEnqueued( - { role: "task", queue, taskName: task.name }, - message.attempt, - ); - } + await dispatch( + queue, + messages, + { + delay: getDurationIfDefined(options.delay), + orderingKey: options.orderingKey, + deduplicationKey: claim.forwardedDeduplicationKey, + }, + { + meterProvider: ctx.federation.meterProvider, + taskName: task.name, + }, + ); } catch (error) { if (claim.rollback != null) { try { @@ -261,7 +260,12 @@ async function dispatch( orderingKey?: string; deduplicationKey?: string; }, + { meterProvider, taskName }: { + meterProvider: MeterProvider | undefined; + taskName: string; + }, ): Promise { + const metrics = getFederationMetrics(meterProvider); if (messages.length === 1) { await queue.enqueue(messages[0], options); } else if (queue.enqueueMany != null) { @@ -271,8 +275,24 @@ async function dispatch( delay: options.delay, orderingKey: options.orderingKey, }; - await Promise.all(messages.map((m) => queue.enqueue(m, fanoutOptions))); + const settled = await Promise.allSettled( + messages.map(async (message) => { + await queue.enqueue(message, fanoutOptions); + metrics.recordQueueTaskEnqueued( + { role: "task", queue, taskName }, + message.attempt, + ); + }), + ); + const rejected = settled.find((result) => result.status === "rejected"); + if (rejected != null) throw rejected.reason; + return; } + metrics.recordQueueTaskEnqueued( + { role: "task", queue, taskName }, + messages[0].attempt, + messages.length, + ); } /** From f590b309cb8d0a5038f1abe69733a2d4db90a255 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Sat, 4 Jul 2026 10:44:24 +0000 Subject: [PATCH 48/57] Fix docs and versions Version errors are found by `Claude Code:fable-5` https://github.com/fedify-dev/fedify/pull/812#discussion_r3517806147 --- CHANGES.md | 2 +- docs/manual/tasks.md | 2 +- packages/fedify/src/federation/metrics.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b231e61ee..ec5fcb113 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,7 +45,7 @@ Version 2.4.0 `FederationOptions.taskDeduplicationTtl` and `FederationOptions.taskDeduplicationFallback` options. - [[#206], [#797], [#798], [#799] [#803], [#806], [#812] by ChanHaeng Lee] + [[#206], [#797], [#798], [#799], [#803], [#806], [#812] by ChanHaeng Lee] [Standard Schema]: https://standardschema.dev/ [#206]: https://github.com/fedify-dev/fedify/issues/206 diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index 96c90fabe..df9e9ea9b 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -352,7 +352,7 @@ collapsed onto one message. Observability ------------- -*Task-specific telemetry is available since Fedify 2.3.0.* +*Task-specific telemetry is available since Fedify 2.4.0.* Each task the worker dequeues runs inside a `fedify.task` [OpenTelemetry] span (a *consumer* span, since tasks are not part of ActivityPub it is namespaced diff --git a/packages/fedify/src/federation/metrics.ts b/packages/fedify/src/federation/metrics.ts index 838b0bf23..80a067893 100644 --- a/packages/fedify/src/federation/metrics.ts +++ b/packages/fedify/src/federation/metrics.ts @@ -95,7 +95,7 @@ export interface QueueTaskCommonAttributes { /** * The registered name of a custom background task, emitted as the * `fedify.task.name` attribute. Set only for the `"task"` role. - * @since 2.3.0 + * @since 2.4.0 */ taskName?: string; } @@ -225,7 +225,7 @@ export type HttpSignatureMetricFailureReason = * - `validation`: the deserialized payload failed schema validation. * - `unknown_task`: the task name has no registered handler. * - `handler`: the registered handler threw. - * @since 2.3.0 + * @since 2.4.0 */ export type QueueTaskFailureReason = | "deserialization" From 728d4809cde7a75cc7871c021bd1c0085dbfff4b Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Mon, 6 Jul 2026 11:14:49 +0000 Subject: [PATCH 49/57] Clarify result type of `FederationImpl.metrics` --- packages/fedify/src/federation/middleware.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 35e998441..048d5c70b 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -955,7 +955,7 @@ export class FederationImpl return meterProvider; } - get metrics() { + get metrics(): ReturnType { return getFederationMetrics(this.meterProvider); } From 89a1a674f755c53854c0abc33baac26abff12bf5 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Mon, 6 Jul 2026 16:46:47 +0000 Subject: [PATCH 50/57] Clarify task telemetry semantics in docs and code Addresses maintainer review feedback on the custom-task observability additions. All four changes clarify existing semantics; none change telemetry behavior. - The `fedify.queue.task.failed` metric summary said tasks are counted as failed "because processing threw," which excluded the dispatch-stage drops (`deserialization`, `validation`, `unknown_task`) that are recorded as failed without the handler throwing. The summary now covers both a processing throw and a dispatch drop. - The `fedify.queue.task.enqueued` counter is emitted at the enqueue site, not during the worker run, so grouping it under "task run measurement" was misleading. The docs now state that `enqueued` is recorded at the enqueue site while `started`, `completed`, `failed`, and `duration` are recorded during the worker run. - A short operational example now precedes the OpenTelemetry cross-reference, showing how to split schema drift from handler give-ups via `fedify.task.failure_reason` and how to spot retry re-enqueues via `fedify.queue.task.attempt > 0`. - The `completed` outcome returned after scheduling a retry now carries a comment explaining it means "the attempt was folded into a retry," not "the handler succeeded," to keep a future change from regressing the terminal-failure-only task telemetry. https://github.com/fedify-dev/fedify/pull/812#discussion_r3529901316 https://github.com/fedify-dev/fedify/pull/812#discussion_r3529901321 https://github.com/fedify-dev/fedify/pull/812#discussion_r3529901329 https://github.com/fedify-dev/fedify/pull/812#discussion_r3529941552 Assisted-by: Claude Code:claude-opus-4-8 --- docs/manual/opentelemetry.md | 2 +- docs/manual/tasks.md | 19 +++++++++++++++---- packages/fedify/src/federation/middleware.ts | 6 ++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index 78fff942c..dbeac20fd 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -368,7 +368,7 @@ Fedify records the following OpenTelemetry metrics: | `fedify.queue.task.enqueued` | Counter | `{task}` | Counts inbox, outbox, fanout, and custom background tasks Fedify enqueued. | | `fedify.queue.task.started` | Counter | `{task}` | Counts queue tasks Fedify began processing as a worker. | | `fedify.queue.task.completed` | Counter | `{task}` | Counts queue tasks Fedify finished processing without throwing. | -| `fedify.queue.task.failed` | Counter | `{task}` | Counts queue tasks Fedify abandoned because processing threw. | +| `fedify.queue.task.failed` | Counter | `{task}` | Counts queue tasks Fedify recorded as failed after processing or dispatch did not succeed. | | `fedify.queue.task.duration` | Histogram | `ms` | Measures queue task processing duration in Fedify workers. | | `fedify.queue.task.in_flight` | UpDownCounter | `{task}` | Tracks queue tasks currently in flight in this Fedify process. | | `fedify.queue.depth` | Gauge | `{message}` | Reports queued, ready, and delayed queue depth when the queue backend supports it. | diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index df9e9ea9b..42796ed3d 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -375,11 +375,13 @@ failure. Tasks also reuse the `fedify.queue.task.*` metric family (`enqueued`, `started`, `completed`, `failed`, `duration`, `in_flight`) that the inbox, -outbox, and fanout workers already report. On a task run measurement -(`enqueued`, `started`, `completed`, `failed`, `duration`), +outbox, and fanout workers already report. Across these measurements `fedify.queue.role` is `task` and `fedify.task.name` names the task; the process-local `in_flight` UpDownCounter omits `fedify.task.name` so its -increments and decrements pair up cleanly. +increments and decrements pair up cleanly. `fedify.queue.task.enqueued` is +recorded at the enqueue site—both the initial enqueue and a retry +re-enqueue—whereas `started`, `completed`, `failed`, and `duration` are +recorded during the worker run. `fedify.queue.backend` reflects the queue actually used after routing—so a task that falls back to the `outboxQueue` (see [Routing](#queue-routing-and-isolation)) is labeled with the outbox queue's @@ -414,7 +416,16 @@ The bounded value set keeps metric cardinality finite: a metric's task name is a registered, known-at-startup value, never derived from message content—an `unknown_task` drop carries a wire-supplied name, so that name is kept off the metrics (it still appears on the span, which does not aggregate into time -series). See the [OpenTelemetry](./opentelemetry.md) manual for the full span, +series). + +For example, grouping `fedify.queue.task.failed` by +`fedify.task.failure_reason` separates schema drift +(`deserialization`/`validation` drops) from `handler` give-ups. Or, filtering +`fedify.queue.task.enqueued` for `fedify.queue.task.attempt > 0` (or reading +`fedify.task.attempt` on the `fedify.task` span) surfaces retry re-enqueues +before they become give-ups. + +See the [OpenTelemetry](./opentelemetry.md) manual for the full span, attribute, and metric reference. [OpenTelemetry]: https://opentelemetry.io/ diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 048d5c70b..551ae5ff4 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -2255,6 +2255,12 @@ export class FederationImpl { role: "task", queue, taskName: message.taskName }, retryMessage.attempt, ); + // `completed` here means the attempt was folded into a scheduled retry, + // not that the handler succeeded; only a terminal give-up records + // `failed`. This mirrors the inbox/outbox worker-boundary convention— + // do not change it to `failed` without regressing terminal-failure-only + // task telemetry. + // See also: https://fedify.dev/manual/tasks#observability return { outcome: "completed" }; } else { logger.error( From b48b549ef805d95c33380040dcde501e58547fef Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Mon, 6 Jul 2026 17:45:33 +0000 Subject: [PATCH 51/57] Distinguish task retry re-enqueue failures When a custom task's handler failed and a retry was scheduled, but re-enqueuing the retry message to the queue backend itself threw, the worker boundary recorded `fedify.task.failure_reason` as `handler` and re-threw the enqueue error, shadowing the original handler error. An infrastructure failure was thus indistinguishable from a handler fault, and the documented failure reasons map to dispatch decision points, none of which is a re-enqueue failure. Because the failure-reason set is a small bounded set that is open to refinement, this adds a distinct `retry_enqueue` value rather than folding the case into `handler`. One extra value keeps the metric cardinality bounded (`taskName x |failure_reason| x backend`). - `QueueTaskFailureReason` gains `retry_enqueue`. - The retry re-enqueue is now wrapped narrowly; on failure the worker logs it and re-throws a `TaskRetryEnqueueError` that preserves the original handler error as its `cause`. The throw keeps the message nacked rather than dropped, and the worker boundary records `retry_enqueue` with the handler error surfaced on the span. - `TaskRetryEnqueueError` and `QueueTaskDispatchResult` move out of middleware.ts into dedicated `tasks/error.ts` and `tasks/types.ts` modules, re-exported through the task barrel. - The tasks and OpenTelemetry manuals document the new value, and a worker test covers the re-enqueue-failure path. Assisted-by: Claude Code:claude-opus-4-8 --- docs/manual/opentelemetry.md | 155 +++++++++--------- docs/manual/tasks.md | 13 +- packages/fedify/src/federation/metrics.ts | 8 +- packages/fedify/src/federation/middleware.ts | 34 ++-- packages/fedify/src/federation/tasks/error.ts | 6 + packages/fedify/src/federation/tasks/mod.ts | 2 + .../fedify/src/federation/tasks/tasks.test.ts | 56 +++++++ packages/fedify/src/federation/tasks/types.ts | 8 + 8 files changed, 189 insertions(+), 93 deletions(-) create mode 100644 packages/fedify/src/federation/tasks/error.ts create mode 100644 packages/fedify/src/federation/tasks/types.ts diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index dbeac20fd..22114d0f9 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -817,7 +817,7 @@ Fedify records the following OpenTelemetry metrics: registered names. A failed task outcome (`fedify.queue.task.result=failed`) additionally carries `fedify.task.failure_reason`, one of `deserialization`, `validation`, - `unknown_task`, or `handler`. + `unknown_task`, `handler`, or `retry_enqueue`. `fedify.queue.native_retrial` reflects the queue backend's `nativeRetrial` flag when set on the queue. `activitypub.activity.type` is recorded whenever Fedify knows the activity type for the queued message; for inbox @@ -835,7 +835,9 @@ Fedify records the following OpenTelemetry metrics: is recorded as `failed` with a `fedify.task.failure_reason`, or the handler threw and the retry policy declined another attempt, recorded with the `handler` reason—a handler error folded into a scheduled retry - records `completed` instead), and `aborted` when the worker re-threw an + records `completed` instead, unless re-enqueuing that retry itself fails, + which records `failed` with the `retry_enqueue` reason and nacks the + message), and `aborted` when the worker re-threw an `AbortError` (for example, because a graceful-shutdown `AbortSignal` interrupted processing) or when an aborted custom task attempt was abandoned without a retry. When the queue backend does not declare @@ -960,79 +962,82 @@ for ActivityPub as of November 2024. However, Fedify provides a set of semantic [attributes] for ActivityPub. The following table shows the semantic attributes for ActivityPub: -| Attribute | Type | Description | Example | -| -------------------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | -| `activitypub.activity.id` | string | The URI of the activity object. | `"https://example.com/activity/1"` | -| `activitypub.activity.type` | string[] | The qualified URI(s) of the activity type(s). | `["https://www.w3.org/ns/activitystreams#Create"]` | -| `activitypub.activity.to` | string[] | The URI(s) of the recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | -| `activitypub.activity.cc` | string[] | The URI(s) of the carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | -| `activitypub.activity.bto` | string[] | The URI(s) of the blind recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | -| `activitypub.activity.bcc` | string[] | The URI(s) of the blind carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | -| `activitypub.activity.retries` | int | The ordinal number of activity resending attempt (if and only if it's retried). | `3` | -| `activitypub.delivery.attempt` | int | The zero-based delivery attempt number for a queued outgoing activity. | `0` | -| `activitypub.delivery.permanent_failure` | boolean | Whether an outgoing delivery failure will be abandoned instead of retried. | `true` | -| `activitypub.circuit_breaker.previous_state` | string | Previous queued outbox circuit breaker state: `closed`, `open`, or `half_open`. | `"closed"` | -| `activitypub.circuit_breaker.state` | string | Current queued outbox circuit breaker state: `closed`, `open`, or `half_open`. | `"open"` | -| `activitypub.processing.result` | string | Lifecycle outcome of an inbox or outbox activity: `queued`, `processed`, `retried`, `rejected`, or `abandoned`. | `"retried"` | -| `activitypub.actor.discovery.result` | string | Terminal outcome of `getActorHandle()`: `resolved`, `not_found`, or `error`. | `"resolved"` | -| `activitypub.actor.id` | string | The URI of the actor object. | `"https://example.com/actor/1"` | -| `activitypub.actor.key.cached` | boolean | Whether the actor's public keys are cached. | `true` | -| `activitypub.actor.type` | string[] | The qualified URI(s) of the actor type(s). | `["https://www.w3.org/ns/activitystreams#Person"]` | -| `activitypub.key.id` | string | The URI of the cryptographic key being verified. | `"https://example.com/actor/1#main-key"` | -| `activitypub.key_ownership.method` | string | The method used to verify key ownership (`owner_id` or `actor_fetch`). | `"actor_fetch"` | -| `activitypub.key_ownership.verified` | boolean | Whether the key ownership was successfully verified. | `true` | -| `activitypub.collection.id` | string | The URI of the collection object. | `"https://example.com/collection/1"` | -| `activitypub.collection.kind` | string | The bounded collection kind: `inbox`, `outbox`, `following`, `followers`, `liked`, `featured`, `featured_tags`, or `custom`. | `"followers"` | -| `activitypub.collection.page` | boolean | Whether the collection request targets a cursor page rather than the collection object. | `false` | -| `activitypub.collection.result` | string | Terminal collection request outcome: `served`, `not_found`, `not_acceptable`, `unauthorized`, or `error`. | `"served"` | -| `activitypub.collection.type` | string[] | The qualified URI(s) of the collection type(s). | `["https://www.w3.org/ns/activitystreams#OrderedCollection"]` | -| `activitypub.collection.total_items` | int | The total number of items in the collection. | `42` | -| `activitypub.object.id` | string | The URI of the object or the object enclosed by the activity. | `"https://example.com/object/1"` | -| `activitypub.object.type` | string[] | The qualified URI(s) of the object type(s). | `["https://www.w3.org/ns/activitystreams#Note"]` | -| `activitypub.object.in_reply_to` | string[] | The URI(s) of the original object to which the object reply. | `["https://example.com/object/1"]` | -| `activitypub.inboxes` | int | The number of inboxes the activity is sent to. | `12` | -| `activitypub.remote.host` | string | The host of the remote ActivityPub server, including any non-default port. | `"example.com:8443"` | -| `activitypub.shared_inbox` | boolean | Whether the activity is sent to the shared inbox. | `true` | -| `docloader.context_url` | string | The URL of the JSON-LD context document (if provided via Link header). | `"https://www.w3.org/ns/activitystreams"` | -| `docloader.document_url` | string | The final URL of the fetched document (after following redirects). | `"https://example.com/object/1"` | -| `fedify.actor.identifier` | string | The identifier of the actor. | `"1"` | -| `fedify.endpoint` | string | The bounded endpoint category that classified an inbound HTTP request handled by `Federation.fetch()`. | `"actor"` | -| `fedify.federation.instance_id` | string | Opaque per-Federation instance identifier used to distinguish queue depth series on a shared `MeterProvider`. | `"fedify-1"` | -| `fedify.route.template` | string | The matched URI Template, with parameter names (not values). | `"/users/{identifier}"` | -| `fedify.inbox.recipient` | string | The identifier of the inbox recipient. | `"1"` | -| `fedify.object.type` | string | The URI of the object type. | `"https://www.w3.org/ns/activitystreams#Note"` | -| `fedify.object.values.{parameter}` | string[] | The argument values of the object dispatcher. | `["1", "2"]` | -| `fedify.collection.dispatcher` | string | The collection dispatcher family: `built_in` or `custom`. | `"built_in"` | -| `fedify.collection.cursor` | string | The cursor of the collection. | `"eyJpZCI6IjEiLCJ0eXBlIjoiT3JkZXJlZENvbGxlY3Rpb24ifQ=="` | -| `fedify.collection.items` | number | The number of materialized items in the collection response or page. It can be less than the total items. | `10` | -| `fedify.queue.role` | string | The Fedify queue role: `inbox`, `outbox`, `fanout`, or `shared` for queue depth rows where one queue backs multiple roles. | `"outbox"` | -| `fedify.queue.backend` | string | The queue implementation's constructor name (best-effort backend identifier). | `"RedisMessageQueue"` | -| `fedify.queue.native_retrial` | boolean | Whether the queue backend declares `nativeRetrial`, meaning Fedify defers retry handling to the backend. | `true` | -| `fedify.queue.depth.state` | string | Queue depth count kind: `queued`, `ready`, or `delayed`. | `"queued"` | -| `fedify.queue.roles` | string | Comma-separated queue roles when one queue instance backs multiple roles. | `"fanout,inbox,outbox"` | -| `fedify.queue.task.attempt` | int | The zero-based attempt number recorded on `fedify.queue.task.enqueued`; non-zero for retry re-enqueues. | `1` | -| `fedify.queue.task.result` | string | The terminal outcome of queue task processing: `completed`, `failed`, or `aborted`. | `"failed"` | -| `http.redirect.url` | string | The redirect URL when a document fetch results in a redirect. | `"https://example.com/new-location"` | -| `http.response.status_code` | int | The HTTP response status code. | `200` | -| `http_signatures.signature` | string | The signature of the HTTP request in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | -| `http_signatures.algorithm` | string | The algorithm of the HTTP request signature. | `"rsa-sha256"` | -| `http_signatures.key_id` | string | The public key ID of the HTTP request signature. | `"https://example.com/actor/1#main-key"` | -| `http_signatures.verified` | boolean | Whether the HTTP request signature was verified successfully. | `false` | -| `http_signatures.failure_reason` | string | Why HTTP signature verification failed (`noSignature`, `invalidSignature`, or `keyFetchError`). | `"keyFetchError"` | -| `http_signatures.key_fetch_status` | int | The HTTP status code from a failed signing-key fetch, when available. | `410` | -| `http_signatures.key_fetch_error` | string | The error type from a non-HTTP signing-key fetch failure, when available. | `"TypeError"` | -| `http_signatures.digest.{algorithm}` | string | The digest of the HTTP request body in hexadecimal. The `{algorithm}` is the digest algorithm (e.g., `sha`, `sha-256`). | `"d41d8cd98f00b204e9800998ecf8427e"` | -| `ld_signatures.key_id` | string | The public key ID of the Linked Data signature. | `"https://example.com/actor/1#main-key"` | -| `ld_signatures.signature` | string | The signature of the Linked Data in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | -| `ld_signatures.type` | string | The algorithm of the Linked Data signature. | `"RsaSignature2017"` | -| `object_integrity_proofs.cryptosuite` | string | The cryptographic suite of the object integrity proof. | `"eddsa-jcs-2022"` | -| `object_integrity_proofs.key_id` | string | The public key ID of the object integrity proof. | `"https://example.com/actor/1#main-key"` | -| `object_integrity_proofs.signature` | string | The integrity proof of the object in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | -| `url.full` | string | The full URL being fetched by the document loader. | `"https://example.com/actor/1"` | -| `webfinger.handle.result` | string | Terminal outcome of an incoming WebFinger request: `resolved`, `invalid`, `not_found`, `tombstoned`, or `error`. | `"resolved"` | -| `webfinger.lookup.result` | string | Terminal outcome of an outgoing WebFinger lookup: `found`, `not_found`, `invalid`, `network_error`, or `error`. | `"found"` | -| `webfinger.resource` | string | The queried resource URI. | `"acct:fedify@hollo.social"` | -| `webfinger.resource.scheme` | string | The scheme of the queried resource URI. Metric attribute is bucketed to `acct`, `http`, `https`, `mailto`, or `other`. | `"acct"` | +| Attribute | Type | Description | Example | +| -------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | +| `activitypub.activity.id` | string | The URI of the activity object. | `"https://example.com/activity/1"` | +| `activitypub.activity.type` | string[] | The qualified URI(s) of the activity type(s). | `["https://www.w3.org/ns/activitystreams#Create"]` | +| `activitypub.activity.to` | string[] | The URI(s) of the recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | +| `activitypub.activity.cc` | string[] | The URI(s) of the carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | +| `activitypub.activity.bto` | string[] | The URI(s) of the blind recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | +| `activitypub.activity.bcc` | string[] | The URI(s) of the blind carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | +| `activitypub.activity.retries` | int | The ordinal number of activity resending attempt (if and only if it's retried). | `3` | +| `activitypub.delivery.attempt` | int | The zero-based delivery attempt number for a queued outgoing activity. | `0` | +| `activitypub.delivery.permanent_failure` | boolean | Whether an outgoing delivery failure will be abandoned instead of retried. | `true` | +| `activitypub.circuit_breaker.previous_state` | string | Previous queued outbox circuit breaker state: `closed`, `open`, or `half_open`. | `"closed"` | +| `activitypub.circuit_breaker.state` | string | Current queued outbox circuit breaker state: `closed`, `open`, or `half_open`. | `"open"` | +| `activitypub.processing.result` | string | Lifecycle outcome of an inbox or outbox activity: `queued`, `processed`, `retried`, `rejected`, or `abandoned`. | `"retried"` | +| `activitypub.actor.discovery.result` | string | Terminal outcome of `getActorHandle()`: `resolved`, `not_found`, or `error`. | `"resolved"` | +| `activitypub.actor.id` | string | The URI of the actor object. | `"https://example.com/actor/1"` | +| `activitypub.actor.key.cached` | boolean | Whether the actor's public keys are cached. | `true` | +| `activitypub.actor.type` | string[] | The qualified URI(s) of the actor type(s). | `["https://www.w3.org/ns/activitystreams#Person"]` | +| `activitypub.key.id` | string | The URI of the cryptographic key being verified. | `"https://example.com/actor/1#main-key"` | +| `activitypub.key_ownership.method` | string | The method used to verify key ownership (`owner_id` or `actor_fetch`). | `"actor_fetch"` | +| `activitypub.key_ownership.verified` | boolean | Whether the key ownership was successfully verified. | `true` | +| `activitypub.collection.id` | string | The URI of the collection object. | `"https://example.com/collection/1"` | +| `activitypub.collection.kind` | string | The bounded collection kind: `inbox`, `outbox`, `following`, `followers`, `liked`, `featured`, `featured_tags`, or `custom`. | `"followers"` | +| `activitypub.collection.page` | boolean | Whether the collection request targets a cursor page rather than the collection object. | `false` | +| `activitypub.collection.result` | string | Terminal collection request outcome: `served`, `not_found`, `not_acceptable`, `unauthorized`, or `error`. | `"served"` | +| `activitypub.collection.type` | string[] | The qualified URI(s) of the collection type(s). | `["https://www.w3.org/ns/activitystreams#OrderedCollection"]` | +| `activitypub.collection.total_items` | int | The total number of items in the collection. | `42` | +| `activitypub.object.id` | string | The URI of the object or the object enclosed by the activity. | `"https://example.com/object/1"` | +| `activitypub.object.type` | string[] | The qualified URI(s) of the object type(s). | `["https://www.w3.org/ns/activitystreams#Note"]` | +| `activitypub.object.in_reply_to` | string[] | The URI(s) of the original object to which the object reply. | `["https://example.com/object/1"]` | +| `activitypub.inboxes` | int | The number of inboxes the activity is sent to. | `12` | +| `activitypub.remote.host` | string | The host of the remote ActivityPub server, including any non-default port. | `"example.com:8443"` | +| `activitypub.shared_inbox` | boolean | Whether the activity is sent to the shared inbox. | `true` | +| `docloader.context_url` | string | The URL of the JSON-LD context document (if provided via Link header). | `"https://www.w3.org/ns/activitystreams"` | +| `docloader.document_url` | string | The final URL of the fetched document (after following redirects). | `"https://example.com/object/1"` | +| `fedify.actor.identifier` | string | The identifier of the actor. | `"1"` | +| `fedify.endpoint` | string | The bounded endpoint category that classified an inbound HTTP request handled by `Federation.fetch()`. | `"actor"` | +| `fedify.federation.instance_id` | string | Opaque per-Federation instance identifier used to distinguish queue depth series on a shared `MeterProvider`. | `"fedify-1"` | +| `fedify.route.template` | string | The matched URI Template, with parameter names (not values). | `"/users/{identifier}"` | +| `fedify.inbox.recipient` | string | The identifier of the inbox recipient. | `"1"` | +| `fedify.object.type` | string | The URI of the object type. | `"https://www.w3.org/ns/activitystreams#Note"` | +| `fedify.object.values.{parameter}` | string[] | The argument values of the object dispatcher. | `["1", "2"]` | +| `fedify.collection.dispatcher` | string | The collection dispatcher family: `built_in` or `custom`. | `"built_in"` | +| `fedify.collection.cursor` | string | The cursor of the collection. | `"eyJpZCI6IjEiLCJ0eXBlIjoiT3JkZXJlZENvbGxlY3Rpb24ifQ=="` | +| `fedify.collection.items` | number | The number of materialized items in the collection response or page. It can be less than the total items. | `10` | +| `fedify.queue.role` | string | The Fedify queue role: `inbox`, `outbox`, `fanout`, or `shared` for queue depth rows where one queue backs multiple roles. | `"outbox"` | +| `fedify.queue.backend` | string | The queue implementation's constructor name (best-effort backend identifier). | `"RedisMessageQueue"` | +| `fedify.queue.native_retrial` | boolean | Whether the queue backend declares `nativeRetrial`, meaning Fedify defers retry handling to the backend. | `true` | +| `fedify.queue.depth.state` | string | Queue depth count kind: `queued`, `ready`, or `delayed`. | `"queued"` | +| `fedify.queue.roles` | string | Comma-separated queue roles when one queue instance backs multiple roles. | `"fanout,inbox,outbox"` | +| `fedify.queue.task.attempt` | int | The zero-based attempt number recorded on `fedify.queue.task.enqueued`; non-zero for retry re-enqueues. | `1` | +| `fedify.queue.task.result` | string | The terminal outcome of queue task processing: `completed`, `failed`, or `aborted`. | `"failed"` | +| `fedify.task.name` | string | The name of a custom background task: always on the `fedify.task` span; on the task's `fedify.queue.task.*` run metrics only for a registered task (omitted for an `unknown_task` drop, keeping cardinality bounded). | `"sendDigest"` | +| `fedify.task.attempt` | int | The zero-based attempt number of a custom background task, on the `fedify.task` span. | `0` | +| `fedify.task.failure_reason` | string | Why a custom background task failed: `deserialization`, `validation`, `unknown_task`, `handler`, or `retry_enqueue`. Set only on a terminal failure. | `"validation"` | +| `http.redirect.url` | string | The redirect URL when a document fetch results in a redirect. | `"https://example.com/new-location"` | +| `http.response.status_code` | int | The HTTP response status code. | `200` | +| `http_signatures.signature` | string | The signature of the HTTP request in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | +| `http_signatures.algorithm` | string | The algorithm of the HTTP request signature. | `"rsa-sha256"` | +| `http_signatures.key_id` | string | The public key ID of the HTTP request signature. | `"https://example.com/actor/1#main-key"` | +| `http_signatures.verified` | boolean | Whether the HTTP request signature was verified successfully. | `false` | +| `http_signatures.failure_reason` | string | Why HTTP signature verification failed (`noSignature`, `invalidSignature`, or `keyFetchError`). | `"keyFetchError"` | +| `http_signatures.key_fetch_status` | int | The HTTP status code from a failed signing-key fetch, when available. | `410` | +| `http_signatures.key_fetch_error` | string | The error type from a non-HTTP signing-key fetch failure, when available. | `"TypeError"` | +| `http_signatures.digest.{algorithm}` | string | The digest of the HTTP request body in hexadecimal. The `{algorithm}` is the digest algorithm (e.g., `sha`, `sha-256`). | `"d41d8cd98f00b204e9800998ecf8427e"` | +| `ld_signatures.key_id` | string | The public key ID of the Linked Data signature. | `"https://example.com/actor/1#main-key"` | +| `ld_signatures.signature` | string | The signature of the Linked Data in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | +| `ld_signatures.type` | string | The algorithm of the Linked Data signature. | `"RsaSignature2017"` | +| `object_integrity_proofs.cryptosuite` | string | The cryptographic suite of the object integrity proof. | `"eddsa-jcs-2022"` | +| `object_integrity_proofs.key_id` | string | The public key ID of the object integrity proof. | `"https://example.com/actor/1#main-key"` | +| `object_integrity_proofs.signature` | string | The integrity proof of the object in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | +| `url.full` | string | The full URL being fetched by the document loader. | `"https://example.com/actor/1"` | +| `webfinger.handle.result` | string | Terminal outcome of an incoming WebFinger request: `resolved`, `invalid`, `not_found`, `tombstoned`, or `error`. | `"resolved"` | +| `webfinger.lookup.result` | string | Terminal outcome of an outgoing WebFinger lookup: `found`, `not_found`, `invalid`, `network_error`, or `error`. | `"found"` | +| `webfinger.resource` | string | The queried resource URI. | `"acct:fedify@hollo.social"` | +| `webfinger.resource.scheme` | string | The scheme of the queried resource URI. Metric attribute is bucketed to `acct`, `http`, `https`, `mailto`, or `other`. | `"acct"` | [attributes]: https://opentelemetry.io/docs/specs/otel/common/#attribute [OpenTelemetry Semantic Conventions]: https://opentelemetry.io/docs/specs/semconv/ diff --git a/docs/manual/tasks.md b/docs/manual/tasks.md index 42796ed3d..20517f3fe 100644 --- a/docs/manual/tasks.md +++ b/docs/manual/tasks.md @@ -365,7 +365,7 @@ parent. The span carries: - `fedify.task.attempt` — the zero-based attempt number; a retry re-enqueue increments it. - `fedify.task.failure_reason` — set only on a terminal failure, one of the - four bounded values below. + five bounded values below. On a terminal failure the span's status is also set to `ERROR`, so trace-based error views surface dropped and given-up tasks together with their @@ -389,8 +389,8 @@ backend, not a task queue's. A failed outcome also carries `fedify.task.failure_reason` on `fedify.queue.task.failed` and `fedify.queue.task.duration`. -The `fedify.task.failure_reason` attribute takes one of four bounded values, -mapping to the worker's dispatch decision points: +The `fedify.task.failure_reason` attribute takes one of five bounded values, +mapping to the worker's dispatch and retry-scheduling decision points: | Value | Meaning | | ----------------- | -------------------------------------------------- | @@ -398,6 +398,7 @@ mapping to the worker's dispatch decision points: | `validation` | The deserialized payload failed schema validation. | | `unknown_task` | The task name has no registered handler. | | `handler` | The registered handler threw. | +| `retry_enqueue` | A retry was scheduled, but re-enqueuing it failed. | The first three are *drops*: the payload cannot succeed by retrying, so the worker acknowledges the message and does not re-enqueue it. Telemetry still @@ -406,7 +407,11 @@ left drained—so a drop is observable without being retried. A `handler` failure follows the configured retry policy (see [Retries](#retry-and-error-handling)): an attempt folded into a scheduled retry records a `completed` outcome, and only the terminal give-up records -`failed` with the `handler` reason. A worker shutdown is never counted as a +`failed` with the `handler` reason. If a retry *is* scheduled but re-enqueuing +the retry message to the queue backend fails, the worker instead records +`failed` with the `retry_enqueue` reason and re-throws (so the message is +nacked, not dropped), preserving the original handler error as the thrown +error's cause. A worker shutdown is never counted as a failure: an interrupted attempt carries no `fedify.task.failure_reason`—it is recorded as an `aborted` outcome when the abort propagates (on a `nativeRetrial` queue) or when the retry policy declines another attempt, and diff --git a/packages/fedify/src/federation/metrics.ts b/packages/fedify/src/federation/metrics.ts index 80a067893..248e1977b 100644 --- a/packages/fedify/src/federation/metrics.ts +++ b/packages/fedify/src/federation/metrics.ts @@ -219,19 +219,23 @@ export type HttpSignatureMetricFailureReason = /** * The reason a custom background task terminated unsuccessfully, emitted as the * `fedify.task.failure_reason` attribute. A small bounded set mapping to the - * worker's dispatch decision points; open to later refinement. + * worker's dispatch and retry-scheduling decision points; open to later + * refinement. * * - `deserialization`: the wire payload could not be deserialized. * - `validation`: the deserialized payload failed schema validation. * - `unknown_task`: the task name has no registered handler. * - `handler`: the registered handler threw. + * - `retry_enqueue`: the handler threw and a retry was scheduled, but + * re-enqueuing the retry to the queue backend failed. * @since 2.4.0 */ export type QueueTaskFailureReason = | "deserialization" | "validation" | "unknown_task" - | "handler"; + | "handler" + | "retry_enqueue"; /** * Bounded values recorded as `ld_signatures.type` on the signature diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 551ae5ff4..5bf42114d 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -170,20 +170,15 @@ import { } from "./send.ts"; import { enqueueTasks, + type QueueTaskDispatchResult, TaskCodec, type TaskDefinition, type TaskEnqueueOptions, + TaskRetryEnqueueError, } from "./tasks/mod.ts"; import { hasMalformedKnownTemporalLiteral } from "./temporal.ts"; import { handleWebFinger } from "./webfinger.ts"; -type QueueTaskDispatchResult = - | { readonly outcome: "completed" | "aborted" } - | { - readonly outcome: "failed"; - readonly failureReason: QueueTaskFailureReason; - }; - const circuitBreakerCasWarningKvStores = new WeakSet(); let nextQueueDepthGaugeSourceId = 0; const retryAfterHttpDate = new RegExp( @@ -1262,7 +1257,9 @@ export class FederationImpl ); } catch (e) { if (isAbortError(e)) recordOutcome("aborted", undefined); - else recordOutcome("failed", "handler", e); + else if (e instanceof TaskRetryEnqueueError) { + recordOutcome("failed", "retry_enqueue", e.cause ?? e); + } else recordOutcome("failed", "handler", e); throw e; } finally { this.metrics.decrementQueueTaskInFlight(common); @@ -2247,10 +2244,23 @@ export class FederationImpl ...message, attempt: message.attempt + 1, } satisfies TaskMessage; - await queue.enqueue(retryMessage, { - delay: clampNegativeDelay(delay), - orderingKey: message.orderingKey, - }); + try { + await queue.enqueue(retryMessage, { + delay: clampNegativeDelay(delay), + orderingKey: message.orderingKey, + }); + } catch (enqueueError) { + logger.error( + "Custom task {taskName} could not be re-enqueued for a retry " + + "(attempt #{attempt}):\n{error}", + { + taskName: message.taskName, + attempt: retryMessage.attempt, + error: enqueueError, + }, + ); + throw new TaskRetryEnqueueError(error); + } this.metrics.recordQueueTaskEnqueued( { role: "task", queue, taskName: message.taskName }, retryMessage.attempt, diff --git a/packages/fedify/src/federation/tasks/error.ts b/packages/fedify/src/federation/tasks/error.ts new file mode 100644 index 000000000..da7a185e9 --- /dev/null +++ b/packages/fedify/src/federation/tasks/error.ts @@ -0,0 +1,6 @@ +export class TaskRetryEnqueueError extends Error { + constructor(cause: unknown) { + super("Failed to re-enqueue a custom task for a retry.", { cause }); + this.name = "TaskRetryEnqueueError"; + } +} diff --git a/packages/fedify/src/federation/tasks/mod.ts b/packages/fedify/src/federation/tasks/mod.ts index 62072ac45..e16acc687 100644 --- a/packages/fedify/src/federation/tasks/mod.ts +++ b/packages/fedify/src/federation/tasks/mod.ts @@ -8,6 +8,7 @@ */ export { default as TaskCodec } from "./codec.ts"; export { default as enqueueTasks } from "./enqueue.ts"; +export { TaskRetryEnqueueError } from "./error.ts"; export type { TaskDefinition, TaskDefinitionInternal, @@ -16,3 +17,4 @@ export type { TaskHandler, TaskRegistry, } from "./task.ts"; +export type { QueueTaskDispatchResult } from "./types.ts"; diff --git a/packages/fedify/src/federation/tasks/tasks.test.ts b/packages/fedify/src/federation/tasks/tasks.test.ts index 28fce13d1..e783c9113 100644 --- a/packages/fedify/src/federation/tasks/tasks.test.ts +++ b/packages/fedify/src/federation/tasks/tasks.test.ts @@ -1564,6 +1564,62 @@ test("task observability", async (t) => { }, ); + await t.step( + "attributes retry_enqueue when the retry re-enqueue itself fails", + async () => { + // A queue whose enqueue always rejects: the only enqueue here is the + // retry re-enqueue, so it drives the re-enqueue-failure path. + class RejectingQueue implements MessageQueue { + enqueue(): Promise { + return Promise.reject(new Error("backend unavailable")); + } + listen(): Promise { + return Promise.resolve(); + } + } + const queue = new RejectingQueue(); + const { federation, recorder, exporter } = instrument({ + ...baseOptions, + queue: { task: queue }, + }); + const handlerError = new Error("boom"); + federation.defineTask("retry-enqueue-fails", { + schema: stringSchema, + handler: () => { + throw handlerError; + }, + retryPolicy: () => Temporal.Duration.from({ milliseconds: 1 }), + }); + const message = await makeTaskMessage("retry-enqueue-fails", "payload"); + // The message is nacked (re-thrown), and the thrown marker preserves the + // original handler error as its cause. + await rejects( + () => federation.processQueuedTask(undefined, message), + (e: unknown) => e instanceof Error && e.cause === handlerError, + ); + + const failed = recorder.getMeasurements("fedify.queue.task.failed"); + strictEqual(failed.length, 1); + strictEqual( + failed[0].attributes["fedify.task.failure_reason"], + "retry_enqueue", + ); + // No successful retry re-enqueue was recorded. + strictEqual( + recorder.getMeasurements("fedify.queue.task.enqueued").length, + 0, + ); + const span = exporter.getSpans("fedify.task")[0]; + strictEqual( + span.attributes["fedify.task.failure_reason"], + "retry_enqueue", + ); + strictEqual(span.status.code, SpanStatusCode.ERROR); + // The span surfaces the original handler error, not the enqueue error. + ok(span.status.message?.includes("boom")); + }, + ); + await t.step( "records an abort as aborted, without a failure reason or error status", async () => { diff --git a/packages/fedify/src/federation/tasks/types.ts b/packages/fedify/src/federation/tasks/types.ts new file mode 100644 index 000000000..2ae2d3869 --- /dev/null +++ b/packages/fedify/src/federation/tasks/types.ts @@ -0,0 +1,8 @@ +import type { QueueTaskFailureReason } from "../metrics.ts"; + +export type QueueTaskDispatchResult = + | { readonly outcome: "completed" | "aborted" } + | { + readonly outcome: "failed"; + readonly failureReason: QueueTaskFailureReason; + }; From 0052bb932d0002e48204fb37942d981b8d5c9253 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Mon, 6 Jul 2026 20:07:49 +0000 Subject: [PATCH 52/57] Deduplicate queue worker-boundary instrumentation The fanout, outbox, inbox, and task branches of FederationImpl.processQueuedTask() each repeated the same consumer-span and metric boilerplate: open a CONSUMER span, scope the extracted trace context, record the started/outcome/duration metrics, pair the in-flight increment with its decrement, set the span status, and end the span. The task branch had also drifted from the other three, recording its outcome outside the finally block that guarantees it. Collapse the four copies onto two helpers. #runWorkerSpan() opens the span and scopes the trace context; #instrumentWorkerBody() wraps a worker body with the shared boundary telemetry. Each branch now supplies only its span name, span attributes, common metric attributes, body, and error classifier. The outcome record lives in one finally block again, so a "failed" outcome that arrives through the body's return value (a dropped or given-up task) is instrumented the same way as one that arrives through a thrown error. Unify the former WorkerSpanOutcome type into QueueTaskDispatchResult (failureReason and error are now optional on the failed variant), so the task dispatch result and the worker-boundary outcome share one type, and move the classifyAbortableError and classifyTaskError classifiers into tasks/error.ts. https://github.com/fedify-dev/fedify/pull/812 Assisted-by: Claude Code:claude-opus-4-8 --- packages/fedify/src/federation/middleware.ts | 320 +++++++----------- packages/fedify/src/federation/tasks/error.ts | 21 ++ packages/fedify/src/federation/tasks/mod.ts | 6 +- packages/fedify/src/federation/tasks/types.ts | 3 +- 4 files changed, 149 insertions(+), 201 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 5bf42114d..412c5e2d3 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -32,6 +32,7 @@ import type { import { lookupWebFinger } from "@fedify/webfinger"; import { getLogger, withContext } from "@logtape/logtape"; import { + type Attributes, context, type MeterProvider, metrics, @@ -141,8 +142,6 @@ import { isAbortError, type QueueDepthGaugeEntry, type QueueTaskCommonAttributes, - type QueueTaskFailureReason, - type QueueTaskResult, recordCircuitBreakerStateChange, recordCollectionRequest, recordFanoutRecipients, @@ -169,6 +168,8 @@ import { type SenderKeyPair, } from "./send.ts"; import { + classifyAbortableError, + classifyTaskError, enqueueTasks, type QueueTaskDispatchResult, TaskCodec, @@ -1025,7 +1026,6 @@ export class FederationImpl contextData: TContextData, message: Message, ): Promise { - const tracer = this._getTracer(); const extractedContext = propagation.extract( context.active(), message.traceContext, @@ -1037,54 +1037,17 @@ export class FederationImpl queue: this.fanoutQueue, activityType: message.activityType, }; - await tracer.startActiveSpan( + await this.#runWorkerSpan( "activitypub.fanout", - { - kind: SpanKind.CONSUMER, - attributes: { - "activitypub.activity.type": message.activityType, - }, - }, + { "activitypub.activity.type": message.activityType }, extractedContext, - async (span) => { - const spanCtx = span.spanContext(); - return await withContext( - { traceId: spanCtx.traceId, spanId: spanCtx.spanId }, - async () => { - if (message.activityId != null) { - span.setAttribute( - "activitypub.activity.id", - message.activityId, - ); - } - this.metrics.recordQueueTaskStarted(common); - this.metrics.incrementQueueTaskInFlight(common); - const startedAt = performance.now(); - let outcome: QueueTaskResult = "completed"; - try { - await this.#listenFanoutMessage(contextData, message); - } catch (e) { - const aborted = isAbortError(e); - outcome = aborted ? "aborted" : "failed"; - if (!aborted) { - span.setStatus({ - code: SpanStatusCode.ERROR, - message: String(e), - }); - } - throw e; - } finally { - this.metrics.recordQueueTaskOutcome( - common, - outcome, - getDurationMs(startedAt), - ); - this.metrics.decrementQueueTaskInFlight(common); - span.end(); - } - }, - ); - }, + this.#instrumentWorkerBody(common, async (span) => { + if (message.activityId != null) { + span.setAttribute("activitypub.activity.id", message.activityId); + } + await this.#listenFanoutMessage(contextData, message); + return { outcome: "completed" }; + }, classifyAbortableError), ); } else if (message.type === "outbox") { const common: QueueTaskCommonAttributes = { @@ -1092,110 +1055,41 @@ export class FederationImpl queue: this.outboxQueue, activityType: message.activityType, }; - await tracer.startActiveSpan( + await this.#runWorkerSpan( "activitypub.outbox", { - kind: SpanKind.CONSUMER, - attributes: { - "activitypub.activity.type": message.activityType, - "activitypub.activity.retries": message.attempt, - }, + "activitypub.activity.type": message.activityType, + "activitypub.activity.retries": message.attempt, }, extractedContext, - async (span) => { - const spanCtx = span.spanContext(); - return await withContext( - { traceId: spanCtx.traceId, spanId: spanCtx.spanId }, - async () => { - if (message.activityId != null) { - span.setAttribute( - "activitypub.activity.id", - message.activityId, - ); - } - this.metrics.recordQueueTaskStarted(common); - this.metrics.incrementQueueTaskInFlight(common); - const startedAt = performance.now(); - let outcome: QueueTaskResult = "completed"; - try { - await this.#listenOutboxMessage(contextData, message, span); - } catch (e) { - const aborted = isAbortError(e); - outcome = aborted ? "aborted" : "failed"; - if (!aborted) { - span.setStatus({ - code: SpanStatusCode.ERROR, - message: String(e), - }); - } - throw e; - } finally { - this.metrics.recordQueueTaskOutcome( - common, - outcome, - getDurationMs(startedAt), - ); - this.metrics.decrementQueueTaskInFlight(common); - span.end(); - } - }, - ); - }, + this.#instrumentWorkerBody(common, async (span) => { + if (message.activityId != null) { + span.setAttribute("activitypub.activity.id", message.activityId); + } + await this.#listenOutboxMessage(contextData, message, span); + return { outcome: "completed" }; + }, classifyAbortableError), ); } else if (message.type === "inbox") { const common: QueueTaskCommonAttributes = { role: "inbox", queue: this.inboxQueue, }; - await tracer.startActiveSpan( + await this.#runWorkerSpan( "activitypub.inbox", - { - kind: SpanKind.CONSUMER, - attributes: { - "activitypub.shared_inbox": message.identifier == null, - }, - }, + { "activitypub.shared_inbox": message.identifier == null }, extractedContext, - async (span) => { - const spanCtx = span.spanContext(); - return await withContext( - { traceId: spanCtx.traceId, spanId: spanCtx.spanId }, - async () => { - this.metrics.recordQueueTaskStarted(common); - this.metrics.incrementQueueTaskInFlight(common); - const startedAt = performance.now(); - let outcome: QueueTaskResult = "completed"; - try { - await this.#listenInboxMessage( - contextData, - message, - span, - (activityType) => { - common.activityType = activityType; - }, - ); - } catch (e) { - const aborted = isAbortError(e); - outcome = aborted ? "aborted" : "failed"; - if (!aborted) { - span.setStatus({ - code: SpanStatusCode.ERROR, - message: String(e), - }); - } - throw e; - } finally { - this.metrics.recordQueueTaskOutcome( - common, - outcome, - getDurationMs(startedAt), - ); - this.metrics.decrementQueueTaskInFlight(common); - span.end(); - } + this.#instrumentWorkerBody(common, async (span) => { + await this.#listenInboxMessage( + contextData, + message, + span, + (activityType) => { + common.activityType = activityType; }, ); - }, + return { outcome: "completed" }; + }, classifyAbortableError), ); } else if (message.type === "task") { const registered = this.taskDefinitions.get(message.taskName) != null; @@ -1204,75 +1098,103 @@ export class FederationImpl queue: this.resolveTaskQueue(message.taskName), taskName: registered ? message.taskName : undefined, }; - await tracer.startActiveSpan( + await this.#runWorkerSpan( "fedify.task", { - kind: SpanKind.CONSUMER, - attributes: { - "fedify.task.name": message.taskName, - "fedify.task.attempt": message.attempt, - }, + "fedify.task.name": message.taskName, + "fedify.task.attempt": message.attempt, }, extractedContext, - async (span) => { - const spanCtx = span.spanContext(); - return await withContext( - { traceId: spanCtx.traceId, spanId: spanCtx.spanId }, - async () => { - this.metrics.recordQueueTaskStarted(common); - this.metrics.incrementQueueTaskInFlight(common); - const startedAt = performance.now(); - const recordOutcome = ( - outcome: QueueTaskResult, - failureReason: QueueTaskFailureReason | undefined, - error?: unknown, - ): void => { - if (failureReason != null) { - span.setAttribute( - "fedify.task.failure_reason", - failureReason, - ); - span.setStatus({ - code: SpanStatusCode.ERROR, - ...(error == null ? {} : { message: String(error) }), - }); - } - this.metrics.recordQueueTaskOutcome( - common, - outcome, - getDurationMs(startedAt), - outcome === "failed" ? failureReason : undefined, - ); - }; - try { - const result = await this.#listenTaskMessage( - contextData, - message, - ); - recordOutcome( - result.outcome, - result.outcome === "failed" - ? result.failureReason - : undefined, - ); - } catch (e) { - if (isAbortError(e)) recordOutcome("aborted", undefined); - else if (e instanceof TaskRetryEnqueueError) { - recordOutcome("failed", "retry_enqueue", e.cause ?? e); - } else recordOutcome("failed", "handler", e); - throw e; - } finally { - this.metrics.decrementQueueTaskInFlight(common); - span.end(); - } - }, - ); - }, + this.#instrumentWorkerBody( + common, + () => this.#listenTaskMessage(contextData, message), + classifyTaskError, + ), ); } }); } + /** + * Opens a CONSUMER span for a queue worker and scopes the extracted trace + * context around its body. The body—typically built by + * {@link FederationImpl.instrumentWorkerBody}—receives the span and returns + * the context-scoped worker callback. + * @param spanName The consumer span name (for example, `"fedify.task"`). + * @param spanAttributes The initial span attributes. + * @param extractedContext The trace context extracted from the message. + * @param instrumentedBody Builds the context-scoped worker callback for the + * opened span. + */ + async #runWorkerSpan( + spanName: string, + spanAttributes: Attributes, + extractedContext: ReturnType, + instrumentedBody: (span: Span) => () => Promise, + ): Promise { + await this._getTracer().startActiveSpan( + spanName, + { kind: SpanKind.CONSUMER, attributes: spanAttributes }, + extractedContext, + async (span: Span) => { + const spanCtx = span.spanContext(); + return await withContext( + { traceId: spanCtx.traceId, spanId: spanCtx.spanId }, + instrumentedBody(span), + ); + }, + ); + } + + /** + * Wraps a worker body with the shared `fedify.queue.task.*` boundary + * telemetry, returning the instrumented body keyed by its span. The + * instrumentation records the started, outcome, and duration metrics, pairs + * the in-flight increment with its decrement, sets the span's error status + * from the terminal outcome, and ends the span. The fanout, outbox, inbox, + * and task workers differ only in `common`, the body, and how a thrown error + * maps to an outcome. + * @param common The shared queue-task metric attributes for this worker. + * @param run Runs the worker body and resolves to its terminal outcome. + * @param classifyError Maps a thrown error to a terminal outcome. + */ + #instrumentWorkerBody = ( + common: QueueTaskCommonAttributes, + run: (span: Span) => Promise, + classifyError: (error: unknown) => QueueTaskDispatchResult, + ) => + (span: Span) => + async () => { + this.metrics.recordQueueTaskStarted(common); + this.metrics.incrementQueueTaskInFlight(common); + const startedAt = performance.now(); + let result: QueueTaskDispatchResult = { outcome: "completed" }; + try { + result = await run(span); + } catch (e) { + result = classifyError(e); + throw e; + } finally { + if (result.outcome === "failed") { + if (result.failureReason != null) { + span.setAttribute("fedify.task.failure_reason", result.failureReason); + } + span.setStatus({ + code: SpanStatusCode.ERROR, + ...(result.error == null ? {} : { message: String(result.error) }), + }); + } + this.metrics.recordQueueTaskOutcome( + common, + result.outcome, + getDurationMs(startedAt), + result.outcome === "failed" ? result.failureReason : undefined, + ); + this.metrics.decrementQueueTaskInFlight(common); + span.end(); + } + }; + async #listenFanoutMessage( data: TContextData, message: FanoutMessage, diff --git a/packages/fedify/src/federation/tasks/error.ts b/packages/fedify/src/federation/tasks/error.ts index da7a185e9..fc82fd15d 100644 --- a/packages/fedify/src/federation/tasks/error.ts +++ b/packages/fedify/src/federation/tasks/error.ts @@ -1,6 +1,27 @@ +import { isAbortError } from "../metrics.ts"; +import type { QueueTaskDispatchResult } from "./types.ts"; + export class TaskRetryEnqueueError extends Error { constructor(cause: unknown) { super("Failed to re-enqueue a custom task for a retry.", { cause }); this.name = "TaskRetryEnqueueError"; } } + +export const classifyAbortableError = ( + error: unknown, +): QueueTaskDispatchResult => + isAbortError(error) ? { outcome: "aborted" } : { outcome: "failed", error }; + +export const classifyTaskError = ( + error: unknown, +): QueueTaskDispatchResult => + isAbortError(error) + ? { outcome: "aborted" } + : error instanceof TaskRetryEnqueueError + ? { + outcome: "failed", + failureReason: "retry_enqueue", + error: error.cause ?? error, + } + : { outcome: "failed", failureReason: "handler", error }; diff --git a/packages/fedify/src/federation/tasks/mod.ts b/packages/fedify/src/federation/tasks/mod.ts index e16acc687..a0b04d5bb 100644 --- a/packages/fedify/src/federation/tasks/mod.ts +++ b/packages/fedify/src/federation/tasks/mod.ts @@ -8,7 +8,11 @@ */ export { default as TaskCodec } from "./codec.ts"; export { default as enqueueTasks } from "./enqueue.ts"; -export { TaskRetryEnqueueError } from "./error.ts"; +export { + classifyAbortableError, + classifyTaskError, + TaskRetryEnqueueError, +} from "./error.ts"; export type { TaskDefinition, TaskDefinitionInternal, diff --git a/packages/fedify/src/federation/tasks/types.ts b/packages/fedify/src/federation/tasks/types.ts index 2ae2d3869..e08eb940b 100644 --- a/packages/fedify/src/federation/tasks/types.ts +++ b/packages/fedify/src/federation/tasks/types.ts @@ -4,5 +4,6 @@ export type QueueTaskDispatchResult = | { readonly outcome: "completed" | "aborted" } | { readonly outcome: "failed"; - readonly failureReason: QueueTaskFailureReason; + readonly failureReason?: QueueTaskFailureReason; + readonly error?: unknown; }; From b37fdfda633554e03b17617f10104145998956d3 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Mon, 6 Jul 2026 20:30:21 +0000 Subject: [PATCH 53/57] Add `wait_for = ["build"]` in `tasks."test:deno"` Running only `mise run test:deno` by itself is not a problem. However, when running `mise run test`, `test:deno` and `build` would run simultaneously, often causing errors. To prevent this, the `wait_for` attributes are added in the `test:deno` task with `build` item. --- mise.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/mise.toml b/mise.toml index 1fbf82a53..703bb6940 100644 --- a/mise.toml +++ b/mise.toml @@ -248,6 +248,7 @@ for pkg in ($env.usage_packages | split row " " | where ($it | is-not-empty)) { # Testing [tasks."test:deno"] +wait_for = ["build"] description = "Run the test suite using Deno" run = "deno test --check --doc --allow-all --unstable-kv --trace-leaks --parallel" From 6eb234c2c1b4c36c8439bcfcb091f6ca55026fed Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Mon, 6 Jul 2026 21:14:51 +0000 Subject: [PATCH 54/57] Deflake the stale-rollback dedup marker test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test drove the first marker's expiry through a real 1 ms taskDeduplicationTtl, but that TTL equally applied to the marker the second enqueue re-claimed. Every assertion after the re-claim therefore had to run within 1 ms of wall clock before MemoryKvStore's lazy expiry check judged the marker expired. Local runs finish those microtask hops in time, but under CI load—especially `mise run test:bun`, which runs every package's tests in parallel on a Temporal polyfill—the window was routinely blown, failing the test intermittently. Simulate the first marker's expiry deterministically by deleting the key instead, and stretch the TTL to one minute so the second marker comfortably outlives the assertions. This removes the wall-clock dependence entirely and also drops the 20 ms delay. Assisted-by: Claude Code:claude-fable-5 --- packages/fedify/src/federation/tasks/enqueue.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/fedify/src/federation/tasks/enqueue.test.ts b/packages/fedify/src/federation/tasks/enqueue.test.ts index 4ec9112da..e461ce4ef 100644 --- a/packages/fedify/src/federation/tasks/enqueue.test.ts +++ b/packages/fedify/src/federation/tasks/enqueue.test.ts @@ -1030,7 +1030,7 @@ test( ...baseOptions, kv, queue: { task: queue }, - taskDeduplicationTtl: { milliseconds: 1 }, + taskDeduplicationTtl: { minutes: 1 }, }); const task = federation.defineTask("stale-rollback", { schema: stringSchema, @@ -1043,7 +1043,7 @@ test( const first = ctx.enqueueTask(task, "first", { deduplicationKey: "k" }); await firstEntered; - await delay(20); + await kv.delete(markerKey); await ctx.enqueueTask(task, "second", { deduplicationKey: "k" }); const secondToken = await kv.get(markerKey); ok(secondToken != null); From eae895c9859fc8b990e60881e2884dae2f5f5ca1 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Tue, 7 Jul 2026 05:59:32 +0000 Subject: [PATCH 55/57] Combine duplicate `@fedify/fedify` sections in `Version 2.4.0` and add the final PR number in changelog --- CHANGES.md | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ec5fcb113..0f46c75a2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,13 @@ Version 2.4.0 ### @fedify/fedify + - Added local `did:key` verification method resolution for + [FEP-8b32] Object Integrity Proofs. `verifyProof()` can now verify + Ed25519 `eddsa-jcs-2022` proofs whose `verificationMethod` is a + `did:key:z...#z...` DID URL without fetching the verification method + as a remote JSON-LD document, which is required for [FEP-ef61] + portable objects. [[#827], [#915]] + - Added a custom background task API that generalizes Fedify's enqueue-and-process-later pattern to arbitrary application-defined jobs: @@ -45,8 +52,11 @@ Version 2.4.0 `FederationOptions.taskDeduplicationTtl` and `FederationOptions.taskDeduplicationFallback` options. - [[#206], [#797], [#798], [#799], [#803], [#806], [#812] by ChanHaeng Lee] + [[#206], [#797], [#798], [#799], [#803], [#806], [#812], [#923] by + ChanHaeng Lee] +[FEP-8b32]: https://w3id.org/fep/8b32 +[FEP-ef61]: https://w3id.org/fep/ef61 [Standard Schema]: https://standardschema.dev/ [#206]: https://github.com/fedify-dev/fedify/issues/206 [#797]: https://github.com/fedify-dev/fedify/issues/797 @@ -55,20 +65,9 @@ Version 2.4.0 [#803]: https://github.com/fedify-dev/fedify/pull/803 [#806]: https://github.com/fedify-dev/fedify/pull/806 [#812]: https://github.com/fedify-dev/fedify/pull/812 - -### @fedify/fedify - - - Added local `did:key` verification method resolution for - [FEP-8b32] Object Integrity Proofs. `verifyProof()` can now verify - Ed25519 `eddsa-jcs-2022` proofs whose `verificationMethod` is a - `did:key:z...#z...` DID URL without fetching the verification method - as a remote JSON-LD document, which is required for [FEP-ef61] - portable objects. [[#827], [#915]] - -[FEP-8b32]: https://w3id.org/fep/8b32 -[FEP-ef61]: https://w3id.org/fep/ef61 [#827]: https://github.com/fedify-dev/fedify/issues/827 [#915]: https://github.com/fedify-dev/fedify/pull/915 +[#923]: https://github.com/fedify-dev/fedify/pull/923 ### @fedify/vocab From 3b0a4534201eda3e6ff91684b012fe343a856413 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Tue, 7 Jul 2026 06:13:09 +0000 Subject: [PATCH 56/57] Revive `To be released.` --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 0f46c75a2..1236c5b4d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,8 @@ Fedify changelog Version 2.4.0 ------------- +To be released. + ### @fedify/fedify - Added local `did:key` verification method resolution for From 2c99806d36293bb6806e20b5e323b31f41473062 Mon Sep 17 00:00:00 2001 From: ChanHaeng Lee <2chanhaeng@gmail.com> Date: Tue, 7 Jul 2026 06:57:02 +0000 Subject: [PATCH 57/57] Update `markdown-it-jsr-ref` version to 0.5.0 By [JSR #1473](https://github.com/jsr-io/jsr/pull/1473), the docs building task broken. For that, @dahlia updates [`markdown-it-jsr-ref`](https://github.com/dahlia/markdown-it-jsr-ref/issues/1). Assisted-by: Claude Code:claude-opus-4-8 --- docs/package.json | 2 +- pnpm-lock.yaml | 21 +++++++-------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/docs/package.json b/docs/package.json index 6af010382..9fdd43087 100644 --- a/docs/package.json +++ b/docs/package.json @@ -62,7 +62,7 @@ "markdown-it-abbr": "^2.0.0", "markdown-it-deflist": "^3.0.0", "markdown-it-footnote": "^4.0.0", - "markdown-it-jsr-ref": "0.4.4", + "markdown-it-jsr-ref": "0.5.0", "mermaid": "^11.4.1", "postgres": "catalog:", "srvx": "^0.11.15", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3bf0cf09c..fe81f5cef 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -377,8 +377,8 @@ importers: specifier: ^4.0.0 version: 4.0.0 markdown-it-jsr-ref: - specifier: 0.4.4 - version: 0.4.4 + specifier: 0.5.0 + version: 0.5.0 mermaid: specifier: ^11.4.1 version: 11.7.0 @@ -9648,9 +9648,6 @@ packages: lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} - linkify-it@5.0.0: - resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==} - linkify-it@5.0.1: resolution: {integrity: sha512-wVoTjP4Q6R0NW5hiZkVJaFZPWgtXfoGF+6LucL3/FtiNjmcHhYjEr5f1Kqjirc1nBW07J/ZuRFumqr2oqccEWg==} @@ -9767,8 +9764,8 @@ packages: markdown-it-footnote@4.0.0: resolution: {integrity: sha512-WYJ7urf+khJYl3DqofQpYfEYkZKbmXmwxQV8c8mO/hGIhgZ1wOe7R4HLFNwqx7TjILbnC98fuyeSsin19JdFcQ==} - markdown-it-jsr-ref@0.4.4: - resolution: {integrity: sha512-z6abJJXW0vNHaLC9fxVvT9iaHw2pGjiQDn+rFlQKV3+JgfmNH6bwF8uzWGh1d5Wo7UUAx1w7drmyQFG0OWwh8g==} + markdown-it-jsr-ref@0.5.0: + resolution: {integrity: sha512-lPo1of7KhZ3wrSenZG7W1Ddoal2sfpVPNuPLlF24BxO8xwwfOA/CxdnZ8HwJ+6YZFkUR7y1uCJKRqF1NUym3LQ==} markdown-it@14.1.0: resolution: {integrity: sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==} @@ -21671,10 +21668,6 @@ snapshots: lines-and-columns@1.2.4: {} - linkify-it@5.0.0: - dependencies: - uc.micro: 2.1.0 - linkify-it@5.0.1: dependencies: uc.micro: 2.1.0 @@ -21803,16 +21796,16 @@ snapshots: markdown-it-footnote@4.0.0: {} - markdown-it-jsr-ref@0.4.4: + markdown-it-jsr-ref@0.5.0: dependencies: '@deno/shim-deno': 0.18.2 - markdown-it: 14.1.0 + markdown-it: 14.2.0 markdown-it@14.1.0: dependencies: argparse: 2.0.1 entities: 4.5.0 - linkify-it: 5.0.0 + linkify-it: 5.0.1 mdurl: 2.0.0 punycode.js: 2.3.1 uc.micro: 2.1.0