From ba202058feae14bf9a70ba6f4e5adf94889cb3bf Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Mon, 6 Jul 2026 22:31:38 +0530 Subject: [PATCH 01/16] docs(java): workflow guides --- .../docs/java/guides/workflows/analysis.mdx | 68 +++++++ .../docs/java/guides/workflows/caching.mdx | 58 ++++++ .../docs/java/guides/workflows/canvas.mdx | 69 +++++++ .../docs/java/guides/workflows/conditions.mdx | 124 +++++++++++++ .../docs/java/guides/workflows/fan-out.mdx | 168 ++++++++++++++++++ .../docs/java/guides/workflows/gates.mdx | 130 ++++++++++++++ .../docs/java/guides/workflows/index.mdx | 78 ++++++++ .../docs/java/guides/workflows/meta.json | 4 + .../docs/java/guides/workflows/saga.mdx | 145 +++++++++++++++ .../java/guides/workflows/sub-workflows.mdx | 123 +++++++++++++ 10 files changed, 967 insertions(+) create mode 100644 docs/content/docs/java/guides/workflows/analysis.mdx create mode 100644 docs/content/docs/java/guides/workflows/caching.mdx create mode 100644 docs/content/docs/java/guides/workflows/canvas.mdx create mode 100644 docs/content/docs/java/guides/workflows/conditions.mdx create mode 100644 docs/content/docs/java/guides/workflows/fan-out.mdx create mode 100644 docs/content/docs/java/guides/workflows/gates.mdx create mode 100644 docs/content/docs/java/guides/workflows/index.mdx create mode 100644 docs/content/docs/java/guides/workflows/meta.json create mode 100644 docs/content/docs/java/guides/workflows/saga.mdx create mode 100644 docs/content/docs/java/guides/workflows/sub-workflows.mdx diff --git a/docs/content/docs/java/guides/workflows/analysis.mdx b/docs/content/docs/java/guides/workflows/analysis.mdx new file mode 100644 index 00000000..a777ee41 --- /dev/null +++ b/docs/content/docs/java/guides/workflows/analysis.mdx @@ -0,0 +1,68 @@ +--- +title: Analysis & Visualization +description: "Introspect a workflow definition's DAG — order, levels, ancestors, and diagrams." +--- + +`WorkflowAnalysis` answers pure-graph questions about a `Workflow`'s DAG (its +steps and their `after` edges) before a run starts — useful for validation and +tooling. All methods are static and read-only. + +```java +// a → {b, c} → d (a diamond) +Workflow wf = Workflow.named("graph") + .step("a", task, 1) + .step("b", task, 1, "a") + .step("c", task, 1, "a") + .step("d", task, 1, "b", "c"); +``` + +## Structure + +```java +WorkflowAnalysis.roots(wf); // ["a"] — steps with no dependencies +WorkflowAnalysis.leaves(wf); // ["d"] — steps nothing depends on +WorkflowAnalysis.ancestors(wf, "d"); // {"a", "b", "c"} — transitive predecessors +WorkflowAnalysis.descendants(wf, "a"); // {"b", "c", "d"} — transitive successors +WorkflowAnalysis.topologicalOrder(wf); // a valid execution order +``` + +## Levels + +`levels(wf)` groups steps by dependency depth — everything in one level can +run in parallel: + +```java +WorkflowAnalysis.levels(wf); // [["a"], ["b", "c"], ["d"]] +``` + +## Validation + +Every query first runs `WorkflowAnalysis.validate(wf)`, which throws a +`WorkflowException` if any `after` edge names an undeclared step. +`topologicalOrder` additionally throws when the DAG contains a cycle — call it +in a test to catch a bad graph before submitting: + +```java +Workflow cyclic = Workflow.named("cyc") + .step("x", task, 1, "y") + .step("y", task, 1, "x"); +WorkflowAnalysis.topologicalOrder(cyclic); // throws WorkflowException: has a cycle +``` + +## Diagrams + +`WorkflowVisualization` renders the same DAG as text for docs or debugging — +Mermaid (`graph TD`) or Graphviz DOT. Node labels note the step kind (gate, +fan-out, fan-in, sub-workflow, conditional), and gates render as Mermaid +decision diamonds: + +```java +WorkflowVisualization.mermaid(wf); // "graph TD\n n0[\"a\"]\n ..." +WorkflowVisualization.dot(wf); // "digraph \"graph\" {\n \"a\" [label=\"a\"];\n ..." +``` + + + Analysis and visualization operate on the workflow **definition**, not on a + submitted run — they need no queue or storage. For a live run's per-node + statuses, use `run.status()` and walk `WorkflowStatus.nodes`. + diff --git a/docs/content/docs/java/guides/workflows/caching.mdx b/docs/content/docs/java/guides/workflows/caching.mdx new file mode 100644 index 00000000..c7dfcf7b --- /dev/null +++ b/docs/content/docs/java/guides/workflows/caching.mdx @@ -0,0 +1,58 @@ +--- +title: Incremental Runs +description: "Reuse a workflow step's result across runs — cache hits skip re-execution within a TTL." +--- + +Mark a step cacheable with `.cache(Duration)` and re-running the same workflow +within the TTL skips the expensive work: the step becomes a **cache hit** and +its successors advance as if it had completed. + +```java +Workflow report = Workflow.named("report") + .step("seed", seedTask, 0) + .step(Step.of("crunch", crunchTask, 7) + .cache(Duration.ofMinutes(5)) + .after("seed") + .build()) + .step("render", renderTask, 1, "crunch"); +``` + +On the first run `crunch` executes normally; on a second run within five +minutes its node status is `CACHE_HIT` and the task is not re-executed — +`render` still runs. + +```java +WorkflowStatus second = queue.submitWorkflow(report).await(Duration.ofSeconds(30)); +second.node("crunch").orElseThrow().status; // CACHE_HIT +second.node("render").orElseThrow().status; // COMPLETED +``` + +## The cache key + +A cached step is keyed by its **workflow name**, its **step name**, and a +SHA-256 hash of its **payload**. Changing any of them (or letting the TTL +lapse) misses the cache and re-runs the step. + +The cache lives in the tracking worker's process memory — it survives across +runs handled by the same worker, but not across worker restarts, and it is not +shared between workers. + +## TTL + +The TTL is required and must be positive; `.cache(...)` rejects zero or +negative durations. Expired entries are swept when new results are cached and +dropped lazily on lookup. + +## Constraints + +- **A cacheable step must have a predecessor.** A cacheable step is a deferred + node, and a deferred root is never promoted — `Step.build()` rejects a + cacheable step with an empty `after`, since it would wedge the run in + `PENDING`. +- **A cached step cannot feed a fan-out.** A cache hit skips the task, so it + produces no fresh result list to expand — a fan-out whose producer is + cacheable is rejected at submit. +- **A cache hit has no forward result downstream.** A callable + `condition` on a successor won't see the cached step's result, and a cache + hit is never compensated by a [saga](/java/guides/workflows/saga) (its side + effects were not performed in this run). diff --git a/docs/content/docs/java/guides/workflows/canvas.mdx b/docs/content/docs/java/guides/workflows/canvas.mdx new file mode 100644 index 00000000..e27348f8 --- /dev/null +++ b/docs/content/docs/java/guides/workflows/canvas.mdx @@ -0,0 +1,69 @@ +--- +title: Canvas Primitives +description: "Chain, group, and chord shorthands for common workflow shapes." +--- + +The `Canvas` helpers express common DAG shapes without hand-wiring `after` +edges. Each takes `Canvas.link(name, task, payload)` building blocks and +returns a `Workflow`, so the result submits like any other workflow — and you +can keep adding steps to it with the regular builder methods. + +## chain + +Run steps in sequence — each after the previous one: + +```java +Workflow etl = Canvas.chain("etl", + Canvas.link("extract", extractTask, 1), + Canvas.link("transform", transformTask, 2), + Canvas.link("load", loadTask, 3)); + +queue.submitWorkflow(etl); +// extract → transform → load +``` + +## group + +Run steps in parallel (no dependencies between them): + +```java +Workflow notify = Canvas.group("notify", + Canvas.link("email", sendEmail, message), + Canvas.link("sms", sendSms, message), + Canvas.link("push", sendPush, message)); + +queue.submitWorkflow(notify); +// email | sms | push +``` + +To hang a parallel group off an existing step, add the steps with the builder +instead — `Canvas` shortcuts always start from roots: + +```java +Workflow wf = Workflow.named("notify") + .step("prepare", prepareTask, 1) + .step("email", sendEmail, message, "prepare") + .step("sms", sendSms, message, "prepare"); +// prepare → (email | sms) +``` + +## chord + +A parallel group joined by a callback that runs once every member completes: + +```java +Workflow report = Canvas.chord("report", + Canvas.link("merge", mergeTask, 0), // callback + Canvas.link("q1", queryRegion, "east"), // group... + Canvas.link("q2", queryRegion, "west")); + +queue.submitWorkflow(report); +// (q1 | q2) → merge +``` + + + A `chord`'s callback runs *after* the group but receives its own payload — + the members' results are not auto-passed. To aggregate results into one + step, use [fan-out / fan-in](/java/guides/workflows/fan-out), which collects + child results into the combiner's payload. + diff --git a/docs/content/docs/java/guides/workflows/conditions.mdx b/docs/content/docs/java/guides/workflows/conditions.mdx new file mode 100644 index 00000000..f4bc2d7a --- /dev/null +++ b/docs/content/docs/java/guides/workflows/conditions.mdx @@ -0,0 +1,124 @@ +--- +title: Conditions & Error Handling +description: "Run a step only when its predecessors succeeded, failed, or either — or per a predicate." +--- + +Every step runs by default only when all of its predecessors completed +successfully. Set a condition on the `Step` builder to change that: + +```java +Workflow pipeline = Workflow.named("resilient-pipeline") + .step(Step.of("risky", riskyTask, 1).maxRetries(0).build()) + .step(Step.of("celebrate", celebrateTask, "yay") + .onSuccess() // default — only runs if risky completed + .after("risky") + .build()) + .step(Step.of("recover", recoverTask, "fix") + .onFailure() // only runs if risky dead-lettered + .after("risky") + .build()); + +WorkflowRun run = queue.submitWorkflow(pipeline); +// worker: .trackWorkflows(pipeline) — conditional steps are deferred nodes, +// so the tracker must hold their payloads. +WorkflowStatus status = run.await(Duration.ofSeconds(30)); +status.state; // FAILED — even if recover ran (see below) +``` + +## Condition values + +| Builder method | Wire value | When the step runs | +|---|---|---| +| `.onSuccess()` | `"on_success"` | All predecessors completed successfully (default) | +| `.onFailure()` | `"on_failure"` | At least one predecessor failed | +| `.always()` | `"always"` | Once predecessors settle, regardless of outcome | +| `.condition(Condition)` | callable | The predicate returned `true` (see below) | + +`.condition(String)` accepts the three wire values directly and rejects +anything else. A step whose condition is not met transitions to `SKIPPED`. +Skipped steps propagate: all of their descendants are skipped too, unless +those descendants have a separate predecessor that did complete. + +## How it works + +The `WorkflowTracker` evaluates conditions when a predecessor node settles. It +reads the run plan from storage, checks each dependent node's condition against +the settled outcome, and either creates the deferred job or marks the node +`SKIPPED`. Because the tracker reconstructs run state from storage on every +event, submit and execute may be different processes. + + celebrate["celebrate\n(on_success)"] + risky --> recover["recover\n(on_failure)"]`} +/> + +When `risky` fails: `celebrate` → `SKIPPED`, `recover` → enqueued and runs. +When `risky` succeeds: `recover` → `SKIPPED`, `celebrate` → enqueued and runs. + +## Callable conditions + +`.condition(Condition)` takes a predicate over the run's settled state — a +`WorkflowContext` with completed nodes' results, every settled node's status, +and success/failure counts: + +```java +Workflow gate = Workflow.named("threshold") + .step("producer", producer, 10) + .step(Step.of("check", check, 0) + .condition(ctx -> ((Number) ctx.result("producer").orElse(0)).intValue() > 5) + .after("producer") + .build()); +``` + +A callable condition is code, so it cannot be persisted to storage. The +workflow must be registered on the running worker with +`trackWorkflows(workflow)` — and a child workflow passed to `subWorkflow` may +not use one at all (rejected at submit), because there is no registry across +the submit boundary. + +`WorkflowContext` exposes `runId()`, `result(nodeName)`, +`status(nodeName)`, `results()`, `statuses()`, `successCount()`, and +`failureCount()`. + +## A failed run stays failed + +When `risky` fails and `recover` runs, the workflow run still ends in state +`FAILED`. The `on_failure` handler executes but does not "recover" the run — +it is an error-handling side effect, not a circuit-breaker. To treat a failure +as a non-fatal branch, model it differently (for example, wrap the risky logic +in a task that catches internally and returns a status sentinel). + + + To roll back already-completed steps when a run fails, use + [Saga compensation](/java/guides/workflows/saga) instead of `on_failure` conditions. + + +## `always` steps + +Use `.always()` for teardown or notification steps that should run regardless +of the upstream outcome: + +```java +Workflow withCleanup = Workflow.named("with-cleanup") + .step("provision", provisionTask, spec) + .step("work", workTask, input, "provision") + .step(Step.of("cleanup", cleanupTask, spec) + .always() + .after("work") + .build()); +``` + +`cleanup` runs whether `work` succeeded or failed. + +## Step options + +| Builder method | Type | Description | +|---|---|---| +| `after(...)` | `String...` | Predecessor step name(s) | +| `onSuccess()` / `onFailure()` / `always()` | — | String condition shorthand (default `on_success`) | +| `condition(...)` | `String` or `Condition` | Wire value or runtime predicate | +| `maxRetries(...)` | `int` | Retry limit | +| `timeoutMs(...)` | `long` | Per-attempt timeout | +| `priority(...)` | `int` | Queue priority | +| `queue(...)` | `String` | Queue name | diff --git a/docs/content/docs/java/guides/workflows/fan-out.mdx b/docs/content/docs/java/guides/workflows/fan-out.mdx new file mode 100644 index 00000000..553f35cb --- /dev/null +++ b/docs/content/docs/java/guides/workflows/fan-out.mdx @@ -0,0 +1,168 @@ +--- +title: Fan-Out & Fan-In +description: "Split a step's result into parallel children, collect results into a downstream step." +--- + +Split a step's result into parallel child jobs, then collect all results into +a downstream step. + + square_0["square[0]"] + seed --> square_1["square[1]"] + seed --> square_2["square[2]"] + square_0 --> sum:::sink + square_1 --> sum + square_2 --> sum`} +/> + +## Basic fan-out and fan-in + +Use `.fanOut()` to expand the predecessor's list result into one child job per +item, and `.fanIn()` to collect the children's results into a combiner task. + +```java +Task seed = Task.of("seed", Integer.class); +Task square = Task.of("square", Integer.class); +Task> sum = Task.of("sum", new TypeReference>() {}); + +try (Taskito queue = Taskito.builder().url("fan.db").open()) { + // seed(4) -> [1,2,3,4]; square each -> [1,4,9,16]; sum all -> 30 + Workflow batch = Workflow.named("batch") + .step("seed", seed, 4) + .fanOut("square", square, FanMode.EACH, "seed") + .fanIn("sum", sum, FanMode.ALL, "square"); + + WorkflowRun run = queue.submitWorkflow(batch); + try (Worker worker = queue.worker() + .handle(seed, n -> IntStream.rangeClosed(1, n).boxed().toList()) + .handle(square, x -> x * x) + .handle(sum, xs -> xs.stream().mapToInt(Integer::intValue).sum()) + .trackWorkflows() + .start()) { + WorkflowStatus status = run.await(Duration.ofSeconds(30)); + String sumJob = status.node("sum").orElseThrow().jobId; + queue.getResult(sumJob, Integer.class); // Optional[30] + } +} +``` + +The single predecessor named in `after` is the producer: its return value must +be a list, and each item is passed to the child task as its payload. A fan-out +or fan-in step requires **exactly one** predecessor — the builder rejects zero +or several. + +Child nodes are named `square[0]`, `square[1]`, `square[2]` and appear in +status queries. The parent node carries `fanOutCount` (the number of children +spawned). + +```java +WorkflowStatus status = run.status().orElseThrow(); +status.node("square").orElseThrow().fanOutCount; // 3 +status.node("square[0]").orElseThrow().status; // COMPLETED +``` + +## How it works + +1. `seed` completes — the tracker reads its return value (must be a list) +2. The tracker expands the fan-out, creating N child nodes and N jobs — each + receives one item from the list; children are ready immediately +3. Children execute in parallel; their statuses surface as `square[0]`, + `square[1]`, … +4. As each child settles, the tracker checks whether every sibling is terminal +5. Once all children complete, it collects results in item order and creates + the deferred fan-in job +6. The `sum` fan-in node runs with the results list as its payload + +>T: outcome(seed, success) + T->>R: load run plan(runId) + T->>R: expandFanOut(3 children) + R-->>T: [childJobIds] + Note over R: Children execute in parallel + R->>T: outcome(square[0], success) + R->>T: outcome(square[1], success) + R->>T: outcome(square[2], success) + T->>R: checkFanOutCompletion → all done + T->>R: createDeferredJob(sum)`} +/> + +The `WorkflowTracker` is driven entirely by the worker outcome stream and +reconstructs the run plan from storage on each event, so submission and +execution may run in different processes. + +## Empty fan-out + +If the producer returns an empty list, the fan-out parent is marked completed +immediately with `fanOutCount` 0 and the fan-in runs with an empty list: + +```java +// seed handler returns List.of() → no children spawn +// sum receives [] → result is 0 +``` + +## Downstream steps + +Steps after the fan-in work normally — declare them with `after` pointing to +the fan-in node: + +```java +Workflow pipeline = Workflow.named("full-pipeline") + .step("seed", seed, 4) + .fanOut("square", square, FanMode.EACH, "seed") + .fanIn("sum", sum, FanMode.ALL, "square") + .step("notify", notify, "done", "sum"); // runs after sum +``` + +## Per-step options + +For per-child overrides, build the fan-out step explicitly: + +```java +Workflow batch = Workflow.named("batch") + .step("seed", seed, 4) + .step(Step.of("square", square) + .fanOut(FanMode.EACH) + .after("seed") + .queue("bulk") + .maxRetries(2) + .timeoutMs(30_000) + .priority(5) + .build()) + .fanIn("sum", sum, FanMode.ALL, "square"); +``` + +| Option | Description | +|--------|-------------| +| `after(...)` | The producer node (exactly one, required) | +| `fanOut(FanMode.EACH)` | Run the task once per item of the producer's result list | +| `fanIn(FanMode.ALL)` | Collect every child's result into one list | +| `queue(...)` | Queue name for each child | +| `maxRetries(...)` | Retry limit for each child | +| `timeoutMs(...)` | Timeout for each child | +| `priority(...)` | Priority for each child | + +Each child inherits the fan-out step's `queue`, `maxRetries`, `timeoutMs`, and +`priority`. A single fan-out is capped at 10,000 children — expansion beyond +that fails the fan-out node. + +## Failure handling + +If any child dead-letters, the tracker waits for the remaining children to +settle, then: + +- The fan-out parent is marked `failed` +- The fan-in and downstream `on_success` steps are `skipped` +- The workflow run transitions to `failed` + +```java +WorkflowStatus status = run.await(Duration.ofSeconds(30)); +if (status.state == WorkflowState.FAILED) { + status.failedStep().ifPresent(name -> System.err.println("failed node: " + name)); +} +``` diff --git a/docs/content/docs/java/guides/workflows/gates.mdx b/docs/content/docs/java/guides/workflows/gates.mdx new file mode 100644 index 00000000..674c4205 --- /dev/null +++ b/docs/content/docs/java/guides/workflows/gates.mdx @@ -0,0 +1,130 @@ +--- +title: Approval Gates +description: "Pause a workflow run until a human or external system approves or rejects." +--- + +An approval gate pauses a workflow run until something external resolves it. +No task runs at the gate — the run simply waits. + +```java +Workflow release = Workflow.named("release-pipeline") + .step("build", buildTask, artifact) + .gate("approve-deploy", + GateConfig.timeout( + Duration.ofHours(24), + GateAction.REJECT, + "Review build artifacts before deploying to production."), + "build") + .step("deploy", deployTask, artifact, "approve-deploy"); + +WorkflowRun run = queue.submitWorkflow(release); + +try (Worker worker = queue.worker() + .handle(buildTask, p -> p) + .handle(deployTask, p -> p) + .trackWorkflows(release) // the tracker holds deploy's payload + .start()) { + // Later — once a human signs off: + worker.approveGate(run.runId(), "approve-deploy"); + run.await(Duration.ofSeconds(30)); +} +``` + + gate["approve-deploy\n(waiting_approval)"]:::waiting + gate -- approved --> deploy:::run + gate -- rejected --> deploy_skip["deploy\n(skipped)"]:::skip`} +/> + + + A gated workflow must be registered on the tracking worker with + `trackWorkflows(workflow)`. The gate's downstream steps are deferred nodes — + their jobs are only created when the gate resolves, and the tracker supplies + their payloads from the registered definition. + + +## Gate node status + +While a gate is pending, its node status is `WAITING_APPROVAL`. The run itself +stays in `RUNNING` state. Downstream steps do not start until the gate +resolves. + +```java +WorkflowStatus status = run.status().orElseThrow(); +status.node("approve-deploy").orElseThrow().status; // WAITING_APPROVAL +``` + +## Resolving a gate + +Resolution goes through the `Worker` that is tracking workflows — calling +either method on a worker built without `trackWorkflows()` throws a +`WorkflowException`. + +```java +// Approve — the gate completes and downstream steps are enqueued. +worker.approveGate(runId, "approve-deploy"); + +// Reject with a reason — downstream steps are skipped, run transitions to FAILED. +worker.rejectGate(runId, "approve-deploy", "Artifacts failed QA review."); +``` + +## Timeout behaviour + +Build the gate with one of the `GateConfig` factories: + +```java +GateConfig.manual(); // waits forever +GateConfig.timeout(Duration.ofHours(24), GateAction.REJECT); // auto-reject after 24 h +GateConfig.timeout(Duration.ofHours(24), GateAction.APPROVE, "Auto-ships unless stopped."); +``` + +When the timeout elapses the gate auto-resolves according to its `GateAction`: + +| `GateAction` | Effect | +|---|---| +| `REJECT` (default) | Same as `rejectGate` — downstream skipped, run `FAILED` | +| `APPROVE` | Same as `approveGate` — downstream proceeds | + +The timeout must be positive; the timer is driven by the tracking worker and +is cancelled when the gate is resolved manually first. + +## Gate options + +| `GateConfig` component | Type | Default | Description | +|---|---|---|---| +| `timeout` | `Duration` | `null` (wait forever) | Auto-resolve after this long | +| `onTimeout` | `GateAction` | `REJECT` | Resolution taken when the timeout elapses | +| `message` | `String` | `null` | Human-readable description shown to the approver | + +## Rejection behaviour + +Rejecting a gate marks the gate node `FAILED` and skips its default +(`on_success`) successors; the workflow run ends in state `FAILED`. Any +`on_failure` / `always` successors still run — use those (see +[conditions](/java/guides/workflows/conditions)) if you need a rejection +branch. + + + Calling `approveGate` or `rejectGate` on a gate that has already been + resolved (approved, rejected, or timed out) is a no-op — the tracker ignores + stale resolutions. + + +## Webhooks and external triggers + +Any external system can drive a gate by reaching the process that hosts the +tracking worker. A common pattern is an HTTP endpoint in that process which +receives a webhook and resolves the gate: + +```java +// Resolving a gate is a privileged state transition: authenticate the caller, +// verify the webhook signature, and authorize access to this run first. +void onDeployApproval(String runId, boolean approved, String reason) { + if (approved) { + worker.approveGate(runId, "approve-deploy"); + } else { + worker.rejectGate(runId, "approve-deploy", reason); + } +} +``` diff --git a/docs/content/docs/java/guides/workflows/index.mdx b/docs/content/docs/java/guides/workflows/index.mdx new file mode 100644 index 00000000..b6a90cfe --- /dev/null +++ b/docs/content/docs/java/guides/workflows/index.mdx @@ -0,0 +1,78 @@ +--- +title: Workflows +description: "Orchestrate multi-step DAGs in Java with the Taskito workflow builder." +--- + +Orchestrate multi-step DAGs where each step is a registered task and its +`after` list declares its dependencies. The Rust core schedules steps in +topological order; a worker that calls `trackWorkflows()` advances the run as +each step settles. + +```java +Task extract = Task.of("extract", Integer.class); +Task transform = Task.of("transform", Integer.class); +Task load = Task.of("load", Integer.class); + +try (Taskito queue = Taskito.builder().url("pipeline.db").open()) { + Workflow etl = Workflow.named("etl") + .step("extract", extract, 1) + .step("transform", transform, 2, "extract") + .step(Step.of("load", load, 3).after("transform").maxRetries(5).build()); + + WorkflowRun run = queue.submitWorkflow(etl); + + try (Worker worker = queue.worker() + .handle(extract, p -> p) + .handle(transform, p -> p) + .handle(load, p -> p) + .trackWorkflows() + .start()) { + WorkflowStatus status = run.await(Duration.ofSeconds(30)); // blocks until terminal + System.out.println(status.state); // COMPLETED | FAILED | ... + status.nodes.forEach(n -> System.out.println(n.nodeName + " " + n.status)); + } +} +``` + +Each `.step(name, task, payload, after...)` binds a typed `Task` and its +payload; `Step.of(...)` opens a builder for per-step overrides (`queue`, +`maxRetries`, `timeoutMs`, `priority`, and the specialised kinds below). You +can also declare structural steps with `stepAfter(name, task, deps...)` and +supply payloads at submit time via +`queue.submitWorkflow(workflow, Map.of("extract", 5, ...))`. + + + The worker must opt in with `trackWorkflows()` — workflow node and run state + are driven from that worker's job outcomes. Workflows that use gates, + callable conditions, or sub-workflows must be registered on the tracking + worker with `trackWorkflows(workflow)` so the tracker holds their deferred + payloads and predicates. + + + + + + + + + diff --git a/docs/content/docs/java/guides/workflows/meta.json b/docs/content/docs/java/guides/workflows/meta.json new file mode 100644 index 00000000..55117da3 --- /dev/null +++ b/docs/content/docs/java/guides/workflows/meta.json @@ -0,0 +1,4 @@ +{ + "title": "Workflows", + "pages": ["index", "fan-out", "conditions", "gates", "sub-workflows", "saga", "canvas", "caching", "analysis"] +} diff --git a/docs/content/docs/java/guides/workflows/saga.mdx b/docs/content/docs/java/guides/workflows/saga.mdx new file mode 100644 index 00000000..517d36a5 --- /dev/null +++ b/docs/content/docs/java/guides/workflows/saga.mdx @@ -0,0 +1,145 @@ +--- +title: Sagas +description: "Automatically roll back completed steps in reverse order when a workflow run fails." +--- + +Saga compensation runs rollback tasks in reverse-dependency order when a +workflow run fails. Each step that completed and declared a compensator gets +rolled back; steps that never ran are left alone. + +## Basic example + +```java +Task reserve = Task.of("reserveInventory", Integer.class); +Task unreserve = Task.of("unreserveInventory", Integer.class); +Task charge = Task.of("chargePayment", Integer.class); +Task refund = Task.of("refundPayment", Integer.class); +Task ship = Task.of("shipOrder", Integer.class); +// No compensator for ship — if it never succeeded, there's nothing to undo. + +Workflow checkout = Workflow.named("checkout") + .step(Step.of("reserve", reserve, orderId).compensate(unreserve).build()) + .step(Step.of("charge", charge, orderId).after("reserve").compensate(refund).build()) + .step(Step.of("ship", ship, orderId).after("charge").maxRetries(0).build()); + +WorkflowRun run = queue.submitWorkflow(checkout); + +try (Worker worker = queue.worker() + .handle(reserve, id -> id) + .handle(charge, id -> id) + .handle(ship, id -> { throw new IllegalStateException("Warehouse offline"); }) + .handle(refund, chargeResult -> chargeResult) // undo charge + .handle(unreserve, reserveResult -> reserveResult) // undo reserve + .trackWorkflows() + .start()) { + WorkflowStatus status = run.await(Duration.ofSeconds(30)); + // ship fails → charge rolled back (refund) → reserve rolled back (unreserve) + status.state; // COMPENSATED + status.node("charge").orElseThrow().status; // COMPENSATED +} +``` + + charge --> ship + ship -. "run fails" .-> refund + refund --> unreserve`} +/> + +## How it works + +1. `ship` dead-letters — the tracker detects the run's failure. +2. The tracker collects every completed node that declared a compensator and + groups them into reverse-dependency waves: deepest (most downstream) nodes + first. +3. It enqueues each wave's compensation jobs, passing the forward step's + result as the payload. Compensators are ordinary registered tasks — they + run in the worker pool like any other job. +4. Rollbacks within one wave run in parallel; the next wave only dispatches + after the previous one drains. The run's state is `COMPENSATING` while + rollbacks are in flight. +5. Each rollback inherits the forward step's `queue`, `maxRetries`, + `timeoutMs`, and `priority`. +6. When all rollbacks complete the run transitions to `COMPENSATED`. +7. If a rollback itself dead-letters, the run ends `COMPENSATION_FAILED` — + remaining un-rolled-back steps are left as-is (fail-stop; no partial + retry). + +Each compensation job is enqueued with a deterministic idempotency key, so a +tracker restart during compensation does not double-run a rollback. The jobs +carry `compensation:true` metadata and never advance the forward run. + +## Run terminal states + +| State | Meaning | +|---|---| +| `COMPLETED` | All steps succeeded | +| `FAILED` | A step failed; no compensators applied | +| `COMPENSATED` | A step failed; all compensators completed | +| `COMPENSATION_FAILED` | A step failed; at least one compensator also failed | + + + `COMPENSATION_FAILED` is a manual-intervention state. Inspect + `status.nodes` for `COMPENSATION_FAILED` node statuses to identify which + rollback failed and whether data is partially consistent. + + +## Compensator payload + +The compensation task receives the forward step's return value as its payload. +Design forward tasks to return enough context for the compensator to act: + +```java +record Charge(String chargeId, String orderId) {} + +Task charge = Task.of("chargePayment", String.class); +Task refund = Task.of("refundPayment", Charge.class); + +worker.handle(charge, orderId -> new Charge(stripe.charge(orderId).id(), orderId)); +worker.handle(refund, result -> { stripe.refund(result.chargeId()); return null; }); +``` + +## Steps without compensators + +Steps that do not declare `compensate` are skipped during rollback. A step +that never ran (because it was downstream of the failing step) is also +skipped — only **completed** steps with a compensator are rolled back. + +```java +Workflow partial = Workflow.named("partial-saga") + .step(Step.of("a", taskA, 1).compensate(undoA).build()) // rolled back if run fails after a completes + .step("b", taskB, 2, "a") // no compensator — left as-is + .step(Step.of("c", taskC, 3).after("b").compensate(undoC).build()); + +// If b fails: only a is rolled back (b never completed; c never ran). +// If c fails: c has no completed result, so only a is rolled back. +``` + +Cache-hit nodes are also never compensated: a `CACHE_HIT` skipped its forward +task in this run, so there are no side effects here to undo. And a +compensator may only be declared on a plain task step — gates, sub-workflows, +and fan-out nodes have no forward result to replay, so `Step.build()` rejects +`compensate` on them. + +## Combining with conditions + +Conditions and compensation compose. An `onFailure()` handler runs first (as +the run settles toward failure); once the run is failed, the tracker then +rolls back the completed compensable steps: + +```java +Workflow safeCheckout = Workflow.named("safe-checkout") + .step(Step.of("reserve", reserve, orderId).compensate(unreserve).build()) + .step(Step.of("charge", charge, orderId).after("reserve").compensate(refund).build()) + .step("ship", ship, orderId, "charge") + .step(Step.of("notifyFailure", sendFailureEmail, orderId) + .onFailure() + .after("ship") + .build()); +``` diff --git a/docs/content/docs/java/guides/workflows/sub-workflows.mdx b/docs/content/docs/java/guides/workflows/sub-workflows.mdx new file mode 100644 index 00000000..65b06f23 --- /dev/null +++ b/docs/content/docs/java/guides/workflows/sub-workflows.mdx @@ -0,0 +1,123 @@ +--- +title: Sub-Workflows & Composition +description: "Compose workflow runs hierarchically — a parent step delegates to a child workflow." +--- + +A sub-workflow step delegates to a fully independent child workflow run. The +parent node goes `RUNNING` while the child executes and resolves when the +child reaches a terminal state. + +## Basic composition + +Build the child as an ordinary `Workflow` (do **not** submit it) and pass it +to `.subWorkflow()` on the parent: + +```java +// Child workflow — never submitted directly. +Workflow processChunk = Workflow.named("process-chunk") + .step("validate", validateTask, 10) + .step("transform", transformTask, 20, "validate") + .step("store", storeTask, 30, "transform"); + +// Parent workflow. +Workflow pipeline = Workflow.named("full-pipeline") + .step("prepare", prepareTask, 1) + .subWorkflow("process", processChunk, "prepare") + .step("finish", finishTask, 2, "process"); + +WorkflowRun run = queue.submitWorkflow(pipeline); + +try (Worker worker = queue.worker() + .handle(prepareTask, p -> p) + .handle(validateTask, p -> p) + .handle(transformTask, p -> p) + .handle(storeTask, p -> p) + .handle(finishTask, p -> p) + .trackWorkflows(pipeline) // required — the tracker submits the child + .start()) { + WorkflowStatus status = run.await(Duration.ofSeconds(30)); + status.state; // COMPLETED | FAILED +} +``` + + process["process\n(sub-workflow)"]:::sub + subgraph child ["child run: process-chunk"] + v[validate] --> t[transform] --> s[store] + end + process -. spawns .-> child + process --> finish:::run`} +/> + +## How it works + +1. When `prepare` completes, the tracker reaches the `process` node and + submits the child as a linked run — a separate entry in storage with its + own nodes and state. +2. The parent `process` node status is `RUNNING` while the child is in flight. +3. The tracker follows the child run's outcome events. +4. When the child run finalises: + - **Child completed** → parent node transitions to `COMPLETED`; `finish` is + enqueued normally. + - **Child failed** → parent node transitions to `FAILED`; `finish` and any + other downstream nodes are `SKIPPED`; the parent run ends `FAILED`. + +The child is a first-class run with its own run id, node statuses, and +lifecycle — it can be inspected independently through the dashboard and status +queries. + +## Submit-time validation + +The child crosses the submit boundary as data, so anything that lives only in +process memory is rejected when the parent is submitted: + +- A child step with no payload and no submit-time payload map cannot run — + `submitWorkflow` throws. +- A child step with a callable `condition` (a predicate is code, not data) — + `submitWorkflow` throws. + +Use string conditions (`onSuccess()` / `onFailure()` / `always()`) inside +child workflows instead. + +## Nesting + +Child workflows may themselves contain sub-workflow steps. There is no +enforced depth limit, but deeply nested hierarchies make observability +harder — prefer fan-out for homogeneous parallel work and sub-workflows for +distinct named pipelines. + +A workflow's entry step must be a plain step — a deferred kind (sub-workflow, +gate, fan-out, or conditioned step) only runs once a predecessor settles — so +give each sub-workflow step an `after`: + +```java +Workflow inner = Workflow.named("inner").step("a", taskA, 1); + +Workflow middle = Workflow.named("middle") + .step("x", taskX, 1) + .subWorkflow("inner", inner, "x"); + +Workflow outer = Workflow.named("outer") + .step("init", initTask, 1) + .subWorkflow("middle", middle, "init"); + +queue.submitWorkflow(outer); +``` + + + Pass the child `Workflow` object itself — never call `submitWorkflow` on it. + Submitting the child directly starts an immediate standalone run unlinked + from the parent. + + +## Failure propagation + +If the child run fails, the parent node is marked `FAILED` and the parent run +ends `FAILED`. The child's own node statuses are preserved on the child run. +This mirrors how a failed step propagates through the DAG — downstream steps +are skipped. + +To handle child failures without extra fallout, branch after the sub-workflow +node: place an `onFailure()` step after it and an `onSuccess()` step for the +happy path. See [Conditions](/java/guides/workflows/conditions). From fba3d54d3002fa707c733c1b64a34fcbde001f25 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Mon, 6 Jul 2026 22:35:51 +0530 Subject: [PATCH 02/16] docs(java): getting started and core guides --- .../java/getting-started/capabilities.mdx | 69 +++++++++++ .../docs/java/getting-started/concepts.mdx | 66 ++++++++++ .../java/getting-started/installation.mdx | 91 ++++++++++++++ .../docs/java/getting-started/meta.json | 1 + .../docs/java/getting-started/quickstart.mdx | 56 +++++++++ .../docs/java/guides/core/cancellation.mdx | 89 ++++++++++++++ .../docs/java/guides/core/enqueue-options.mdx | 59 +++++++++ .../docs/java/guides/core/execution-model.mdx | 60 +++++++++ docs/content/docs/java/guides/core/index.mdx | 18 +++ docs/content/docs/java/guides/core/meta.json | 1 + .../docs/java/guides/core/predicates.mdx | 92 ++++++++++++++ docs/content/docs/java/guides/core/queues.mdx | 55 +++++++++ .../docs/java/guides/core/scheduling.mdx | 45 +++++++ .../docs/java/guides/core/streaming.mdx | 61 ++++++++++ docs/content/docs/java/guides/core/tasks.mdx | 115 ++++++++++++++++++ .../content/docs/java/guides/core/workers.mdx | 83 +++++++++++++ docs/content/docs/java/guides/index.mdx | 70 +++++++++++ docs/content/docs/java/guides/meta.json | 5 + docs/content/docs/java/meta.json | 1 + 19 files changed, 1037 insertions(+) create mode 100644 docs/content/docs/java/getting-started/capabilities.mdx create mode 100644 docs/content/docs/java/getting-started/concepts.mdx create mode 100644 docs/content/docs/java/getting-started/installation.mdx create mode 100644 docs/content/docs/java/getting-started/meta.json create mode 100644 docs/content/docs/java/getting-started/quickstart.mdx create mode 100644 docs/content/docs/java/guides/core/cancellation.mdx create mode 100644 docs/content/docs/java/guides/core/enqueue-options.mdx create mode 100644 docs/content/docs/java/guides/core/execution-model.mdx create mode 100644 docs/content/docs/java/guides/core/index.mdx create mode 100644 docs/content/docs/java/guides/core/meta.json create mode 100644 docs/content/docs/java/guides/core/predicates.mdx create mode 100644 docs/content/docs/java/guides/core/queues.mdx create mode 100644 docs/content/docs/java/guides/core/scheduling.mdx create mode 100644 docs/content/docs/java/guides/core/streaming.mdx create mode 100644 docs/content/docs/java/guides/core/tasks.mdx create mode 100644 docs/content/docs/java/guides/core/workers.mdx create mode 100644 docs/content/docs/java/guides/index.mdx create mode 100644 docs/content/docs/java/guides/meta.json create mode 100644 docs/content/docs/java/meta.json diff --git a/docs/content/docs/java/getting-started/capabilities.mdx b/docs/content/docs/java/getting-started/capabilities.mdx new file mode 100644 index 00000000..57e0d812 --- /dev/null +++ b/docs/content/docs/java/getting-started/capabilities.mdx @@ -0,0 +1,69 @@ +--- +title: Capabilities at a glance +description: "Everything the Java SDK can do, with a link to each guide." +--- + +A map of the Java SDK's features. Everything here runs over the shared Rust +core — pick a backend, describe tasks, run workers. + +## Core + +| Feature | Guide | +|---|---| +| Typed task descriptors, enqueue, read results | [Tasks](/java/guides/core/tasks) | +| SQLite / Postgres / Redis backends | [Installation](/java/getting-started/installation#pick-a-backend) | +| Named queues, priority, pause/resume | [Queues](/java/guides/core/queues) | +| Cron-scheduled & delayed jobs | [Scheduling](/java/guides/core/scheduling) | +| Cooperative cancellation & progress | [Cancellation](/java/guides/core/cancellation) | +| Partial results via task logs | [Streaming](/java/guides/core/streaming) | +| Execution model (thread pools, autoscaling) | [Execution model](/java/guides/core/execution-model) | +| Compile-time `TaskHandler` bindings (no reflection) | [Tasks](/java/guides/core/tasks#typed-tasks-from-taskhandler) | + +## Reliability + +| Feature | Guide | +|---|---| +| Retries with backoff curves (`RetryPolicy`), dead-letter queue | [Retries](/java/guides/reliability/retries) · [Dead-letter](/java/guides/reliability/dead-letter) | +| Per-attempt timeouts | [Timeouts](/java/guides/reliability/timeouts) | +| Idempotent enqueue (`uniqueKey`) | [Idempotency](/java/guides/reliability/idempotency) | +| Distributed locks | [Locks](/java/guides/reliability/locks) | +| At-least-once delivery | [Guarantees](/java/guides/reliability/guarantees) | + +## Orchestration + +| Feature | Guide | +|---|---| +| DAG workflows: fan-out/in, conditions, gates, sub-workflows, sagas, visualization | [Workflows](/java/guides/workflows) | +| Decentralized work-stealing mesh | [Mesh](/java/guides/operations/mesh) | + +## Extensibility + +| Feature | Guide | +|---|---| +| Lifecycle events & middleware | [Events](/java/guides/extensibility/events) · [Middleware](/java/guides/extensibility/middleware) | +| Signed webhook delivery | [Webhooks](/java/guides/extensibility/webhooks) | +| Resource injection, enqueue predicates & interception | [Resources](/java/guides/resources) · [Predicates](/java/guides/core/predicates) | +| Signed cross-process references (proxies) | [Resources](/java/guides/resources) | +| Pluggable serializers (JSON / MessagePack / signed / encrypted) & payload codecs | [Serializers](/java/guides/extensibility/serializers) | +| Producer-side batching (`Batcher`) | [Extensibility](/java/guides/extensibility) | + +## Observability & ops + +| Feature | Guide | +|---|---| +| Stats, metrics, progress, worker heartbeats | [Monitoring](/java/guides/observability/monitoring) | +| Micrometer Observation / Sentry middleware | [Integrations](/java/guides/integrations) | +| React dashboard + REST API | [Dashboard](/java/guides/operations/dashboard) | +| In-process autoscaling + KEDA scaler endpoint | [KEDA](/java/guides/operations/keda) | +| Standalone CLI | [CLI](/java/guides/operations/cli) | + +## Framework integrations + +[Spring Boot 3 starter](/java/guides/integrations/spring) — auto-configures a +`Taskito` bean from `taskito.url` / `taskito.pool-size` / `taskito.namespace`. + + + Observability and MessagePack dependencies are `compileOnly` — add + `io.micrometer:micrometer-observation`, `io.sentry:sentry`, or + `org.msgpack:jackson-dataformat-msgpack` only if you use them. + diff --git a/docs/content/docs/java/getting-started/concepts.mdx b/docs/content/docs/java/getting-started/concepts.mdx new file mode 100644 index 00000000..57e12ff8 --- /dev/null +++ b/docs/content/docs/java/getting-started/concepts.mdx @@ -0,0 +1,66 @@ +--- +title: Concepts +description: "Client, task, job, worker, result — the Java mental model over the Rust core." +--- + +The Java SDK is a typed shell; the scheduler, dispatcher, worker pool, and +storage are all in the shared Rust engine. Five concepts cover the surface. + + + +## Client + +A `Taskito` is the handle to one store (SQLite file, Postgres schema, or Redis +prefix), opened with `Taskito.builder()...open()`. You enqueue jobs, run +workers, and inspect state through it; it is `AutoCloseable`, so hold it in +try-with-resources. Multiple processes pointed at the same storage share one +logical queue. `taskito.queue(name)` returns a handle to a single named queue +(pause / resume). + +```java +try (Taskito taskito = Taskito.builder().sqlite("taskito.db").open()) { ... } +``` + +## Task + +A **task** is a named unit of work described by a `Task` — the name plus the +payload type it deserializes to. Enqueuing references the task by name, so the +producer never needs the handler — only the worker does. Per-task defaults +(queue, priority, retries, timeout) are attached fluently on the descriptor. + +## Job + +Enqueuing a task creates a **job** — a row in storage with payload, priority, +status, and result. Jobs move through a state machine: `PENDING → RUNNING → +COMPLETE`, or `FAILED` (retrying), or `DEAD` (retries exhausted), or +`CANCELLED`. The engine claims jobs atomically, so the same job never runs +twice concurrently. + +## Worker + +`taskito.worker()` builds a worker: register handlers with `.handle(task, fn)`, +then `.start()`. The Rust core polls storage, claims due jobs, and hands each +one to your handler on a thread pool; the result is serialized and written +back. `close()` stops dispatch and drains in-flight jobs. + +## Result + +`taskito.getResult(id, type)` returns the handler's return value once the job +finishes, deserialized with the queue's +[serializer](/java/guides/extensibility/serializers). Results live in storage, +so any process sharing the store can read them; `awaitJob(id, timeout)` blocks +until the job is terminal. + + + The architecture is engine-level and shared across SDKs — see + [Architecture](/architecture/overview) for the job lifecycle, scheduler, + storage schema, and mesh internals. + diff --git a/docs/content/docs/java/getting-started/installation.mdx b/docs/content/docs/java/getting-started/installation.mdx new file mode 100644 index 00000000..13e60ba3 --- /dev/null +++ b/docs/content/docs/java/getting-started/installation.mdx @@ -0,0 +1,91 @@ +--- +title: Installation +description: "Install the Java SDK — Maven Central coordinates, bundled native binaries, three backends." +--- + +The Java SDK is a typed shell over the Taskito Rust core, bound through a +hand-written JNI layer. Baseline: **Java 17**. + + + + +```kotlin +implementation("org.byteveda:taskito:0.18.0") +annotationProcessor("org.byteveda:taskito-processor:0.18.0") // compile-time TaskHandler bindings +``` + + + + +```xml + + org.byteveda + taskito + 0.18.0 + +``` + +```xml + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.byteveda + taskito-processor + 0.18.0 + + + + +``` + + + + +The `taskito-processor` artifact is optional but recommended: it turns +`@TaskHandler`-annotated methods into typed `Task` constants at compile time — +zero runtime reflection, GraalVM-native-image friendly. See +[Tasks](/java/guides/core/tasks#typed-tasks-from-taskhandler). + +Companion artifacts: `org.byteveda:taskito-test` (in-memory backend for unit +tests, no native library) and `org.byteveda:taskito-spring` (Spring Boot 3 +starter). + +## Bundled native library + +The Rust core ships inside the JAR as a prebuilt library per platform — no +separate install: + +- linux x86_64 / aarch64 +- macOS x86_64 / aarch64 +- Windows x86_64 + +At runtime the platform binary is extracted (content-addressed, safe under +concurrent processes) and loaded. Two system properties tune this: +`-Dtaskito.native.lib=/abs/path` loads an explicit library (local development +against a fresh build), and `-Dtaskito.native.workdir=` relocates extraction on +hardened `noexec /tmp` hosts. + + + On JDK 22+ hot byte operations automatically take a Project Panama (FFM) fast + path; older JDKs transparently use JNI. FFM calls a restricted native method, + so launch with `--enable-native-access=ALL-UNNAMED` to grant access and + silence the JDK warning. + + +## Pick a backend + +```java +import org.byteveda.taskito.Taskito; + +Taskito.builder().sqlite("taskito.db").open(); // SQLite (default) +Taskito.builder().postgres(System.getenv("PG_URL")).open(); +Taskito.builder().redis("redis://localhost").open(); +``` + +`Taskito.builder().open()` with no backend configured uses SQLite at +`.taskito/taskito.db`. SQLite needs nothing else — the whole queue lives in one +file. Next: the [quickstart](/java/getting-started/quickstart). diff --git a/docs/content/docs/java/getting-started/meta.json b/docs/content/docs/java/getting-started/meta.json new file mode 100644 index 00000000..c18d0a61 --- /dev/null +++ b/docs/content/docs/java/getting-started/meta.json @@ -0,0 +1 @@ +{ "title": "Getting Started", "root": true, "pages": ["installation", "quickstart", "concepts", "capabilities"] } diff --git a/docs/content/docs/java/getting-started/quickstart.mdx b/docs/content/docs/java/getting-started/quickstart.mdx new file mode 100644 index 00000000..225e9e40 --- /dev/null +++ b/docs/content/docs/java/getting-started/quickstart.mdx @@ -0,0 +1,56 @@ +--- +title: Quickstart +description: "Define a task, enqueue a job, run a worker, read the result — in Java." +--- + +Four steps: open a client, describe a task, enqueue a job, run a worker. + +```java +import com.fasterxml.jackson.core.type.TypeReference; +import java.time.Duration; +import java.util.Map; +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.task.Task; +import org.byteveda.taskito.worker.Worker; + +// 1. Describe the task: a name plus its payload type. +Task> add = + Task.of("add", new TypeReference>() {}) + .retries(3) + .timeout(Duration.ofSeconds(30)); + +try (Taskito taskito = Taskito.builder().sqlite("taskito.db").open()) { + // 2. Enqueue a job (producer). + String id = taskito.enqueue(add, Map.of("a", 2, "b", 3)); + + // 3. Start a worker — same process or a separate one that shares the DB. + try (Worker worker = taskito.worker() + .handle(add, p -> p.get("a") + p.get("b")) + .start()) { + + // 4. Read the result once the job is terminal. + taskito.awaitJob(id, Duration.ofSeconds(20)); + int result = taskito.getResult(id, Integer.class).orElseThrow(); // 5 + } +} +``` + +The producer and the worker only need to share storage — `enqueue` from your web +app, run the worker from a separate process pointed at the same file / DSN. +`awaitJob` blocks by polling, which keeps demos and tests simple; long-lived +services react to completion through +[worker events](/java/guides/core/workers#events) instead. + + + `Task.of(name, type)` is a descriptor, not a function — the producer never + needs the handler body. The worker binds the function with + `.handle(task, fn)`, and payloads/results round-trip through the queue's + serializer (JSON by default). + + +## Where to next + +- [Concepts](/java/getting-started/concepts) — the mental model. +- [Tasks](/java/guides/core/tasks) — typed descriptors and per-task config. +- [Workers](/java/guides/core/workers) — running and stopping workers. +- [Workflows](/java/guides/workflows) — multi-step DAGs. diff --git a/docs/content/docs/java/guides/core/cancellation.mdx b/docs/content/docs/java/guides/core/cancellation.mdx new file mode 100644 index 00000000..e2ea8be1 --- /dev/null +++ b/docs/content/docs/java/guides/core/cancellation.mdx @@ -0,0 +1,89 @@ +--- +title: Cancellation +description: "Cancel pending jobs, cooperatively stop running ones, report progress." +--- + +A **pending** job is cancelled outright; a **running** job is cancelled +cooperatively — the producer sets a flag, the handler observes it and stops. + +## Cancel a pending job + +```java +boolean cancelled = taskito.cancel(jobId); // true if it was still pending +``` + +## Cancel a running job + +`requestCancel(jobId)` flips the cancel flag; the handler polls it with +`isCancelRequested(jobId)` and unwinds: + +```java +taskito.requestCancel(jobId); // caller side + +// handler side: poll between units of work +for (Chunk chunk : chunks) { + if (taskito.isCancelRequested(jobId)) { + throw new InterruptedException("cancelled"); + } + process(chunk); +} +``` + +When a handler throws while cancellation was requested, the core records the +job as `CANCELLED`, not failed — no retries, no dead-letter. A handler that +ignores the flag runs to completion. + + + +## Getting the job id into a handler + +A `TaskFunction` receives only the payload. Middleware `before` runs on the +same thread as the handler, so a `ThreadLocal` carries the id across: + +```java +static final ThreadLocal CURRENT_JOB = new ThreadLocal<>(); + +taskito.use(new Middleware() { + @Override public void before(TaskContext ctx) { CURRENT_JOB.set(ctx.jobId); } + @Override public void after(TaskContext ctx, Object result) { CURRENT_JOB.remove(); } + @Override public void onError(TaskContext ctx, Throwable t) { CURRENT_JOB.remove(); } +}); +``` + +Inside the handler, `CURRENT_JOB.get()` now yields the id for +`isCancelRequested`, `setProgress`, or `writeTaskLog`. + +## Progress + +Report progress 0–100 against a job; it surfaces as `Job.progress`, on the +dashboard, and via inspection: + +```java +taskito.setProgress(jobId, 40); +``` + +## React to cancellations + +Subscribe a worker listener to the `CANCELLED` outcome, or the `onCancel` +middleware hook: + +```java +taskito.worker() + .handle(export, p -> run(p)) + .on(EventName.CANCELLED, e -> cleanup(e.jobId)) + .start(); +``` + + + Cancellation is cooperative by design: the core never kills a handler thread, + so resources held by the task (transactions, files) are always released by + your own unwind path. + diff --git a/docs/content/docs/java/guides/core/enqueue-options.mdx b/docs/content/docs/java/guides/core/enqueue-options.mdx new file mode 100644 index 00000000..08229156 --- /dev/null +++ b/docs/content/docs/java/guides/core/enqueue-options.mdx @@ -0,0 +1,59 @@ +--- +title: Enqueue options +description: "Priority, delay, idempotency, metadata, and queue at enqueue time." +--- + +`EnqueueOptions` sets per-job options; pass it as the third argument to +`enqueue`. Unset fields fall back to the task's defaults, then to the core's. + +```java +taskito.enqueue(sendEmail, payload, EnqueueOptions.builder() + .queue("emails") + .priority(5) + .delay(Duration.ofSeconds(30)) + .uniqueKey("welcome:" + user.id()) + .metadata("source=signup") + .build()); +``` + +| Option | Description | +|---|---| +| `queue` | The queue name to route to. | +| `priority` | Higher dequeues first within a queue. | +| `maxRetries` | Override the task's retry budget for this job. | +| `timeout(duration)` / `timeoutMs` | Override the per-attempt timeout for this job. | +| `delay(duration)` / `delayMs` | Delay first execution (scheduled run). | +| `uniqueKey` | Idempotency key — a duplicate enqueue is a no-op while the first job is pending/running. | +| `metadata` | Free-form string stored with the job (surfaces on the dashboard / inspection). | +| `namespace` | Partition the store — jobs are only visible to clients/workers in the same namespace. | + +`toBuilder()` derives a modified copy from an existing instance, and the fluent +`Task` methods ([tasks](/java/guides/core/tasks#per-task-defaults)) cover the +common cases without touching the builder. + +## Idempotency + +`uniqueKey` dedupes concurrent producers — only one job runs for a given key +while an earlier one is still pending or running. `jobId(key)` is an alias in +the guide's vocabulary. See +[idempotency](/java/guides/reliability/idempotency). + +## Delayed jobs + +`delay` schedules the first attempt in the future; the scheduler picks it up +when due. For recurring schedules use +[periodic tasks](/java/guides/core/scheduling). + +## Batch enqueue + +`enqueueMany` inserts many jobs of one task in a single storage call. Payloads +share one `EnqueueOptions`; it returns the new ids in input order. + +```java +List ids = taskito.enqueueMany(resize, List.of(a, b, c), + EnqueueOptions.builder().priority(5).build()); +``` + +A `uniqueKey` on the batch dedupes there too: a payload whose key already has +an active job resolves to the existing job's id instead of failing the batch. +`enqueueAll` is an alias of `enqueueMany`. diff --git a/docs/content/docs/java/guides/core/execution-model.mdx b/docs/content/docs/java/guides/core/execution-model.mdx new file mode 100644 index 00000000..899bbe4f --- /dev/null +++ b/docs/content/docs/java/guides/core/execution-model.mdx @@ -0,0 +1,60 @@ +--- +title: Execution Models +description: "How the Java worker runs jobs — thread pools, concurrency, and scaling out." +--- + +The Rust core owns scheduling — it claims due jobs from storage and dispatches +them. Execution happens back in **your JVM**: the core hands each job to the +bound handler, which runs on the worker's `ExecutorService`. Payload +deserialization, middleware hooks, and result serialization all happen on that +handler thread. + +## Concurrency + +Jobs run **concurrently** on real threads — there is no event loop to block and +no interpreter lock. The pool is chosen at `start()`: + +- **Default** — a cached pool: threads are created on demand and reused. +- **`concurrency(n)`** — a fixed pool of `n` handler threads; jobs queue behind + them. +- **`autoscale(AutoscaleOptions.of(min, max))`** — a resizable pool an + `Autoscaler` grows and shrinks with queue depth (scoped to the worker's + `queues`, so foreign backlog can't inflate it). + +In-flight work is additionally bounded by `channelCapacity` (default 128), the +worker's dispatch buffer between the core and the pool. + +`batchSize` (default 1) controls how many jobs the scheduler claims per poll — +raise it to amortize polling under high throughput. + +```java +taskito.worker() + .handle(resize, p -> resizeImage(p)) + .concurrency(8) + .channelCapacity(256) + .batchSize(16) + .start(); +``` + +## CPU-bound work + +Handlers run on plain JVM threads, so CPU-heavy tasks are first-class — a long +computation only occupies its own thread. Size `concurrency` near the core +count for CPU-bound workloads; let the cached or autoscaled pool breathe for +I/O-bound ones, where threads mostly wait. + +## Scaling out + +Horizontal scaling is just more worker processes (or machines) pointed at +shared Postgres or Redis storage. The core claims each job for a single worker, +so adding workers adds throughput without duplicate execution. Within one +process, `autoscale` handles bursts; across processes, a queue-depth scaler +endpoint feeds KEDA — see [KEDA](/java/guides/operations/keda). Workers can +also join a work-stealing [mesh](/java/guides/operations/mesh) with +`mesh(options)`. + + + Handler exceptions are caught, reported to middleware `onError`, and fail the + attempt — the core then retries or dead-letters per the task's retry config. + A worker thread is never killed by a failing job. + diff --git a/docs/content/docs/java/guides/core/index.mdx b/docs/content/docs/java/guides/core/index.mdx new file mode 100644 index 00000000..580f1ef3 --- /dev/null +++ b/docs/content/docs/java/guides/core/index.mdx @@ -0,0 +1,18 @@ +--- +title: Core +description: "The building blocks of every Java taskito application." +--- + +The building blocks of every taskito application. + +| Guide | Description | +|---|---| +| [Tasks](/java/guides/core/tasks) | Describe tasks with `Task.of()`, configure retries, timeouts, and options | +| [Queues & Priority](/java/guides/core/queues) | Named queues, priority levels, and routing | +| [Workers](/java/guides/core/workers) | Start workers, control concurrency, graceful shutdown | +| [Execution Models](/java/guides/core/execution-model) | How tasks move from enqueue to completion | +| [Enqueue options](/java/guides/core/enqueue-options) | Per-job delay, priority, uniqueness, and metadata | +| [Predicates](/java/guides/core/predicates) | Gate an enqueue on a runtime condition | +| [Scheduling](/java/guides/core/scheduling) | Periodic tasks with cron expressions | +| [Cancellation](/java/guides/core/cancellation) | Cancel queued and in-flight jobs | +| [Streaming partial results](/java/guides/core/streaming) | Emit progress from a running task | diff --git a/docs/content/docs/java/guides/core/meta.json b/docs/content/docs/java/guides/core/meta.json new file mode 100644 index 00000000..13277466 --- /dev/null +++ b/docs/content/docs/java/guides/core/meta.json @@ -0,0 +1 @@ +{ "title": "Core", "pages": ["tasks", "queues", "workers", "execution-model", "enqueue-options", "predicates", "scheduling", "cancellation", "streaming"] } diff --git a/docs/content/docs/java/guides/core/predicates.mdx b/docs/content/docs/java/guides/core/predicates.mdx new file mode 100644 index 00000000..76e65701 --- /dev/null +++ b/docs/content/docs/java/guides/core/predicates.mdx @@ -0,0 +1,92 @@ +--- +title: Predicates +description: "Gate enqueues with composable predicates and richer allow/skip/defer/reject gates." +--- + +A gate is evaluated when a job is enqueued. If it rejects, no job is created — +a declarative way to say "only enqueue this under condition X". Two flavors: +boolean `Predicate`s and richer `EnqueueGate`s. + +## Boolean predicates + +`predicate(taskName, p)` registers a `Predicate` — `false` throws +`PredicateRejectedException` and nothing is enqueued: + +```java +Task charge = Task.of("charge", Integer.class); + +taskito.predicate("charge", ctx -> (Integer) ctx.payload() > 0); + +taskito.enqueue(charge, 50); // ok +taskito.enqueue(charge, -1); // throws PredicateRejectedException +``` + +The `PredicateContext` carries the task name and the payload — after +`onEnqueue` [middleware](/java/guides/extensibility/middleware) has run, so +gates see the rewritten payload. Multiple predicates on one task must all pass. + +Combine predicates with `Predicates.allOf`, `anyOf`, and `not`: + +```java +Predicate positive = ctx -> (Integer) ctx.payload() > 0; +Predicate small = ctx -> (Integer) ctx.payload() <= 10_000; + +taskito.predicate("charge", Predicates.allOf(positive, small)); +taskito.predicate("dispatch", Predicates.not(isHoliday)); +``` + +## Richer gates: allow / skip / defer / reject + +`gate(taskName, g)` registers an `EnqueueGate`, whose `EnqueueDecision` can do +more than pass/fail. Gates run in registration order; the first non-allow +decision wins. + +```java +taskito.gate("charge", ctx -> switch (classify(ctx.payload())) { + case OK -> EnqueueDecision.allow(); + case DUPLICATE -> EnqueueDecision.skip("already charged"); + case TOO_EARLY -> EnqueueDecision.defer(Duration.ofHours(1)); + case FRAUD -> EnqueueDecision.reject("fraud check failed"); +}); +``` + +| Decision | `enqueue` | `tryEnqueue` | +|---|---|---| +| `allow()` | Job created. | Job id present. | +| `skip(reason)` | Throws `EnqueueSkippedException`. | Returns empty. | +| `defer(delay)` / `deferUntil(instant)` | Job created, scheduled after `delay` (overrides any delay in the options). | Job id present. | +| `reject(reason)` | Throws `PredicateRejectedException`. | Throws too. | + +`tryEnqueue` is the gate-aware form — a skip becomes an empty `Optional` +instead of an exception: + +```java +Optional id = taskito.tryEnqueue(charge, amount); +``` + +## Recipes + +`Recipes` ships ready-made gates for common policies — time-based ones defer +out-of-window enqueues to the next open moment; filters skip silently: + +```java +taskito.gate("dispatch", Recipes.businessHours(ZoneId.of("America/New_York"))); +taskito.gate("digest", Recipes.dayOfWeek(zone, DayOfWeek.MONDAY, DayOfWeek.THURSDAY)); +taskito.gate("sync", Recipes.timeWindow(zone, LocalTime.of(22, 0), LocalTime.of(6, 0))); +taskito.gate("beta", Recipes.featureFlag("beta-export", flags)); +taskito.gate("charge", Recipes.payloadMatches(p -> ((Order) p).amount() > 0)); +``` + +## Batches + +`enqueueMany` gates each payload; a single rejection throws and no jobs are +enqueued — the batch API can't bypass a gate. Pre-filter the batch if you'd +rather drop rejected entries silently. + + + Gates run synchronously on the **producer** thread, so keep them fast and + side-effect-free — they decide whether work is created at all. To react once + a job is already running, use + [middleware](/java/guides/extensibility/middleware) or + [conditions](/java/guides/workflows) inside a workflow. + diff --git a/docs/content/docs/java/guides/core/queues.mdx b/docs/content/docs/java/guides/core/queues.mdx new file mode 100644 index 00000000..5d59a24a --- /dev/null +++ b/docs/content/docs/java/guides/core/queues.mdx @@ -0,0 +1,55 @@ +--- +title: Queues & Priority +description: "Route jobs to named queues, set priority, and pause/resume." +--- + +Every job belongs to a queue (default `"default"`). Queues are just routing +labels in shared storage — a worker chooses which queues it serves. + +```java +Task sendEmail = Task.of("send_email", EmailPayload.class).queue("emails"); + +String id = taskito.enqueue(sendEmail, payload); + +taskito.worker() + .handle(sendEmail, p -> deliver(p)) + .queues("emails", "default") // serve two queues + .start(); +``` + +The queue can also be set per enqueue via +`EnqueueOptions.builder().queue("emails")` — see +[enqueue options](/java/guides/core/enqueue-options). + +## Priority + +Higher `priority` dequeues first within a queue: + +```java +taskito.enqueue(report, payload, EnqueueOptions.builder().priority(10).build()); +// ahead of priority 0 jobs +``` + +## Pause and resume + +Pausing stops dispatch for a queue without stopping the worker — in-flight jobs +finish, new ones wait. Pause/resume live on the named-queue handle from +`taskito.queue(name)`: + +```java +Queue emails = taskito.queue("emails"); +emails.pause(); +emails.isPaused(); // true +taskito.listPausedQueues(); // ["emails"] +emails.resume(); +``` + +## Per-queue stats + +`statsByQueue(name)` scopes counters to one queue; `statsAllQueues()` maps every +queue name to its stats: + +```java +long backlog = taskito.statsByQueue("emails").pending; +Map all = taskito.statsAllQueues(); +``` diff --git a/docs/content/docs/java/guides/core/scheduling.mdx b/docs/content/docs/java/guides/core/scheduling.mdx new file mode 100644 index 00000000..94508eb5 --- /dev/null +++ b/docs/content/docs/java/guides/core/scheduling.mdx @@ -0,0 +1,45 @@ +--- +title: Scheduling +description: "Register periodic tasks on a cron expression; a worker fires them when due." +--- + +Schedule a task on a cron expression. A running worker enqueues it when due — +the scheduler's maintenance loop drives this, so no extra process is needed. + +```java +long nextFire = taskito.registerPeriodic( + PeriodicTask.builder("daily-digest", "digest", "0 0 9 * * *") + .payload(Map.of("edition", "morning")) + .queue("emails") + .timezone("America/New_York") + .build()); +``` + +`registerPeriodic` returns the next fire time (Unix ms). Re-registering the +same name replaces the schedule. + +| Builder method | Description | +|---|---| +| `payload(object)` | Payload passed to the task on each fire. | +| `queue(name)` | Queue to enqueue into. | +| `timezone(iana)` | IANA timezone for the cron expression (default UTC). | +| `enabled(flag)` | Register paused when `false` (default `true`). | + + + The cron field order is **seconds first** (`sec min hour dom mon dow`). + `"0 0 9 * * *"` is 09:00:00 daily, not every minute. + + +## Manage schedules + +```java +List all = taskito.listPeriodic(); // enabled and paused + +taskito.pausePeriodic("daily-digest"); // stop firing, keep the registration +taskito.resumePeriodic("daily-digest"); +taskito.deletePeriodic("daily-digest"); // unschedule; false if unknown +``` + +For a one-off future run, use +[`delay`](/java/guides/core/enqueue-options#delayed-jobs) instead of a periodic +schedule. diff --git a/docs/content/docs/java/guides/core/streaming.mdx b/docs/content/docs/java/guides/core/streaming.mdx new file mode 100644 index 00000000..6b964fe5 --- /dev/null +++ b/docs/content/docs/java/guides/core/streaming.mdx @@ -0,0 +1,61 @@ +--- +title: Streaming partial results +description: "Publish intermediate values from a task via task logs and consume them live." +--- + +A long-running task can publish intermediate results as it works; a consumer +polls them as they arrive — useful for ETL, ML training steps, or batch +progress. In Java, partials ride on **task logs**: durable, per-job log rows in +shared storage. + +## Publish + +From inside the handler, write a log row against the job (get the id across +with the [middleware pattern](/java/guides/core/cancellation#getting-the-job-id-into-a-handler)): + +```java +for (int epoch = 1; epoch <= epochs; epoch++) { + double loss = trainEpoch(epoch); + taskito.writeTaskLog(jobId, "train", "result", + "{\"epoch\": " + epoch + ", \"loss\": " + loss + "}"); +} +``` + +`writeTaskLog(jobId, taskName, level, message)` takes any level string; an +overload adds an `extra` payload. By the cross-SDK contract, published partials +use level `"result"` — workers in other processes that publish partials store +them the same way, so they are readable here. + +## Consume + +`getTaskLogs(jobId)` returns every row for the job, oldest first; poll it until +the job is terminal: + +```java +String id = taskito.enqueue(train, config); + +int seen = 0; +while (taskito.getJob(id).map(j -> j.status == JobStatus.PENDING || j.status == JobStatus.RUNNING).orElse(false)) { + List logs = taskito.getTaskLogs(id); + for (TaskLog log : logs.subList(seen, logs.size())) { + if ("result".equals(log.level)) { + render(log.message); + } + } + seen = logs.size(); + Thread.sleep(200); +} + +int result = taskito.getResult(id, Integer.class).orElseThrow(); +``` + +It works from any process sharing the storage, and a late consumer still reads +every value already published. Each `TaskLog` carries `jobId`, `taskName`, +`level`, `message`, `extra`, and `loggedAt` (Unix ms). + + + Partials are stored as task-log rows, so they survive until the log is + purged. Pipe the poll loop straight to an SSE / WebSocket endpoint to push + progress to a browser. For a single scalar 0–100, prefer + [`setProgress`](/java/guides/core/cancellation#progress). + diff --git a/docs/content/docs/java/guides/core/tasks.mdx b/docs/content/docs/java/guides/core/tasks.mdx new file mode 100644 index 00000000..ba1ab2bd --- /dev/null +++ b/docs/content/docs/java/guides/core/tasks.mdx @@ -0,0 +1,115 @@ +--- +title: Tasks +description: "Describe tasks with Task.of, attach per-task defaults, and bind handlers." +--- + +A task is a named unit of work. The name is the contract: producers enqueue by +name, workers execute the bound handler. In Java a task is described by an +immutable `Task` — the name plus the type its payload deserializes to. + +```java +Task sendEmail = Task.of("send_email", EmailPayload.class); +``` + +For generic payloads that a `Class` token can't express, use a Jackson +`TypeReference`: + +```java +Task> sendEmail = + Task.of("send_email", new TypeReference>() {}); +``` + +## Per-task defaults + +Fluent methods attach default enqueue options; each returns a new descriptor +(the type is immutable): + +```java +Task sendEmail = Task.of("send_email", EmailPayload.class) + .queue("emails") + .priority(5) + .retries(3) + .timeout(Duration.ofSeconds(30)) + .delay(Duration.ofSeconds(10)); +``` + +| Method | Description | +|---|---| +| `queue(name)` | Route jobs to a named queue ([queues](/java/guides/core/queues)). | +| `priority(n)` | Higher dequeues first within a queue. | +| `retries(n)` / `maxRetries(n)` | Attempts before dead-lettering. | +| `timeout(duration)` / `timeoutMs(ms)` | Per-attempt timeout. | +| `delay(duration)` / `delayMs(ms)` | Delay first execution. | +| `retryPolicy(policy)` | Retry-backoff curve (below). | +| `codecs(names...)` | Named payload codecs applied to this task's payload. | +| `withOptions(options)` | Replace the defaults with an `EnqueueOptions` wholesale. | + +`RetryPolicy` shapes the wait between attempts — the budget still comes from +`retries`: + +```java +Task settle = Task.of("settle", Order.class) + .retries(5) + .retryPolicy(RetryPolicy.exponential(Duration.ofSeconds(1), Duration.ofMinutes(1))); +// or exact per-attempt delays, applied without jitter: +Task poll = Task.of("poll", Order.class) + .retries(3) + .retryPolicy(RetryPolicy.delays(Duration.ofSeconds(5), Duration.ofSeconds(30), Duration.ofMinutes(2))); +``` + +The core scheduler owns retry execution, so retries stay durable and survive +worker crashes. Retry policies are registered with the worker on `start()`. + +## Enqueue and bind + +Producers enqueue against the descriptor (or a bare name); workers bind the +function: + +```java +String id = taskito.enqueue(sendEmail, payload); // typed +String id2 = taskito.enqueue("send_email", payload); // by name, default options + +try (Worker worker = taskito.worker() + .handle(sendEmail, p -> deliver(p)) // Task + TaskFunction + .handle("resize", ImageJob.class, p -> resize(p)) // name + payload type + .start()) { ... } +``` + +A handler is a `TaskFunction`: it receives the deserialized payload and +returns a result (or `null`). The result is serialized and stored as the job +result. `Handler.of(task, fn)` pairs the two for registration via +`register(handler)` or a `HandlerRegistry`. + +## Typed tasks from TaskHandler + +Annotate handler methods with `@TaskHandler`; the compile-time processor +(`org.byteveda:taskito-processor`) generates a `Tasks` companion with a +typed `Task` constant per method plus a `bind(...)` — name declared once, full +generics, zero runtime reflection: + +```java +class EmailTasks { + @TaskHandler("send_email") // explicit name + String send(EmailPayload p) { ... } + + @TaskHandler // name defaults to "report" + Report report(List metrics) { ... } +} + +// generated EmailTasksTasks: +String id = taskito.enqueue(EmailTasksTasks.SEND, payload); + +taskito.worker() + .apply(b -> EmailTasksTasks.bind(b, new EmailTasks())) + .start(); +``` + +The annotation also accepts `queue`, `maxRetries`, `timeoutMs`, and `priority` +for per-task defaults. See +[installation](/java/getting-started/installation) for the processor setup. + + + Register a handler for every task a worker's queues can carry — a job whose + task has no handler on the claiming worker fails with "no handler + registered". Re-registering a task name replaces the previous handler. + diff --git a/docs/content/docs/java/guides/core/workers.mdx b/docs/content/docs/java/guides/core/workers.mdx new file mode 100644 index 00000000..40d16217 --- /dev/null +++ b/docs/content/docs/java/guides/core/workers.mdx @@ -0,0 +1,83 @@ +--- +title: Workers +description: "Build, run, and stop workers; thread pools, events, graceful shutdown." +--- + +A worker polls storage, claims due jobs, dispatches them to bound handlers, and +writes results back. Build one from the client: + +```java +Worker worker = taskito.worker() + .handle(add, p -> p.get("a") + p.get("b")) + .queues("default") + .concurrency(4) + .start(); +// ... later +worker.close(); // stop dispatch, drain in-flight jobs, free the native worker +``` + +`start()` returns immediately; scheduling runs in the background on the Rust +core, and handlers execute on a Java thread pool. The worker registers itself +and heartbeats, so it appears in `listWorkers()` and on the dashboard; +`close()` unregisters it. + +## Builder options + +| Option | Description | +|---|---| +| `handle(task, fn)` | Bind a handler ([tasks](/java/guides/core/tasks)). | +| `queues(names...)` | Queue names to serve (default `"default"`). | +| `concurrency(n)` | Fixed handler-thread count; `0` (default) uses a cached pool. | +| `channelCapacity(n)` | In-flight dispatch buffer (default 128). | +| `batchSize(n)` | Jobs claimed per poll (default 1) — see [execution model](/java/guides/core/execution-model). | +| `autoscale(options)` | Resize the pool between min/max threads by queue depth. | +| `mesh(options)` | Join a work-stealing mesh. | +| `on(event, listener)` | Subscribe to job outcomes (below). | +| `trackWorkflows()` | Drive [workflow](/java/guides/workflows) bookkeeping from this worker's outcomes. | +| `apply(customizer)` | Apply a builder customizer (e.g. a generated `XxxTasks.bind`). | + +## Events + +Outcome listeners fire when the core decides a job's result: + +```java +taskito.worker() + .handle(add, p -> compute(p)) + .on(EventName.SUCCESS, e -> log.info("done: " + e.jobId)) + .on(EventName.DEAD, e -> alert(e.jobId, e.error)) + .start(); +``` + +`EventName` values: `SUCCESS`, `RETRY`, `DEAD`, `CANCELLED`. For richer hooks +(before/after execution, enqueue-time), use +[middleware](/java/guides/extensibility/middleware). + +## Graceful shutdown + +`Worker` is `AutoCloseable`. `close()` stops scheduling, lets in-flight +handlers finish (30 s grace, then interrupt), then frees the native worker; +`stop()` only stops dispatch. `awaitShutdown()` blocks until `close()` is +called — the usual service `main` wires it to a shutdown hook: + +```java +Taskito taskito = Taskito.builder().sqlite("taskito.db").open(); +Worker worker = taskito.worker().handle(add, p -> compute(p)).start(); + +Runtime.getRuntime().addShutdownHook(new Thread(() -> { + worker.close(); + taskito.close(); +})); +worker.awaitShutdown(); +``` + + + Don't put the worker in try-with-resources *and* call `awaitShutdown()` + inside the block — the block can't exit to trigger `close()`, so it + deadlocks. Use try-with-resources for bounded work (tests), the shutdown-hook + pattern for services. + + + + Producers and workers only share storage. Enqueue from a web process and run + workers from a separate process pointed at the same file / DSN. + diff --git a/docs/content/docs/java/guides/index.mdx b/docs/content/docs/java/guides/index.mdx new file mode 100644 index 00000000..c864a65e --- /dev/null +++ b/docs/content/docs/java/guides/index.mdx @@ -0,0 +1,70 @@ +--- +title: Guides +description: Topical guides covering the Java SDK's surface area. +--- + +import { Cards, Card } from "fumadocs-ui/components/card"; +import { + Boxes, + ShieldCheck, + Wrench, + Activity, + Layers, + Workflow, + Plug, + Puzzle, +} from "lucide-react"; + +Pick the topic you're working through. Each guide is self-contained, with code +samples and gotchas. + + + } + title="Core" + href="/java/guides/core" + description="Describe tasks, route to queues, run workers, schedule periodics." + /> + } + title="Reliability" + href="/java/guides/reliability" + description="Retries, dead letters, idempotency, locks, error handling." + /> + } + title="Workflows" + href="/java/guides/workflows" + description="DAGs, fan-out, fan-in, conditions, gates, sub-workflows, saga compensation." + /> + } + title="Extensibility" + href="/java/guides/extensibility" + description="Custom serializers, payload codecs, middleware, events, webhooks." + /> + } + title="Resources" + href="/java/guides/resources" + description="Inject pools, clients, and sessions by name; intercept enqueues; signed proxies." + /> + } + title="Integrations" + href="/java/guides/integrations" + description="Micrometer, Sentry, and the Spring Boot 3 starter." + /> + } + title="Operations" + href="/java/guides/operations" + description="Deployment, scaling, mesh, KEDA, the CLI." + /> + } + title="Observability" + href="/java/guides/observability" + description="Monitor queue health, the dashboard, and structured logs." + /> + diff --git a/docs/content/docs/java/guides/meta.json b/docs/content/docs/java/guides/meta.json new file mode 100644 index 00000000..f0535762 --- /dev/null +++ b/docs/content/docs/java/guides/meta.json @@ -0,0 +1,5 @@ +{ + "title": "Guides", + "root": true, + "pages": ["index", "core"] +} diff --git a/docs/content/docs/java/meta.json b/docs/content/docs/java/meta.json new file mode 100644 index 00000000..12f0a7d6 --- /dev/null +++ b/docs/content/docs/java/meta.json @@ -0,0 +1 @@ +{ "title": "Java", "pages": ["getting-started", "guides"] } From e02013972d41f2bcd037c9d833a0d590da305ab7 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Mon, 6 Jul 2026 22:37:50 +0530 Subject: [PATCH 03/16] docs(java): reliability and resources guides --- .../java/guides/reliability/concurrency.mdx | 56 +++++ .../java/guides/reliability/dead-letter.mdx | 39 ++++ .../guides/reliability/error-handling.mdx | 97 +++++++++ .../java/guides/reliability/guarantees.mdx | 45 ++++ .../java/guides/reliability/idempotency.mdx | 52 +++++ .../docs/java/guides/reliability/index.mdx | 18 ++ .../docs/java/guides/reliability/locks.mdx | 58 +++++ .../docs/java/guides/reliability/meta.json | 1 + .../java/guides/reliability/rate-limiting.mdx | 54 +++++ .../docs/java/guides/reliability/retries.mdx | 46 ++++ .../docs/java/guides/reliability/timeouts.mdx | 32 +++ .../guides/resources/dependency-injection.mdx | 198 ++++++++++++++++++ .../docs/java/guides/resources/index.mdx | 66 ++++++ .../java/guides/resources/interception.mdx | 91 ++++++++ .../docs/java/guides/resources/meta.json | 1 + .../docs/java/guides/resources/proxies.mdx | 154 ++++++++++++++ 16 files changed, 1008 insertions(+) create mode 100644 docs/content/docs/java/guides/reliability/concurrency.mdx create mode 100644 docs/content/docs/java/guides/reliability/dead-letter.mdx create mode 100644 docs/content/docs/java/guides/reliability/error-handling.mdx create mode 100644 docs/content/docs/java/guides/reliability/guarantees.mdx create mode 100644 docs/content/docs/java/guides/reliability/idempotency.mdx create mode 100644 docs/content/docs/java/guides/reliability/index.mdx create mode 100644 docs/content/docs/java/guides/reliability/locks.mdx create mode 100644 docs/content/docs/java/guides/reliability/meta.json create mode 100644 docs/content/docs/java/guides/reliability/rate-limiting.mdx create mode 100644 docs/content/docs/java/guides/reliability/retries.mdx create mode 100644 docs/content/docs/java/guides/reliability/timeouts.mdx create mode 100644 docs/content/docs/java/guides/resources/dependency-injection.mdx create mode 100644 docs/content/docs/java/guides/resources/index.mdx create mode 100644 docs/content/docs/java/guides/resources/interception.mdx create mode 100644 docs/content/docs/java/guides/resources/meta.json create mode 100644 docs/content/docs/java/guides/resources/proxies.mdx diff --git a/docs/content/docs/java/guides/reliability/concurrency.mdx b/docs/content/docs/java/guides/reliability/concurrency.mdx new file mode 100644 index 00000000..8343283e --- /dev/null +++ b/docs/content/docs/java/guides/reliability/concurrency.mdx @@ -0,0 +1,56 @@ +--- +title: Concurrency +description: "Size, fix, or autoscale the worker's handler thread pool." +--- + +Concurrency in the Java SDK is a **worker-level** setting: each worker runs its +handlers on a thread pool, and the pool's size caps how many jobs that worker +executes at once. + +```java +Worker worker = queue.worker() + .handle(TRANSCODE, this::transcode) + .concurrency(2) // at most 2 handlers run at once on this worker + .start(); +``` + +`concurrency(0)` (the default) uses a cached pool that grows with demand; +any positive value fixes the thread count. Jobs beyond the cap stay `PENDING` +and dispatch as running ones finish. + +## Autoscaling + +Instead of a fixed size, let the pool track queue depth — it resizes between +`min` and `max` threads, re-evaluated every couple of seconds: + +```java +Worker worker = queue.worker() + .handle(TRANSCODE, this::transcode) + .autoscale(AutoscaleOptions.of(2, 16)) // 2..16 threads, ~10 queued tasks per thread + .start(); +``` + +## Scaling out + +Total parallelism is the sum across workers — run more worker processes over +the same storage to scale horizontally, and partition with `queues(...)` so a +heavy task class gets its own capacity: + +```java +Worker videoWorker = queue.worker() + .handle(TRANSCODE, this::transcode) + .queues("video") + .concurrency(4) + .start(); +``` + +Pair a bounded pool with [pooled resources](/java/guides/resources/dependency-injection) +when handlers share an expensive client — the resource pool bounds the client +instances while the thread pool bounds the executions. + + + The SDK does not enforce a per-task concurrency cap across workers — the + bound is per worker. To cap a task globally, give it a dedicated queue and a + single worker sized to the limit, or serialize the critical section with a + [distributed lock](/java/guides/reliability/locks). + diff --git a/docs/content/docs/java/guides/reliability/dead-letter.mdx b/docs/content/docs/java/guides/reliability/dead-letter.mdx new file mode 100644 index 00000000..adecb92a --- /dev/null +++ b/docs/content/docs/java/guides/reliability/dead-letter.mdx @@ -0,0 +1,39 @@ +--- +title: Dead-letter queue +description: "Inspect, retry, and purge jobs that exhausted their retries." +--- + +When a job exhausts its retries it moves to the **dead-letter queue** (DLQ) and +the `DEAD` event fires. The DLQ keeps the payload and error history so you can +investigate and replay. + +```java +List dead = queue.listDead(50, 0); // newest first, paged +List charges = queue.listDeadByTask("charge", 50, 0); +String newJobId = queue.retryDead(deadId); // re-enqueue (preserves metadata) +queue.deleteDead(deadId); // drop one entry +long dropped = queue.purgeDead(olderThanMs); // bulk-drop entries older than a cutoff +long removed = queue.purgeDeadByTask("charge"); // drop every entry for one task +``` + +Each `DeadJob` records the original job id, queue, task name, final error, +retry count, failure time, metadata, and how many times it has been replayed +from the DLQ (`dlqRetryCount`). Inspect why a job failed attempt by attempt: + +```java +List history = queue.jobErrors(dead.originalJobId); +``` + +From the CLI: + +```bash +taskito --url taskito.db dlq list +taskito --url taskito.db dlq retry +taskito --url taskito.db dlq delete +``` + + + A retried dead job re-enters as `PENDING` with a fresh retry budget. Metadata + survives the DLQ round-trip, so context attached at enqueue is preserved + through replay. + diff --git a/docs/content/docs/java/guides/reliability/error-handling.mdx b/docs/content/docs/java/guides/reliability/error-handling.mdx new file mode 100644 index 00000000..d9755ade --- /dev/null +++ b/docs/content/docs/java/guides/reliability/error-handling.mdx @@ -0,0 +1,97 @@ +--- +title: Error Handling +description: "What happens when a handler throws — retries, timeouts, and the exception hierarchy." +--- + +When a handler throws, Taskito decides the job's fate from its retry budget. +Each failing attempt runs the `onError` middleware hook, then: + +1. **Retry** — if attempts remain, the job is rescheduled with + [backoff](/java/guides/reliability/retries). +2. **Dead-letter** — once the budget is exhausted, the job moves to the + [dead-letter queue](/java/guides/reliability/dead-letter). + +```java +Task FETCH = Task.of("fetch", String.class) + .maxRetries(3) + .retryPolicy(RetryPolicy.exponential(Duration.ofSeconds(1), Duration.ofMinutes(1))); +``` + +Returning normally is success; an uncaught throw is failure. `TaskFunction` +declares `throws Exception`, so checked exceptions propagate without wrapping. +There is no in-handler "stop retrying" signal — leave `maxRetries` at its +default of `0` for fire-once tasks. + +## Timeouts + +A per-attempt [timeout](/java/guides/reliability/timeouts) counts as a failure; +the outcome carries `timedOut = true` so hooks can tell timeouts apart from +other errors. + +```java +Task SLOW = Task.of("slow", String.class).timeout(Duration.ofSeconds(30)); +``` + +## Inspecting failures + +Every attempt's error is recorded: + +```java +List errors = queue.jobErrors(jobId); // one entry per failed attempt +List failed = queue.listJobs(JobFilter.builder().status(JobStatus.FAILED).build()); +List dead = queue.listDead(50, 0); // exhausted jobs +``` + +## Reacting to outcomes + +[Middleware](/java/guides/extensibility/middleware) hooks and worker event +listeners fire as the core decides each outcome: + +```java +queue.use(new Middleware() { + @Override + public void onError(TaskContext context, Throwable error) { + log.error(context.taskName, error); // each throwing attempt + } + + @Override + public void onRetry(OutcomeEvent event) { + metrics.increment("retry", event.taskName); + } + + @Override + public void onDeadLetter(OutcomeEvent event) { + alertOps(event); + } +}); + +Worker worker = queue.worker() + .handle(FETCH, this::fetch) + .on(EventName.DEAD, event -> pageOncall(event)) + .start(); +``` + +## The exception hierarchy + +Every SDK error is an unchecked `TaskitoException`; the `errors` package +narrows it so callers can catch exactly what they care about: + +| Exception | Raised when | +|---|---| +| `ConfigurationException` | The client was misconfigured — missing connection URL, unusable storage directory. | +| `SerializationException` | A payload or result failed to (de)serialize. | +| `CryptoException` | A signing or encrypting serializer failed — HMAC mismatch, bad tag/IV. Subtype of `SerializationException`. | +| `InterceptionException` | An [interceptor](/java/guides/resources/interception) rejected the enqueue; no job was created. | +| `PredicateRejectedException` | An enqueue gate returned `Reject`; no job was created. | +| `EnqueueSkippedException` | A gate returned `Skip` on `enqueue` — use `tryEnqueue` to get an empty `Optional` instead. | +| `LockException` | A [distributed lock](/java/guides/reliability/locks) operation failed or was interrupted. | +| `ResourceException` | A [worker resource](/java/guides/resources/dependency-injection) could not be resolved — unknown name, scope violation, exhausted pool. | +| `ProxyException` | A [proxy ref](/java/guides/resources/proxies) failed to verify or reconstruct. | +| `WorkflowException` | A workflow could not be driven to completion — run not found, missing payload or condition. | +| `WebhookException` | A webhook could not be stored, loaded, signed, or its payload encoded. | + + + A throw fails only the current *attempt*. To fail fast with no retries, keep + `maxRetries` at `0`; to make failures safe to retry, keep handlers + [idempotent](/java/guides/reliability/idempotency). + diff --git a/docs/content/docs/java/guides/reliability/guarantees.mdx b/docs/content/docs/java/guides/reliability/guarantees.mdx new file mode 100644 index 00000000..2d9a7666 --- /dev/null +++ b/docs/content/docs/java/guides/reliability/guarantees.mdx @@ -0,0 +1,45 @@ +--- +title: Delivery Guarantees +description: "At-least-once execution, job claiming, and staying correct under retries." +--- + +Taskito gives **at-least-once** execution: every enqueued job runs to completion +at least once. A job can also run *more* than once — a worker that crashes +mid-task (after claiming it, before recording the result) leaves the job to be +picked up again. Design handlers to tolerate that. + +## How a job is claimed + +A worker atomically claims a pending job — marking it running in a single +guarded update — before executing, so two workers never run the same job +concurrently. If that worker dies, the claim lapses and the scheduler dispatches +the job again. + +## Staying correct: idempotency + +Because a job may run more than once, make side effects idempotent: + +- Upsert / conditional-write keyed by a stable id rather than blind inserts. +- Dedupe external calls with a stable business id carried in the payload. +- Collapse duplicate *enqueues* into one job with + [`uniqueKey`](/java/guides/reliability/idempotency). + +```java +Worker worker = queue.worker() + .handle(CHARGE, order -> payments.charge(order.total(), order.id())) + .start(); +``` + +## Not guaranteed + +- **Exactly-once** — not offered anywhere in the system; pair at-least-once with + idempotent handlers instead. +- **Strict ordering** — within a queue, jobs dequeue by priority then age, but + concurrent workers run them in parallel; don't rely on one finishing before + another. Use a [workflow](/java/guides/workflows) when order matters. + + + A completed job's result is stored and can be read by id from any process + sharing the storage — `queue.getResult(jobId, Receipt.class)`. In tests, + `queue.awaitJob(jobId, timeout)` blocks until the job settles. + diff --git a/docs/content/docs/java/guides/reliability/idempotency.mdx b/docs/content/docs/java/guides/reliability/idempotency.mdx new file mode 100644 index 00000000..9178ae15 --- /dev/null +++ b/docs/content/docs/java/guides/reliability/idempotency.mdx @@ -0,0 +1,52 @@ +--- +title: Idempotency +description: "Dedupe concurrent enqueues with a uniqueKey; design tasks to re-run safely." +--- + +## Deduping enqueues + +A `uniqueKey` makes a duplicate enqueue resolve to the **existing job** while +the first job for that key is still pending or running: + +```java +EnqueueOptions once = EnqueueOptions.builder() + .uniqueKey("welcome:" + user.id()) + .build(); + +String first = queue.enqueue(WELCOME, user.id(), once); +String second = queue.enqueue(WELCOME, user.id(), once); // same id — no new job +``` + +Backed by a partial unique index in the store, so the dedup is atomic across +producers and processes. Once the first job finishes, the key frees up. +`EnqueueOptions.Builder.jobId(...)` is an alias for `uniqueKey` when the key +you have in hand *is* the business id. + +Batch enqueues participate too — a duplicate key inside `enqueueMany` resolves +to the existing job instead of failing the batch: + +```java +List ids = queue.enqueueMany(WELCOME, payloads, once); +// duplicates map to the already-enqueued job's id, in input order +``` + +## Idempotent tasks + +Separately from enqueue dedup, **handlers should be safe to run more than +once**. At-least-once delivery means a crash after a side effect but before the +result write will re-execute the task on recovery. + +```java +Worker worker = queue.worker() + .handle(CHARGE, order -> { + // key the external effect on a stable id so a re-run is a no-op + return payments.charge(order.total(), order.customer(), order.id()); + }) + .start(); +``` + + + Use the provider's idempotency key (payment APIs and the like), an upsert, or + an "already done?" check at the top of the handler. See the + [failure model](/architecture/failure-model) for the exact re-run windows. + diff --git a/docs/content/docs/java/guides/reliability/index.mdx b/docs/content/docs/java/guides/reliability/index.mdx new file mode 100644 index 00000000..1a724916 --- /dev/null +++ b/docs/content/docs/java/guides/reliability/index.mdx @@ -0,0 +1,18 @@ +--- +title: Reliability +description: "Harden your Java task queue for production workloads." +--- + +Harden your task queue for production workloads. + +| Guide | Description | +|---|---| +| [Error Handling](/java/guides/reliability/error-handling) | Task failure lifecycle, error inspection, the exception hierarchy | +| [Delivery Guarantees](/java/guides/reliability/guarantees) | At-least-once delivery and exactly-once patterns | +| [Retries](/java/guides/reliability/retries) | Automatic retries with exponential backoff | +| [Timeouts](/java/guides/reliability/timeouts) | Bound task runtime per execution attempt | +| [Idempotency](/java/guides/reliability/idempotency) | Deduplicate enqueues with unique keys | +| [Rate Limiting](/java/guides/reliability/rate-limiting) | Throttle and defer enqueues with gates | +| [Concurrency](/java/guides/reliability/concurrency) | Size and autoscale the worker's handler pool | +| [Dead-letter queue](/java/guides/reliability/dead-letter) | Capture and replay exhausted jobs | +| [Distributed Locking](/java/guides/reliability/locks) | Mutual exclusion across workers with database-backed locks | diff --git a/docs/content/docs/java/guides/reliability/locks.mdx b/docs/content/docs/java/guides/reliability/locks.mdx new file mode 100644 index 00000000..3b6697c9 --- /dev/null +++ b/docs/content/docs/java/guides/reliability/locks.mdx @@ -0,0 +1,58 @@ +--- +title: Distributed Locking +description: "TTL-bounded, owner-scoped locks backed by the queue's storage." +--- + +Coordinate across processes without a separate lock server. Locks are +TTL-bounded and owner-scoped, backed by the queue's storage — only the `Lock` +instance that acquired a lock can extend or release it. + + + +## Scoped helper + +`withLock` acquires, runs, and releases — returning whether the body ran: + +```java +boolean ran = queue.withLock("report:2026-06", 60_000, () -> rebuildReport()); +if (!ran) { + log.info("another worker holds the lock"); +} +``` + +## Manual handle + +`Lock` is `AutoCloseable`, so try-with-resources releases it at block exit: + +```java +try (Lock lock = queue.lock("resource", 30_000)) { + if (lock.acquire()) { + // ... critical section + } +} // released at block exit +``` + +| Method | Description | +|---|---| +| `acquire()` | Try once; `false` if another owner holds a live lock. | +| `tryAcquire(Duration timeout)` | Retry every 50ms until obtained or the timeout elapses. | +| `extend(long ttlMs)` | Push the expiry out if still held; `false` means the lock was lost. | +| `release()` / `close()` | Give the lock up (no-op if not held). | +| `queue.lockInfo(name)` | The current holder — owner id, acquired-at, expires-at — or empty. | + +`queue.lock(name)` without a TTL defaults to 30 seconds. + + + Locks do not auto-extend. Choose a TTL comfortably longer than the protected + work's worst case, or call `extend()` at checkpoints — a failed extend means + the lock expired and another owner may now hold it, so stop the critical + section. Expiry is what keeps a crashed owner from holding a lock forever. + diff --git a/docs/content/docs/java/guides/reliability/meta.json b/docs/content/docs/java/guides/reliability/meta.json new file mode 100644 index 00000000..ca953d03 --- /dev/null +++ b/docs/content/docs/java/guides/reliability/meta.json @@ -0,0 +1 @@ +{ "title": "Reliability", "pages": ["error-handling", "guarantees", "retries", "timeouts", "idempotency", "rate-limiting", "concurrency", "dead-letter", "locks"] } diff --git a/docs/content/docs/java/guides/reliability/rate-limiting.mdx b/docs/content/docs/java/guides/reliability/rate-limiting.mdx new file mode 100644 index 00000000..f4d30cc9 --- /dev/null +++ b/docs/content/docs/java/guides/reliability/rate-limiting.mdx @@ -0,0 +1,54 @@ +--- +title: Rate Limiting +description: "Throttle and defer enqueues with gates instead of failing them." +--- + +The Java SDK shapes throughput on the **producer side**: an enqueue gate can +defer a job into the future instead of creating it now, so bursts flatten out +rather than fail. A gate returns one of four decisions — `Allow`, `Skip`, +`Defer(delay)`, or `Reject(reason)` — and the first non-allow decision wins. + +```java +queue.gate("call_api", context -> { + if (bucket.tryConsume()) { + return EnqueueDecision.allow(); + } + return EnqueueDecision.defer(Duration.ofSeconds(1)); // reschedule, don't fail +}); +``` + +A deferred job is enqueued with its `scheduledAt` pushed out by the delay — it +never consumes a retry, and the producer's `enqueue` still returns the job id. +`Skip` creates no job — `tryEnqueue` returns an empty `Optional`, while plain +`enqueue` throws `EnqueueSkippedException`; `Reject` always throws +`PredicateRejectedException`. + +## Built-in recipes + +`Recipes` ships gates for the common time-window cases, each deferring to the +next open slot instead of failing: + +```java +queue.gate("send_report", Recipes.businessHours(ZoneId.of("America/New_York"))); +queue.gate("batch_sync", Recipes.timeWindow(ZoneId.of("UTC"), + LocalTime.of(22, 0), LocalTime.of(6, 0))); // wraps past midnight +queue.gate("weekly_digest", Recipes.dayOfWeek(ZoneId.of("UTC"), + DayOfWeek.MONDAY, DayOfWeek.THURSDAY)); +``` + +Simple boolean checks can use `predicate(...)` instead — it rejects the enqueue +outright when the predicate fails, and composes with `Predicates.allOf`, +`anyOf`, and `not`. + +## Bounding execution + +Gates shape how fast jobs *enter* the queue; to bound how many run at once, +size the worker pool — see [concurrency](/java/guides/reliability/concurrency). +For a fixed drain rate, a fixed `concurrency(n)` worker over a gated queue +gives you both knobs. + + + Gates run in registration order at enqueue time, after + [interceptors](/java/guides/resources/interception) and `onEnqueue` + middleware — so they see the payload that will actually be stored. + diff --git a/docs/content/docs/java/guides/reliability/retries.mdx b/docs/content/docs/java/guides/reliability/retries.mdx new file mode 100644 index 00000000..ca019cee --- /dev/null +++ b/docs/content/docs/java/guides/reliability/retries.mdx @@ -0,0 +1,46 @@ +--- +title: Retries +description: "Automatic retries with exponential backoff before dead-lettering." +--- + +A task whose handler throws is retried up to `maxRetries` times before it +dead-letters. Retries are scheduled by the core with exponential backoff. + +```java +Task CHARGE = Task.of("charge", Order.class) + .maxRetries(5) + .retryPolicy(RetryPolicy.exponential(Duration.ofSeconds(1), Duration.ofMinutes(1))); +``` + +| Option | Description | +|---|---| +| `maxRetries` | Attempts after the first before dead-lettering. Defaults to `0` — never retry — so set it on any task you want retried. | +| `RetryPolicy.exponential(base, max)` | Retry N waits about `base · 2^N`, capped at `max`, plus a random jitter of up to `base`. Without a policy the core defaults apply (1s base, 5min cap). | +| `RetryPolicy.delays(d0, d1, ...)` | Explicit per-attempt delays, applied exactly with no jitter. Supply at least `maxRetries` delays — once the list is exhausted, further retries fire immediately. | + +The retry *budget* travels with the job (`maxRetries`); the *backoff curve* +(`retryPolicy`) is registered with the worker when it starts. Override the +budget per job at enqueue time: + +```java +queue.enqueue(CHARGE, order, EnqueueOptions.builder().maxRetries(1).build()); +``` + +Observe retries as they happen — via middleware `onRetry` or a worker event +listener; once the budget is exhausted the job moves to the +[dead-letter queue](/java/guides/reliability/dead-letter) and the `DEAD` event +fires: + +```java +Worker worker = queue.worker() + .handle(CHARGE, this::charge) + .on(EventName.RETRY, event -> log.info("retrying {0}", event.taskName)) + .on(EventName.DEAD, event -> alertOps(event)) + .start(); +``` + + + Retries re-run the whole handler, so design tasks to be + [idempotent](/java/guides/reliability/idempotency) — a crash after a side + effect but before the result write will re-execute. + diff --git a/docs/content/docs/java/guides/reliability/timeouts.mdx b/docs/content/docs/java/guides/reliability/timeouts.mdx new file mode 100644 index 00000000..525976c0 --- /dev/null +++ b/docs/content/docs/java/guides/reliability/timeouts.mdx @@ -0,0 +1,32 @@ +--- +title: Timeouts +description: "Bound each execution attempt; timed-out jobs fail and may retry." +--- + +`timeoutMs` bounds a single execution attempt. The dispatcher races the handler +against the timeout; if it elapses, the attempt is marked failed and timed-out. +Unset (or `0`) means no limit. + +```java +Task SCRAPE = Task.of("scrape", String.class) + .timeout(Duration.ofSeconds(30)); + +// Per-job override +queue.enqueue(SCRAPE, url, EnqueueOptions.builder().timeoutMs(10_000).build()); +``` + +A timeout counts as a failure: it consumes a retry and, if retries remain, the +job is re-enqueued. When the budget is exhausted the job dead-letters with its +timed-out flag set — surfaced as `OutcomeEvent.timedOut` to event listeners and +middleware outcome hooks, so they can tell timeouts apart from other errors. + + + A timeout releases the job's bookkeeping, not its thread — the handler is not + interrupted and keeps running until it returns (its late result is discarded). + Make long-running handlers cooperative: poll + `queue.isCancelRequested(jobId)` or bound their own blocking calls, so the + underlying work actually stops. + + +Always set a timeout on production tasks — without one, a wedged handler can +hold a worker thread indefinitely. diff --git a/docs/content/docs/java/guides/resources/dependency-injection.mdx b/docs/content/docs/java/guides/resources/dependency-injection.mdx new file mode 100644 index 00000000..4b332575 --- /dev/null +++ b/docs/content/docs/java/guides/resources/dependency-injection.mdx @@ -0,0 +1,198 @@ +--- +title: Dependency Injection +description: "Register external dependencies once and inject them into tasks by scope." +--- + +Tasks often need a database pool, an HTTP client, or a cloud SDK. Rather than +constructing those inside every handler, register them once as **resources** and +let the worker build, share, and tear them down with the right lifetime. + +```java +try (Taskito queue = Taskito.builder().sqlite("tasks.db").open()) { + queue.resource("db", ResourceScope.WORKER, + ctx -> openPool(System.getenv("DATABASE_URL")), + pool -> pool.close()); + + try (Worker worker = queue.worker() + .handle(SYNC_USER, userId -> { + DataSource db = Resources.use("db"); + return markSynced(db, userId); + }) + .start()) { + queue.enqueue(SYNC_USER, "u_123"); + } +} +``` + +The pool is created once when the worker first needs it, shared across every +job, and closed when the worker stops. No connection is ever serialized into the +queue. + +## Scopes + +A resource's `ResourceScope` decides its lifetime: + +| Scope | Built | Lifetime | Use for | +|---|---|---|---| +| `WORKER` (default) | Lazily, on first use | The worker — a shared singleton | Connection pools, HTTP clients, SDK clients | +| `THREAD` | Lazily, once per worker thread | Until worker shutdown | Non-thread-safe clients that are cheap to keep per thread | +| `TASK` | Once per task invocation | That invocation — disposed when it ends | Per-job transactions, request-scoped clients | +| `REQUEST` | Fresh on every `use()` | Disposed with the task; never cached | Values that must not be shared even within one job | +| `POOLED` | Lazily, up to `poolSize` at once | Checked out per task, returned at task end | Expensive clients that must stay bounded but be reused | + +```java +queue.resource("tx", ResourceScope.TASK, + ctx -> beginTransaction(), + tx -> tx.rollbackIfOpen()); // no-op if already committed +``` + +Worker-scoped resources are built at most once per worker even under +concurrency — a concurrent first use does not double-build. A factory that +throws is **not** cached, so the next job retries it. Registering the same name +twice throws `ResourceException`, as does a dependency cycle between factories. + +## Pooled resources + +A `POOLED` resource sits between `WORKER` and `TASK`: instances are reused like +worker resources but bound to a single task at a time, with a hard cap on how +many exist at once. Each task checks out one instance — reusing an idle one, +building a new one only when none is free — and returns it to the pool when the +task finishes. A task that cannot get an instance within `acquireTimeout` fails +with `ResourceException`: + +```java +queue.resource("ftp", + new PoolConfig(4, 1, Duration.ofSeconds(10), Duration.ofMinutes(5)), + ctx -> connectFtp(), + conn -> conn.close()); +``` + +| `PoolConfig` field | Default | Meaning | +|---|---|---| +| `poolSize` | required | Max instances checked out concurrently. Tasks wait when exhausted. | +| `poolMin` | `0` | Instances prewarmed when the worker starts. `0` means fully lazy. | +| `acquireTimeout` | 10s | How long a checkout waits before failing the task. | +| `maxLifetime` | unlimited | An idle instance older than this is disposed and rebuilt instead of reused. | + +`PoolConfig.of(poolSize)` gives the defaults; derive variations with +`withPoolMin`, `withAcquireTimeout`, and `withMaxLifetime`. The disposer runs +when the pool retires an instance (worker shutdown or `maxLifetime` expiry) — +not when a task returns it. Pool capacity is per worker, never shared across +workers. + +## Injecting resources + +Two equivalent ways to reach a resource from a handler. + +### Accessor — `Resources.use` + +Call `Resources.use(name)` anywhere inside a running handler. It resolves +against the current task's scope: + +```java +queue.worker().handle(REPORT, request -> { + DataSource db = Resources.use("db"); + Cache cache = Resources.use("cache"); + // ... + return buildReport(db, cache, request); +}); +``` + +Calling `Resources.use` outside a task throws `ResourceException` — it is only +available while a handler runs. + +### Declarative — `@Resource` parameters + +With the annotation processor, list resources as extra parameters on a +`@TaskHandler` method. The generated companion resolves each by name and passes +it positionally — no runtime reflection: + +```java +public class EmailHandlers { + @TaskHandler("send_email") + void send(EmailPayload payload, @Resource("db") Database db) { + // db is injected, not part of the payload + } +} + +// The processor generates EmailHandlersTasks with a typed Task constant +// per handler and a bind method: +Worker worker = queue.worker() + .apply(builder -> EmailHandlersTasks.bind(builder, new EmailHandlers())) + .start(); + +// Producers still enqueue only the real payload: +queue.enqueue(EmailHandlersTasks.SEND_EMAIL, payload); +``` + +`@Resource` parameters must come *after* the payload parameter; each name is +resolved from the worker's resource runtime, exactly like `Resources.use`. + +## Resources that depend on resources + +A factory receives a `ResourceContext` whose `use` resolves another resource, +so you can compose them: + +```java +queue.resource("config", ctx -> loadConfig()); +queue.resource("db", ctx -> { + Config config = ctx.use("config"); + return openPool(config.databaseUrl()); +}); +``` + +A factory may only depend on same-or-longer-lived resources: `WORKER` and +`POOLED` factories may `use` only worker resources (a pooled instance outlives +the task that built it), a `THREAD` factory may use worker or thread resources, +and `TASK`/`REQUEST` factories may use any scope. Reaching for anything +shorter-lived throws `ResourceException`. + +## Teardown + +Pass a disposer to release a resource when its scope ends — worker and thread +resources when the worker stops, task and request resources when the task +finishes. Disposal runs in **reverse order** of construction (LIFO), so a +resource is always torn down before anything it depended on: + +```java +queue.resource("db", ResourceScope.WORKER, ctx -> openPool(), pool -> pool.close()); +``` + +Disposal errors are logged, never thrown — they cannot fail an already-settled +job. + +## Metrics + +`queue.resourceMetrics()` returns per-resource lifecycle counters — how many +instances were built, disposed, and are currently live: + +```java +Map metrics = queue.resourceMetrics(); +metrics.get("db"); // ResourceStat[created=1, disposed=0, active=1] +metrics.get("tx"); // ResourceStat[created=12, disposed=12, active=0] +``` + +A worker-scoped resource shows `active = 1` while the worker runs and `0` after +it closes; a task-scoped resource's `created`/`disposed` climb per job with +`active` near zero. + +## Testing + +Register a stub factory to swap a real dependency in tests — handlers resolve +by name, so they never know the difference. The `test-support` module's +`InMemoryTaskito` gives you a queue with no native backend at all: + +```java +try (Taskito queue = InMemoryTaskito.open()) { + AtomicInteger built = new AtomicInteger(); + queue.resource("db", ctx -> { + built.incrementAndGet(); + return new FakeDatabase(); + }); + // ... run the task through a worker ... + assertEquals(1, built.get()); + assertEquals(1, queue.resourceMetrics().get("db").created()); +} +``` + +See [testing](/java/guides/operations/testing) for the full worker test setup. diff --git a/docs/content/docs/java/guides/resources/index.mdx b/docs/content/docs/java/guides/resources/index.mdx new file mode 100644 index 00000000..17b6d89b --- /dev/null +++ b/docs/content/docs/java/guides/resources/index.mdx @@ -0,0 +1,66 @@ +--- +title: Resource System +description: "Inject external dependencies into tasks, intercept enqueues, and proxy non-serializable values." +--- + +Task payloads cross a process boundary, so they must be serializable — but most +real dependencies (database pools, HTTP clients, cloud SDKs) are not. The Java +SDK keeps those on the worker and passes only plain data through the queue. + +```java +try (Taskito queue = Taskito.builder().sqlite("tasks.db").open()) { + queue.resource("db", ResourceScope.WORKER, ctx -> openPool(), pool -> pool.close()); + + try (Worker worker = queue.worker() + .handle(SYNC_USER, userId -> { + DataSource db = Resources.use("db"); + return markSynced(db, userId); + }) + .start()) { + queue.enqueue(SYNC_USER, "u_123"); + } +} +``` + +| Page | What it covers | +|---|---| +| [Dependency injection](/java/guides/resources/dependency-injection) | `Taskito.resource()`, the five scopes, `Resources.use`, annotation-driven injection, teardown, metrics | +| [Enqueue interception](/java/guides/resources/interception) | `Interceptor` strategies and the `onEnqueue` hook — validate, rewrite, redirect, and reject jobs before serialization | +| [Resource proxies](/java/guides/resources/proxies) | Signed `ProxyRef`s for non-serializable values — HMAC signing, TTL, purpose binding, sessions | + +## The five scopes + +A resource's `ResourceScope` decides its lifetime on the worker: + +| Scope | Built | Lifetime | +|---|---|---| +| `WORKER` (default) | Lazily, on first use | The worker — one shared instance per worker | +| `THREAD` | Lazily, once per worker thread | Shared by every task on that thread; disposed at worker shutdown | +| `TASK` | Once per task invocation | That invocation — disposed (LIFO) when it ends | +| `REQUEST` | Fresh on every `use()` | Never cached — N uses in one task yield N instances, all disposed with the task | +| `POOLED` | Lazily, up to `poolSize` at once | Checked out per task, returned at task end | + +`POOLED` sits between `WORKER` and `TASK`: instances are reused across tasks +but bound to one task at a time, with a hard cap on how many exist. A +`PoolConfig` sizes the pool — `poolMin` instances are prewarmed at worker +start, a checkout that cannot get an instance within `acquireTimeout` fails the +job with `ResourceException`, and an idle instance older than `maxLifetime` is +disposed and rebuilt instead of reused. Because pooled instances outlive any +single task, a pooled factory may only depend on worker-scoped resources. + +## Proxy sessions + +For values that must travel *through* a payload — file handles and friends — +[proxies](/java/guides/resources/proxies) deconstruct them into signed, +serializable refs. A `ProxySession` wraps that per unit of work: deconstructing +the same instance twice returns the same ref (identity dedup), every ref to the +same resource reconstructs once (memoized by signature, with the signature, +expiry, and purpose re-verified on every resolve), and `close()` runs each +handler's cleanup once per instance in reverse reconstruction order (LIFO). +Sessions are `AutoCloseable`, so try-with-resources scopes the whole lifecycle. + + + Prefer plain payloads plus worker-side injection wherever you can; reach for + proxies only when a non-serializable value genuinely has to be chosen by the + producer. + diff --git a/docs/content/docs/java/guides/resources/interception.mdx b/docs/content/docs/java/guides/resources/interception.mdx new file mode 100644 index 00000000..59661e75 --- /dev/null +++ b/docs/content/docs/java/guides/resources/interception.mdx @@ -0,0 +1,91 @@ +--- +title: Enqueue Interception +description: "Validate, rewrite, redirect, or reject jobs before they are serialized." +--- + +Interceptors run on the **producer side**, inside every `enqueue`, before the +payload is serialized. Use them to validate inputs, redact secrets, or reroute +jobs across every task — in one place instead of at every call site. + +An `Interceptor` sees the task name and payload and returns one of four +`Interception` strategies: + +| Strategy | Effect | +|---|---| +| `Interception.pass()` | Enqueue the payload unchanged. | +| `Interception.convert(payload)` | Replace the payload (e.g. with a [proxy ref](/java/guides/resources/proxies)). | +| `Interception.redirect(taskName, payload)` | Enqueue a different task (and payload) instead. | +| `Interception.reject(reason)` | Block the enqueue — `InterceptionException` is thrown and no job is created. | + +```java +queue.intercept((taskName, payload) -> { + if (payload instanceof String s && s.startsWith("pw:")) { + return Interception.convert("***"); // redact before it reaches storage + } + return Interception.pass(); +}); +``` + +Interceptors run synchronously in registration order; each sees the previous +one's result. A `Redirect` retargets the rest of the pipeline — later +interceptors, middleware, and gates all see the new task name and payload. + +## Validation + +Rejecting from an interceptor aborts the enqueue — the job is never created and +the error propagates to the caller: + +```java +queue.intercept((taskName, payload) -> { + if (taskName.equals("charge") && ((Order) payload).total() < 0) { + return Interception.reject("charge amount must be non-negative"); + } + return Interception.pass(); +}); + +queue.enqueue(CHARGE, invalidOrder); // throws InterceptionException — nothing enqueued +``` + +## Rewriting options — the `onEnqueue` hook + +Interceptors decide the payload's fate; to rewrite enqueue *options* or attach +metadata, use [middleware](/java/guides/extensibility/middleware) `onEnqueue`. +It receives a mutable `EnqueueContext` after interceptors have run: + +```java +queue.use(new Middleware() { + @Override + public void onEnqueue(EnqueueContext context) { + context.metadata().put("enqueuedBy", currentUser()); // travels with the job + context.options(context.options().toBuilder().priority(5).build()); + } +}); +``` + +Mutations take effect: the context's payload is what gets serialized, its +options are what reach the core, and a non-empty metadata map becomes the job's +metadata blob. Throwing from `onEnqueue` aborts the enqueue. + +## Pipeline order + +Every enqueue runs **interceptors → `onEnqueue` middleware → enqueue gates**, +then serializes, codec-encodes, and submits. Gates see the payload that will +actually be stored, after any rewrites. + +## Edge cases + +- **Batches** — `enqueueMany` runs each payload through the interceptors, so a + batch can't bypass the contract. `Convert` rewrites the item; `Reject` fails + the whole batch; `Redirect` is unsupported in a batch (it would move an item + out of the single-task batch) and throws. +- **Redirect + payload codecs** — redirecting a task that has + [payload codecs](/java/guides/core/serialization) throws: the source task's + codec chain can't be applied to the target task without corrupting the + payload. +- **Null result** — an interceptor returning `null` is a bug and throws + `InterceptionException`. + + + Keep interceptors fast and side-effect-free — they run on the producer thread + inside every `enqueue` call. + diff --git a/docs/content/docs/java/guides/resources/meta.json b/docs/content/docs/java/guides/resources/meta.json new file mode 100644 index 00000000..4b4d18b2 --- /dev/null +++ b/docs/content/docs/java/guides/resources/meta.json @@ -0,0 +1 @@ +{ "title": "Resources", "pages": ["dependency-injection", "interception", "proxies"] } diff --git a/docs/content/docs/java/guides/resources/proxies.mdx b/docs/content/docs/java/guides/resources/proxies.mdx new file mode 100644 index 00000000..eeef610c --- /dev/null +++ b/docs/content/docs/java/guides/resources/proxies.mdx @@ -0,0 +1,154 @@ +--- +title: Resource Proxies +description: "Signed, serializable refs for non-serializable values — HMAC signing, TTL, purpose binding, sessions." +--- + +Some values are neither serializable primitives nor DI-injectable — a file +handle chosen by the producer, for instance. Proxies pass them through a +payload **by reference**: deconstruct the value into a signed, serializable +`ProxyRef` on the producer, carry the ref in the payload, and reconstruct the +live value in the handler. + +This is explicit — you call `deconstruct` and `reconstruct` yourself, and a +`ProxyHandler` per resource type does the (de)construction: + +```java +byte[] key = System.getenv("TASKITO_PROXY_KEY").getBytes(StandardCharsets.UTF_8); +Proxies proxies = new Proxies(key).register(new FileProxyHandler()); + +// Producer: reduce the file to a signed ref and enqueue it +ProxyRef ref = proxies.deconstruct(new File("/data/uploads/report.csv")); +queue.enqueue(PROCESS_FILE, ref); + +// Worker: verify and rebuild the file handle +Worker worker = queue.worker() + .handle(PROCESS_FILE, ref2 -> { + File file = proxies.resolve(ref2); + return process(file); + }) + .start(); +``` + +Producer and worker must construct `Proxies` with the **same HMAC key** and +register the same handler ids. + +## Signed refs + +A `ProxyRef` is a record of the handler id, the handler's serializable +reference data, and an HMAC-SHA256 signature over the handler id, the +canonicalized reference, the expiry, and the purpose. Verification happens on +every reconstruct — a modified or forged ref throws `ProxyException`: + +```java +ProxyRef tampered = new ProxyRef(ref.handler(), Map.of("path", "/etc/passwd"), ref.signature()); +proxies.reconstruct(tampered); // throws ProxyException: signature mismatch +``` + +The ref's shape and signature scheme follow the cross-SDK contract, so a ref +produced by one SDK verifies in another sharing the key. + +### TTL + +Bind a ref to a wall-clock expiry; reconstructing after it lapses throws. The +expiry is folded into the signature, so it cannot be extended by tampering: + +```java +ProxyRef ref = proxies.deconstruct(file, Duration.ofHours(1)); +``` + +### Purpose binding + +Bind a ref to a label and require it at reconstruction — a ref minted for one +flow cannot be replayed into another: + +```java +ProxyRef ref = proxies.deconstruct(file, null, "emails"); + +proxies.resolve(ref, "emails"); // ok +proxies.resolve(ref, "billing"); // throws ProxyException: purpose mismatch +proxies.resolve(ref); // ok — purpose not checked when not requested +``` + +## The file handler + +`FileProxyHandler` proxies a `java.io.File` by its absolute path. Give it an +allowlist of root directories and reconstruction refuses any path outside them +— including paths that only *look* inside via a symlinked ancestor, since roots +and candidates are resolved to their real filesystem locations first: + +```java +Proxies proxies = new Proxies(key) + .register(new FileProxyHandler(List.of(Path.of("/data/uploads")))); +``` + +An empty allowlist permits any path — always set one in production. + +## Custom handlers + +Implement `ProxyHandler` for your own types: a stable `id()`, a `handles` +test, `deconstruct` to a serializable map, and `reconstruct` back. Optionally +override `cleanup` to release what `reconstruct` opened — it runs when a +session that produced the value closes: + +```java +final class SftpHandler implements ProxyHandler { + public String id() { return "sftp"; } + + public boolean handles(Object value) { return value instanceof SftpClient; } + + public Map deconstruct(SftpClient client) { + return Map.of("host", client.host(), "port", client.port()); + // never put credentials in a reference — the worker uses its own + } + + public SftpClient reconstruct(Map reference) { + return SftpClient.connect((String) reference.get("host"), (Integer) reference.get("port")); + } + + @Override + public void cleanup(SftpClient client) { client.close(); } +} +``` + +Handler ids must be unique — registering a duplicate throws, since a producer +and worker disagreeing on an id would reconstruct the wrong thing. + +## Sessions + +A `ProxySession` wraps a registry for one unit of work — one producer batch or +one task invocation — adding identity dedup and a cleanup lifecycle: + +```java +try (ProxySession session = proxies.session()) { + ProxyRef first = session.deconstruct(file); + ProxyRef again = session.deconstruct(file); // same instance → same ref, handler runs once + + SftpClient client = session.resolve(sftpRef); + SftpClient same = session.resolve(sftpRef); // memo hit — one live client +} // cleanup runs here, LIFO, once per unique instance +``` + +Within one session: + +- **Deconstruct** memoizes by (instance identity, purpose) — the same object + deconstructs once per purpose, and the first call's TTL wins. Use a new + session to refresh expiry. +- **Reconstruct** memoizes by the ref's signature — every ref to the same + resource resolves to the same instance. Signature, expiry, and purpose are + **re-verified on every call**, memo hit or not, so a ref that expires + mid-session stops resolving. +- **`close()`** runs each handler's `cleanup` once per unique reconstructed + instance, in reverse reconstruction order (LIFO). A cleanup failure is logged + and never skips the rest. Closing is idempotent; sessions are + `AutoCloseable`, so try-with-resources scopes the lifecycle. + +Sessions are not thread-safe — confine one to the thread that created it. +Direct `Proxies.reconstruct` calls have no lifecycle and never trigger +`cleanup`. + + + A reference should carry *identity*, never *credentials* — the worker + authenticates with its own ambient configuration. The signature stops + tampering, but anything you put in the reference map is stored in plain form + with the job payload. + From 3bf80a6abcfab6995cd287ae683f5f7e9968cb4a Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Tue, 7 Jul 2026 02:51:50 +0530 Subject: [PATCH 04/16] docs(java): api reference --- docs/content/docs/java/api-reference/cli.mdx | 37 +++++++ .../docs/java/api-reference/context.mdx | 59 ++++++++++ .../docs/java/api-reference/errors.mdx | 40 +++++++ .../content/docs/java/api-reference/index.mdx | 22 ++++ .../content/docs/java/api-reference/meta.json | 1 + .../content/docs/java/api-reference/queue.mdx | 102 ++++++++++++++++++ .../docs/java/api-reference/resources.mdx | 65 +++++++++++ .../docs/java/api-reference/result.mdx | 52 +++++++++ .../docs/java/api-reference/serializers.mdx | 73 +++++++++++++ docs/content/docs/java/api-reference/task.mdx | 95 ++++++++++++++++ .../docs/java/api-reference/worker.mdx | 70 ++++++++++++ .../docs/java/api-reference/workflows.mdx | 100 +++++++++++++++++ 12 files changed, 716 insertions(+) create mode 100644 docs/content/docs/java/api-reference/cli.mdx create mode 100644 docs/content/docs/java/api-reference/context.mdx create mode 100644 docs/content/docs/java/api-reference/errors.mdx create mode 100644 docs/content/docs/java/api-reference/index.mdx create mode 100644 docs/content/docs/java/api-reference/meta.json create mode 100644 docs/content/docs/java/api-reference/queue.mdx create mode 100644 docs/content/docs/java/api-reference/resources.mdx create mode 100644 docs/content/docs/java/api-reference/result.mdx create mode 100644 docs/content/docs/java/api-reference/serializers.mdx create mode 100644 docs/content/docs/java/api-reference/task.mdx create mode 100644 docs/content/docs/java/api-reference/worker.mdx create mode 100644 docs/content/docs/java/api-reference/workflows.mdx diff --git a/docs/content/docs/java/api-reference/cli.mdx b/docs/content/docs/java/api-reference/cli.mdx new file mode 100644 index 00000000..874cd4f7 --- /dev/null +++ b/docs/content/docs/java/api-reference/cli.mdx @@ -0,0 +1,37 @@ +--- +title: CLI +description: "The taskito command." +--- + +```bash +taskito [--backend ] [--url ] [args] +``` + +`org.byteveda.taskito.cli.Cli` is a picocli command (`Cli.main`) over a +`Taskito` client. `--backend` defaults to `sqlite`; `--url` is the connection +string (a SQLite path, defaulting to `.taskito/taskito.db`, or a +Postgres/Redis URL — required for those backends). Read commands print JSON. + +## Commands + +| Command | Description | +|---|---| +| `stats` | Job counts by status. | +| `enqueue ['']` | Enqueue a job (payload defaults to null) → job id. | +| `jobs [--status ] [--queue ] [--limit ]` | List jobs. | +| `cancel ` | Cancel a pending job (exit 1 if it already started). | +| `pause ` / `resume ` | Queue control. | +| `dlq list [--limit ]` / `dlq retry ` / `dlq delete ` | Dead-letter ops. | +| `dashboard [--port ] [--token ] [--static ]` | Serve the dashboard until interrupted. | + +```bash +taskito --url taskito.db enqueue add '{"a":2,"b":3}' +taskito --url taskito.db jobs --status failed --limit 20 +taskito --backend postgres --url postgres://localhost/taskito stats +``` + +Run it from your application's classpath, e.g.: + +```bash +java -cp app.jar org.byteveda.taskito.cli.Cli --url taskito.db stats +``` diff --git a/docs/content/docs/java/api-reference/context.mdx b/docs/content/docs/java/api-reference/context.mdx new file mode 100644 index 00000000..5403bcaa --- /dev/null +++ b/docs/content/docs/java/api-reference/context.mdx @@ -0,0 +1,59 @@ +--- +title: Context +description: "Resources.use inside handlers and the middleware TaskContext." +--- + +A handler receives exactly one argument — its deserialized payload. The +execution context around it is reached two ways: resource resolution inside +the handler, and the `TaskContext` handed to middleware. + +## `Resources.use` + +```java +import org.byteveda.taskito.resources.Resources; + +taskito.worker().handle(sendEmail, payload -> { + SmtpClient smtp = Resources.use("smtp"); + return smtp.deliver(payload); +}); +``` + +Resolves a [registered resource](/java/api-reference/resources) by name against +the current task's scope. Valid only while a task runs on this worker — the +worker binds the scope around the handler call; outside one it throws a +`ResourceException`. Prefer the declarative `@Resource("smtp")` parameter on a +`@TaskHandler` method, which the generated companion resolves the same way. + +## `TaskContext` (middleware) + +The job's identity travels through [middleware](/java/api-reference/queue) +hooks rather than the handler signature: + +```java +taskito.use(new Middleware() { + @Override + public void before(TaskContext context) { + MDC.put("jobId", context.jobId); + } + + @Override + public void onError(TaskContext context, Throwable error) { + log.warn("{} {} failed", context.taskName, context.jobId, error); + } +}); +``` + +| Member | Description | +|---|---| +| `jobId` / `taskName` | The executing job's identity. | +| `attributes()` | Mutable per-execution scratch map shared across `before`/`after`/`onError` for the same job (e.g. a timer or a span). | +| `job()` | The executing job, including its lazily loaded metadata. | + +## Progress & cancellation + +Progress and cooperative cancellation go through the client by job id: +`taskito.setProgress(jobId, percent)` records 0–100 for inspection and the +dashboard, and `taskito.isCancelRequested(jobId)` lets a long-running handler +poll for a `requestCancel` from another process. Thread the job id into the +payload (or capture it in middleware `attributes()`) when a handler needs +either. diff --git a/docs/content/docs/java/api-reference/errors.mdx b/docs/content/docs/java/api-reference/errors.mdx new file mode 100644 index 00000000..e4ce7b90 --- /dev/null +++ b/docs/content/docs/java/api-reference/errors.mdx @@ -0,0 +1,40 @@ +--- +title: Errors +description: "The TaskitoException hierarchy — catch by specific type or by the base." +--- + +Every error the SDK throws extends `TaskitoException` (an unchecked +`RuntimeException`), so a single `catch` can scope all of them; the subclasses +under `org.byteveda.taskito.errors` let you branch on what went wrong. + +```java +try { + taskito.enqueue(sendPromo, audience); +} catch (PredicateRejectedException rejected) { + respond(409, "blocked by policy"); +} catch (TaskitoException other) { + report(other); +} +``` + +## Hierarchy + +| Class | Extends | Thrown when | +|---|---|---| +| `TaskitoException` | `RuntimeException` | Base; also carries native (JNI) errors directly. | +| `ConfigurationException` | `TaskitoException` | The SDK is misconfigured — e.g. a missing connection URL. | +| `SerializationException` | `TaskitoException` | A payload, options blob, or native response can't be (de)serialized. | +| `CryptoException` | `SerializationException` | A signing/encrypting serializer or codec failed — bad signature, short payload, cipher error. | +| `PredicateRejectedException` | `TaskitoException` | A registered predicate rejected an enqueue; no job was created. | +| `EnqueueSkippedException` | `TaskitoException` | A gate returned `Skip` — `enqueue` throws; use `tryEnqueue` to get an empty `Optional` instead. | +| `InterceptionException` | `TaskitoException` | An interceptor rejected an enqueue. | +| `LockException` | `TaskitoException` | A distributed-lock operation failed or was interrupted. | +| `ResourceException` | `TaskitoException` | A worker resource couldn't be built, resolved, or disposed — including `Resources.use` outside a task. | +| `ProxyException` | `TaskitoException` | A proxy reference couldn't be created or reconstructed — no handler, signature mismatch, allowlist violation. | +| `WebhookException` | `TaskitoException` | A webhook couldn't be stored, loaded, signed, or encoded. | +| `WorkflowException` | `TaskitoException` | Workflow definition, submission, await-timeout, or query error. | + +Handler exceptions are different: an exception thrown *inside* a +`TaskFunction` isn't rethrown to you — it fails that attempt, and the core +retries with the task's backoff until the retry budget is spent, then +dead-letters the job. Inspect those via `jobErrors(id)` and `listDead`. diff --git a/docs/content/docs/java/api-reference/index.mdx b/docs/content/docs/java/api-reference/index.mdx new file mode 100644 index 00000000..3435f7e5 --- /dev/null +++ b/docs/content/docs/java/api-reference/index.mdx @@ -0,0 +1,22 @@ +--- +title: API Reference +description: "The Java SDK public surface — the Taskito client, tasks, workers, workflows, and the CLI." +--- + +The public API lives under `org.byteveda.taskito` (Maven coordinates +`org.byteveda:taskito`). Contrib helpers live under +`org.byteveda.taskito.contrib`; the Spring Boot 3 starter ships separately as +`org.byteveda:taskito-spring`. + +| Page | Covers | +|---|---| +| [Taskito](/java/api-reference/queue) | `Taskito.builder()`, enqueue, inspect, admin, locks, periodic. | +| [Task](/java/api-reference/task) | `Task.of` typed tasks, `EnqueueOptions`, `RetryPolicy`, the annotations. | +| [Worker](/java/api-reference/worker) | The `Worker.Builder` and the running worker handle. | +| [Result](/java/api-reference/result) | Awaiting jobs and reading typed results. | +| [Context](/java/api-reference/context) | `Resources.use` and the middleware `TaskContext`. | +| [Resources](/java/api-reference/resources) | Worker dependency injection — scopes, pools, disposal. | +| [Serializers](/java/api-reference/serializers) | `JsonSerializer`, `MsgpackSerializer`, payload codecs. | +| [Workflows](/java/api-reference/workflows) | The `Workflow` builder, `WorkflowRun`, gates, sagas, analysis. | +| [Errors](/java/api-reference/errors) | The `TaskitoException` hierarchy. | +| [CLI](/java/api-reference/cli) | The `taskito` command. | diff --git a/docs/content/docs/java/api-reference/meta.json b/docs/content/docs/java/api-reference/meta.json new file mode 100644 index 00000000..3d70c26b --- /dev/null +++ b/docs/content/docs/java/api-reference/meta.json @@ -0,0 +1 @@ +{ "title": "API Reference", "root": true, "pages": ["index", "queue", "task", "worker", "result", "context", "resources", "serializers", "workflows", "errors", "cli"] } diff --git a/docs/content/docs/java/api-reference/queue.mdx b/docs/content/docs/java/api-reference/queue.mdx new file mode 100644 index 00000000..e81848eb --- /dev/null +++ b/docs/content/docs/java/api-reference/queue.mdx @@ -0,0 +1,102 @@ +--- +title: Taskito +description: "Construct the client and the full producer/admin API." +--- + +```java +import org.byteveda.taskito.Taskito; + +try (Taskito taskito = Taskito.builder().sqlite("taskito.db").open()) { + // ... +} +``` + +`Taskito` is `AutoCloseable`; `close()` releases the native storage handle. + +## Builder + +| Method | Description | +|---|---| +| `backend(String)` | `"sqlite"` (default) \| `"postgres"` \| `"redis"`. | +| `url(String)` | Connection string: a file path for SQLite, a URL for Postgres/Redis. | +| `sqlite()` / `sqlite(path)` / `postgres(url)` / `redis(url)` | Backend shortcuts. SQLite defaults to `.taskito/taskito.db`. | +| `poolSize(int)` | Connection pool size (SQLite/Postgres). | +| `schema(String)` | Postgres schema (default `taskito`). | +| `prefix(String)` | Redis key prefix. | +| `namespace(String)` | Namespace applied to enqueued jobs + the worker scheduler. | +| `serializer(Serializer)` | Payload/result serializer (default `JsonSerializer`). | +| `codec(PayloadCodec...)` | Global [codec chain](/java/api-reference/serializers) around the serializer. | +| `codec(String, PayloadCodec)` | Register a named codec for per-task selection (`Task.codecs`). | +| `open()` | Open the native backend → `Taskito`. | +| `open(QueueBackend)` | Open over an explicit backend (e.g. a test fake). | + +## Producer + +| Method | Description | +|---|---| +| `enqueue(Task, T)` / `enqueue(Task, T, EnqueueOptions)` | Enqueue a typed payload → job id. | +| `enqueue(String taskName, Object payload)` | Enqueue by task name. | +| `tryEnqueue(...)` | Gate-aware `enqueue`: `Optional.empty()` when a gate skips; a `Reject` still throws. | +| `enqueueMany(Task, List)` / `(..., EnqueueOptions)` | Batch-enqueue in one storage call → ids in input order (no dedup). | +| `enqueueAll(Task, List)` | Alias of `enqueueMany`. | + +## Results & cancellation + +| Method | Description | +|---|---| +| `getJob(id)` → `Optional` | A job snapshot (status, progress, timestamps). | +| `awaitJob(id, Duration)` → `Optional` | Block until terminal; throws on timeout. | +| `getResult(id)` → `Optional` / `getResult(id, Class)` → `Optional` | The job's result, raw or [deserialized](/java/api-reference/result). | +| `cancel(id)` | Cancel a pending job. | +| `requestCancel(id)` / `isCancelRequested(id)` | Cooperative cancellation of a running job. | +| `setProgress(id, int)` | Record 0–100 progress for inspection / the dashboard. | + +## Inspection + +| Method | Description | +|---|---| +| `stats()` / `statsByQueue(q)` / `statsAllQueues()` | `QueueStats` counts by status. | +| `listJobs(JobFilter)` | List jobs — filter by status/queue/task with limit/offset. | +| `jobErrors(id)` → `List` | Per-attempt error history. | +| `metrics(taskName, sinceMs)` → `List` | Per-execution metrics (null task = all). | +| `listWorkers()` → `List` | Registered workers (heartbeat + identity). | +| `writeTaskLog(...)` / `getTaskLogs(id)` | Structured task-log lines for a job. | + +## Admin + +| Method | Description | +|---|---| +| `listDead(limit, offset)` / `listDeadByTask(task, limit, offset)` | Dead-letter entries. | +| `retryDead(deadId)` (alias `retry`) / `deleteDead(deadId)` | Re-enqueue / drop an entry. | +| `purgeDead(olderThanMs)` / `purgeDeadByTask(task)` | Bulk DLQ cleanup → count removed. | +| `purgeCompleted(olderThanMs)` | Drop old completed jobs. | +| `queue(name)` → `Queue` | A named-queue handle: `name()`, `pause()`, `resume()`, `isPaused()`. | +| `listPausedQueues()` | Names of every paused queue. | +| `getSetting` / `setSetting` / `deleteSetting` / `listSettings` | Shared key–value settings in the store. | + +## Locks & periodic + +| Method | Description | +|---|---| +| `lock(name, ttlMs)` / `lock(name)` | A distributed `Lock` (default 30s TTL): `acquire()`, `tryAcquire(timeout)`, `extend(ttlMs)`, `close()`. | +| `withLock(name, ttlMs, Runnable)` | Acquire → run → release; returns whether it ran. | +| `lockInfo(name)` (alias `getLockInfo`) | Current holder metadata, if held. | +| `registerPeriodic(PeriodicTask)` | Register (or replace) a cron task → next fire time (Unix ms). | +| `listPeriodic()` / `deletePeriodic(name)` / `pausePeriodic(name)` / `resumePeriodic(name)` | Periodic task management. | + +## Subsystems + +| Method | Description | +|---|---| +| `worker()` → `Worker.Builder` | Begin building a [worker](/java/api-reference/worker). | +| `use(Middleware)` | Cross-cutting hooks — `onEnqueue`, `before`/`after`/`onError`, and the outcome hooks. | +| `resource(...)` | Register an injectable [resource](/java/api-reference/resources). | +| `predicate(taskName, Predicate)` / `gate(taskName, EnqueueGate)` | Enqueue-time gates: reject, skip, or defer submissions. | +| `intercept(Interceptor)` | Pass / convert / redirect / reject each enqueue before serialization. | +| `submitWorkflow(Workflow)` / `submitWorkflow(Workflow, Map)` | Submit a [workflow](/java/api-reference/workflows) → `WorkflowRun`. | +| `workflowStatus(runId)` / `cancelWorkflow(runId)` | Run queries and cancellation. | +| `resourceMetrics()` | Per-resource created/disposed/active counters. | + +`WebhookManager.attach(taskito)` registers outcome webhooks, and +`Scaler.start(taskito, ScalerOptions.onPort(9090))` serves a queue-depth +endpoint for external autoscalers — both build on this client. diff --git a/docs/content/docs/java/api-reference/resources.mdx b/docs/content/docs/java/api-reference/resources.mdx new file mode 100644 index 00000000..4f67c8eb --- /dev/null +++ b/docs/content/docs/java/api-reference/resources.mdx @@ -0,0 +1,65 @@ +--- +title: Resources +description: "Worker dependency injection — scopes, pools, and disposal." +--- + +Resources are worker-side dependencies (clients, connections, caches) built by +a factory and handed to handlers by name — via `Resources.use("name")` or an +`@Resource("name")` parameter. + +## Registration + +```java +Taskito taskito = Taskito.builder().sqlite("app.db").open(); + +taskito.resource("smtp", context -> SmtpClient.connect(config)); +taskito.resource("tx", ResourceScope.TASK, + context -> context.use("db").begin(), + tx -> tx.rollbackIfOpen()); +taskito.resource("db", PoolConfig.of(10).withMaxLifetime(Duration.ofMinutes(30)), + context -> Database.connect(dsn), + Database::close); +``` + +| Overload | Description | +|---|---| +| `resource(name, factory)` | Worker-scoped (the default). | +| `resource(name, scope, factory)` | Explicit `ResourceScope`. | +| `resource(name, scope, factory, dispose)` | With a disposer run when the scope ends. | +| `resource(name, PoolConfig, factory, dispose)` | A `POOLED` resource: a bounded pool checked out per task. | + +The factory receives a `ResourceContext` — `scope()` plus `use(name)` to depend +on other resources. A factory may only depend on same-or-longer-lived +resources (a `WORKER` factory can't use a `TASK` resource). + +## Scopes + +| `ResourceScope` | Lifetime | +|---|---| +| `WORKER` | Built once, lazily; shared across every task; disposed at worker teardown. | +| `THREAD` | Built lazily once per worker thread; disposed at worker shutdown. | +| `TASK` | Built lazily per task invocation; disposed when the task ends. | +| `REQUEST` | Built fresh on every `use()`; disposed when the task ends — never cached. | +| `POOLED` | A bounded pool; each task checks out one instance and returns it at task end. | + +Task-scoped disposal runs in reverse build order (LIFO), so dependents tear +down before their dependencies. + +## `PoolConfig` + +`PoolConfig.of(poolSize)` plus `withPoolMin(n)` (eager prewarm at worker +start), `withAcquireTimeout(Duration)` (checkout wait limit, default 10s), and +`withMaxLifetime(Duration)` (idle instances older than this are disposed +instead of reused). Part of the cross-SDK contract for pooled resources. + +## Metrics + +`taskito.resourceMetrics()` returns a `Map` of per-resource +counters — `created`, `disposed`, and `active` (`created − disposed`). + + + Registration happens on the `Taskito` client, but resources are built lazily + on the worker — a producer-only process never constructs them. See + [Context](/java/api-reference/context) for resolving resources inside a + handler. + diff --git a/docs/content/docs/java/api-reference/result.mdx b/docs/content/docs/java/api-reference/result.mdx new file mode 100644 index 00000000..8aa69f71 --- /dev/null +++ b/docs/content/docs/java/api-reference/result.mdx @@ -0,0 +1,52 @@ +--- +title: Result +description: "Awaiting jobs and reading typed results." +--- + +```java +String id = taskito.enqueue(add, payload); + +taskito.awaitJob(id, Duration.ofSeconds(30)); // block until terminal +Optional sum = taskito.getResult(id, Integer.class); // 5 +``` + +## Awaiting + +`awaitJob(id, timeout)` polls the store until the job reaches a terminal state +(`COMPLETE`, `DEAD`, or `CANCELLED`) and returns its `Optional` snapshot; +it throws a `TaskitoException` if the timeout elapses first. Any process +sharing the storage can await a job by id. + +## Reading the result + +| Method | Description | +|---|---| +| `getResult(id)` → `Optional` | The raw serialized result, if complete. | +| `getResult(id, Class)` → `Optional` | The result deserialized with the queue's [serializer](/java/api-reference/serializers). | + +Both return `Optional.empty()` while the job is still pending or running — pair +them with `awaitJob` (or an `EventName.SUCCESS` listener on the worker) when +you need to block. + +## The `Job` snapshot + +`getJob(id)` returns the job's current state without waiting: + +```java +taskito.getJob(id).ifPresent(job -> { + System.out.println(job.status); // JobStatus.RUNNING + System.out.println(job.progress); // 0–100, if reported + System.out.println(job.error); // last error, if any +}); +``` + +`Job` carries `id`, `queue`, `taskName`, `status`, `priority`, `createdAt`, +`scheduledAt`, `startedAt`, `completedAt`, `retryCount`, `maxRetries`, +`timeoutMs`, `progress`, `error`, `uniqueKey`, `namespace`, and `metadata`. +All timestamps are Unix milliseconds. + + + `awaitJob` is for waiting on one job (tests, request/response bridges). To + scan many jobs use `listJobs(JobFilter)`; for aggregate outcomes use + `metrics(taskName, sinceMs)`. + diff --git a/docs/content/docs/java/api-reference/serializers.mdx b/docs/content/docs/java/api-reference/serializers.mdx new file mode 100644 index 00000000..62525c1f --- /dev/null +++ b/docs/content/docs/java/api-reference/serializers.mdx @@ -0,0 +1,73 @@ +--- +title: Serializers +description: "JsonSerializer, MsgpackSerializer, payload codecs, and the Serializer interface." +--- + +```java +import org.byteveda.taskito.serialization.JsonSerializer; +import org.byteveda.taskito.serialization.MsgpackSerializer; + +Taskito taskito = Taskito.builder() + .sqlite("taskito.db") + .serializer(new MsgpackSerializer()) + .open(); +``` + +## Built-in serializers + +| Class | Description | +|---|---| +| `JsonSerializer` | Default. JSON via Jackson; accepts a custom `ObjectMapper`. | +| `MsgpackSerializer` | Compact binary (MessagePack via Jackson). Its `org.msgpack:jackson-dataformat-msgpack` dependency is `compileOnly` — add it to your build. | +| `SignedSerializer(delegate, key)` | Prefixes each payload with an HMAC-SHA256 tag; verifies it (constant-time) on read — authentication, not encryption. | +| `EncryptedSerializer(delegate, key)` | AES-GCM with a fresh 12-byte IV per payload — confidentiality **and** integrity. Key must be 16/24/32 bytes. | + +Producers and workers must configure a compatible serializer — the Rust core +treats payloads as opaque bytes. + +## Payload codecs + +A `PayloadCodec` is a byte-to-byte transform (`encode`/`decode`) applied after +serialization and reversed before deserialization — one implementation owns +both directions so the inverse can't drift. Codecs follow the cross-SDK +contract, so payloads survive producer/worker boundaries between SDKs. + +| Codec | Description | +|---|---| +| `GzipCodec` | Compression; decode is capped (default 64 MiB) against zip bombs. | +| `AesGcmCodec(key)` | AES-GCM encryption, `[12-byte IV][ciphertext‖tag]`. | +| `HmacCodec(key)` | HMAC-SHA256 signing, `[32-byte MAC][body]`, constant-time verify. | + +A chain applies in order on encode and in reverse on decode — order `GzipCodec` +*before* a signing/encryption codec so integrity is verified before +decompressing: + +```java +Taskito taskito = Taskito.builder() + .sqlite("taskito.db") + .codec(new GzipCodec(), new HmacCodec(secret)) // global chain + .codec("encrypted", new AesGcmCodec(key)) // named, per-task + .open(); + +Task buildReport = Task.of("build_report", Report.class) + .codecs("encrypted"); // this task only +``` + +The global chain wraps the queue serializer (a `CodecSerializer` under the +hood) and covers results too. Named codecs apply per task via `Task.codecs` +or the `@Compressed`/`@Encrypted` annotations — the same names (and keys) must +be registered on producers and workers. + +## The `Serializer` interface + +```java +public interface Serializer { + byte[] serialize(Object value); + T deserialize(byte[] bytes, Class type); + default Object deserialize(byte[] bytes, Type type) { ... } +} +``` + +The `Type` overload supports generic payloads from a `TypeReference`; the +default implementation handles plain `Class` types and rejects generic ones — +override it in a generics-aware implementation (as `JsonSerializer` does). diff --git a/docs/content/docs/java/api-reference/task.mdx b/docs/content/docs/java/api-reference/task.mdx new file mode 100644 index 00000000..b4129b61 --- /dev/null +++ b/docs/content/docs/java/api-reference/task.mdx @@ -0,0 +1,95 @@ +--- +title: Task +description: "Task.of typed tasks, EnqueueOptions, RetryPolicy, and the annotations." +--- + +```java +import org.byteveda.taskito.task.Task; + +Task sendEmail = Task.of("send_email", EmailPayload.class); +``` + +A `Task` is an immutable descriptor: a name, the payload type, and default +enqueue options. The fluent option methods each return a new descriptor. + +## Factories + +| Method | Description | +|---|---| +| `Task.of(name, Class)` | Payload deserializes to `payloadType`. | +| `Task.of(name, TypeReference)` | Generic payloads a `Class` token can't express, e.g. `new TypeReference>(){}`. | + +## Fluent options + +| Method | Description | +|---|---| +| `queue(String)` | Target queue (default `default`). | +| `priority(int)` | Higher runs first. | +| `maxRetries(int)` (alias `retries`) | Attempts before dead-lettering. | +| `timeoutMs(long)` / `timeout(Duration)` | Per-attempt timeout. | +| `delayMs(long)` / `delay(Duration)` | Schedule after a delay. | +| `retryPolicy(RetryPolicy)` | Backoff curve — registered with the worker on `start()`. | +| `codecs(String...)` | Named [payload codecs](/java/api-reference/serializers) applied to this task. | +| `withOptions(EnqueueOptions)` | Replace the default options wholesale. | + +```java +Task sendEmail = Task.of("send_email", EmailPayload.class) + .queue("emails") + .maxRetries(5) + .timeout(Duration.ofSeconds(30)) + .retryPolicy(RetryPolicy.exponential(Duration.ofSeconds(2), Duration.ofMinutes(5))); +``` + +## `EnqueueOptions` + +Per-enqueue overrides, built with `EnqueueOptions.builder()`; unset fields take +core defaults. Fields: `queue`, `priority`, `maxRetries`, `timeoutMs` / +`timeout(Duration)`, `delayMs` / `delay(Duration)`, `uniqueKey` (alias +`jobId`) for idempotent enqueues, `metadata`, `namespace`. + +```java +taskito.enqueue(sendEmail, payload, EnqueueOptions.builder() + .delay(Duration.ofHours(24)) + .uniqueKey("welcome:" + userId) + .build()); +``` + +## `RetryPolicy` + +| Factory | Description | +|---|---| +| `exponential(base, max)` | Retry N waits about `base · 2^N`, capped at `max`, plus jitter. | +| `delays(Duration...)` | Explicit per-attempt delays, applied exactly (no jitter). | + +The retry *budget* comes from `maxRetries`; the policy only supplies the +timing, which the core scheduler applies durably. + +## Annotations + +`@TaskHandler` marks a method as a task handler. A compile-time processor +(`org.byteveda:taskito` ships it as the `processor` artifact) generates a +`FooTasks` companion per enclosing class `Foo` — a typed `Task` constant per +handler plus `bind(Worker.Builder, Foo)` and `handlers(Foo)` — no runtime +reflection. + +```java +class Mailer { + @TaskHandler("send_email") + Receipt send(EmailPayload payload, @Resource("smtp") SmtpClient smtp) { + return smtp.deliver(payload); + } +} + +Worker worker = taskito.worker() + .register(MailerTasks.handlers(new Mailer())) + .start(); +``` + +| Annotation | Description | +|---|---| +| `@TaskHandler(value, queue, maxRetries, timeoutMs, priority)` | Task name defaults to the method name; other fields default to the core defaults. | +| `@Resource("name")` | Injects a worker [resource](/java/api-reference/resources) into a parameter after the payload. | +| `@Compressed` | Records the `"compressed"` codec on the generated task. | +| `@Encrypted` | Records the `"encrypted"` codec — register the actual codec (and key) at runtime. | + +All are source-retention: read at compile time, never at runtime. diff --git a/docs/content/docs/java/api-reference/worker.mdx b/docs/content/docs/java/api-reference/worker.mdx new file mode 100644 index 00000000..8dc9bea5 --- /dev/null +++ b/docs/content/docs/java/api-reference/worker.mdx @@ -0,0 +1,70 @@ +--- +title: Worker +description: "The Worker.Builder and the running worker handle." +--- + +```java +Worker worker = taskito.worker() + .handle(sendEmail, payload -> mailer.deliver(payload)) + .queues("default", "emails") + .start(); +``` + +`taskito.worker()` returns a `Worker.Builder`; `start()` launches the native +scheduler and returns a running `Worker` (an `AutoCloseable`). + +## `Worker.Builder` + +### Handlers + +| Method | Description | +|---|---| +| `handle(Task, TaskFunction)` | Register a typed handler; the task's retry policy and codecs come with it. | +| `handle(String taskName, Class, TaskFunction)` | Register by name + payload class. | +| `register(Handler)` | A `Handler.of(task, function)` pair. | +| `register(HandlerRegistry)` | Every handler in a bundle (e.g. a generated `FooTasks.handlers(impl)`). | +| `apply(Consumer)` | Apply a customizer (e.g. a generated `FooTasks.bind`). | + +`TaskFunction` is `R apply(T payload) throws Exception` — the return +value becomes the job result; a thrown exception fails the attempt. + +### Options + +| Method | Default | Description | +|---|---|---| +| `queues(String...)` | `default` | Queue names to serve. | +| `concurrency(int)` | 0 (cached pool) | Fixed handler-thread count. | +| `channelCapacity(int)` | 128 | In-flight dispatch channel capacity. | +| `batchSize(int)` | 1 | Jobs claimed per scheduler poll. | +| `autoscale(AutoscaleOptions)` | — | Resize the handler pool between `min..max` threads by queue depth. | +| `mesh(MeshOptions)` | — | Join a scheduling mesh (gossip discovery + work stealing). | +| `on(EventName, Consumer)` | — | Outcome listeners: `SUCCESS`, `RETRY`, `DEAD`, `CANCELLED`. | +| `trackWorkflows()` / `trackWorkflows(Workflow...)` | off | Drive [workflow](/java/api-reference/workflows) state from this worker's outcomes; register workflows whose deferred nodes need payloads. | + +`AutoscaleOptions.of(min, max)` defaults to ~10 tasks per worker, re-evaluated +every 2s. `MeshOptions.builder().port(7946).seed("10.0.0.2:7946").build()` is +all most clusters set — the database stays the source of truth, so mesh is a +pure throughput optimization. + +## `Worker` + +| Method | Description | +|---|---| +| `stop()` | Stop dispatching; in-flight jobs continue to drain. | +| `close()` | Stop, drain in-flight handlers (30s grace), free the native worker, tear down worker resources. Idempotent. | +| `awaitShutdown()` | Block until `close()` is called. | +| `approveGate(runId, node)` / `rejectGate(runId, node, reason)` | Resolve a parked workflow gate (requires `trackWorkflows`). | +| `meshClusterInfo()` | `Optional` — peer count, capacity, load (mesh workers only). | + +```java +try (Worker worker = taskito.worker() + .handle(sendEmail, mailer::deliver) + .concurrency(8) + .on(EventName.DEAD, event -> log.warn("dead: {}", event.jobId)) + .start()) { + worker.awaitShutdown(); +} +``` + +The worker registers itself and heartbeats so it appears on the dashboard +Workers panel; `close()` unregisters it. diff --git a/docs/content/docs/java/api-reference/workflows.mdx b/docs/content/docs/java/api-reference/workflows.mdx new file mode 100644 index 00000000..8181ffde --- /dev/null +++ b/docs/content/docs/java/api-reference/workflows.mdx @@ -0,0 +1,100 @@ +--- +title: Workflows +description: "The Workflow builder, WorkflowRun handle, gates, sagas, and analysis." +--- + +```java +Workflow etl = Workflow.named("etl") + .step("extract", extract, source) + .step("transform", transform, null, "extract") + .step("load", load, null, "transform"); + +WorkflowRun run = taskito.submitWorkflow(etl); +WorkflowStatus done = run.await(Duration.ofMinutes(5)); +``` + +Steps run in topological order; a step waits for every predecessor named in +its `after` list. The running worker must be built with `trackWorkflows()` +(and `trackWorkflows(workflow)` for gates, sub-workflows, callable conditions, +and other deferred nodes). + +## `Workflow` builder + +| Method | Description | +|---|---| +| `Workflow.named(name)` / `version(int)` | Start a definition (version defaults to 1). | +| `step(name, task, payload, after...)` | A task node with a baked-in payload. | +| `stepAfter(name, task, deps...)` | A structural node — payload supplied at submit via `submitWorkflow(wf, payloads)`. | +| `step(Step)` | A fully-configured `Step` (see below). | +| `fanOut(name, task, FanMode.EACH, after)` | One child per item of the predecessor's list result. | +| `fanIn(name, task, FanMode.ALL, after)` | Collect the fan-out children's results into one list. | +| `gate(name, GateConfig, after...)` | Park for approval — a control node, never enqueued. | +| `subWorkflow(name, child, after...)` | Submit `child` as a child run; completes when it finalizes. | + +`Canvas` adds shortcuts: `Canvas.chain(name, links...)` (sequential), +`Canvas.group(name, links...)` (parallel), and `Canvas.chord(name, callback, +group...)` (parallel group joined by a callback), where each link is +`Canvas.link(name, task, payload)`. + +## `Step.of(...)` builder + +`Step.of(name, task, payload)` (or `(name, task)` for runtime-derived +payloads) then: `after(String...)`, `queue`, `maxRetries`, `timeoutMs`, +`priority`, `fanOut`/`fanIn(FanMode)`, `gate(GateConfig)`, +`condition("on_success" | "on_failure" | "always")` — with `onSuccess()` / +`onFailure()` / `always()` shorthands — `condition(Condition)` (a code +predicate over the run's `WorkflowContext`), `compensate(task)` (saga +rollback), and `cache(Duration ttl)` (skip re-running an unchanged step; +requires a predecessor). `build()` validates the combination. + +## Gates + +`GateConfig.manual()` waits indefinitely; `GateConfig.timeout(duration, +GateAction.APPROVE | GateAction.REJECT, message)` auto-resolves after the +timeout. Resolve a parked gate from the tracking worker: + +```java +worker.approveGate(run.id(), "review"); +worker.rejectGate(run.id(), "review", "failed QA"); +``` + +## Sagas + +A step with `compensate(rollbackTask)` registers a rollback: when the run +fails, completed compensable steps roll back in reverse-dependency order, each +compensation job receiving the step's forward result as its payload. The run +ends `COMPENSATED`, or `COMPENSATION_FAILED` if a rollback itself fails. + +## `WorkflowRun` + +| Member | Description | +|---|---| +| `id()` (alias `runId()`) / `name()` | The run's identity. | +| `status()` | `Optional` — the current run + node snapshot. | +| `await(timeout)` / `await(timeout, pollInterval)` | Block until terminal; throws `WorkflowException` on timeout. | +| `cancel()` | Skip pending nodes and mark the run cancelled. | + +`WorkflowStatus` exposes `state` (a `WorkflowState`: `RUNNING`, `COMPLETED`, +`COMPLETED_WITH_FAILURES`, `FAILED`, `CANCELLED`, `COMPENSATING`, +`COMPENSATED`, `COMPENSATION_FAILED`, …), `nodes` (per-step `NodeSnapshot` +with `status`, `jobId`, `fanOutCount`, timestamps), `isTerminal()`, +`node(name)`, and `failedStep()`. `taskito.workflowStatus(runId)` and +`taskito.cancelWorkflow(runId)` work from any process by run id. + +## Analysis & visualization + +`WorkflowAnalysis` is static graph computation over a *definition* — use it to +validate or preview structure before submitting: + +```java +WorkflowAnalysis.validate(etl); // throws on cycles / unknown deps +WorkflowAnalysis.topologicalOrder(etl); // a valid run order +WorkflowAnalysis.levels(etl); // nodes grouped by dependency depth +WorkflowAnalysis.roots(etl); // entry nodes +WorkflowAnalysis.leaves(etl); // exit nodes +WorkflowAnalysis.ancestors(etl, "load"); // transitive upstream deps +WorkflowAnalysis.descendants(etl, "extract"); +``` + +`WorkflowVisualization.mermaid(workflow)` and `.dot(workflow)` render the DAG +for docs or dashboards. From 0ced733ea22ff878ccde9adb119fb687639a2cb4 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Tue, 7 Jul 2026 02:51:50 +0530 Subject: [PATCH 05/16] docs(java): examples --- .../docs/java/more/examples/benchmark.mdx | 113 +++++++++++ .../docs/java/more/examples/bulk-emails.mdx | 114 +++++++++++ .../docs/java/more/examples/data-pipeline.mdx | 141 ++++++++++++++ .../content/docs/java/more/examples/index.mdx | 24 +++ .../content/docs/java/more/examples/meta.json | 14 ++ .../docs/java/more/examples/notifications.mdx | 159 +++++++++++++++ .../more/examples/predicate-gated-jobs.mdx | 119 ++++++++++++ .../docs/java/more/examples/saga-checkout.mdx | 122 ++++++++++++ .../java/more/examples/spring-service.mdx | 182 ++++++++++++++++++ .../docs/java/more/examples/web-scraper.mdx | 181 +++++++++++++++++ .../docs/java/more/examples/workflows.mdx | 159 +++++++++++++++ docs/content/docs/java/more/meta.json | 5 + 12 files changed, 1333 insertions(+) create mode 100644 docs/content/docs/java/more/examples/benchmark.mdx create mode 100644 docs/content/docs/java/more/examples/bulk-emails.mdx create mode 100644 docs/content/docs/java/more/examples/data-pipeline.mdx create mode 100644 docs/content/docs/java/more/examples/index.mdx create mode 100644 docs/content/docs/java/more/examples/meta.json create mode 100644 docs/content/docs/java/more/examples/notifications.mdx create mode 100644 docs/content/docs/java/more/examples/predicate-gated-jobs.mdx create mode 100644 docs/content/docs/java/more/examples/saga-checkout.mdx create mode 100644 docs/content/docs/java/more/examples/spring-service.mdx create mode 100644 docs/content/docs/java/more/examples/web-scraper.mdx create mode 100644 docs/content/docs/java/more/examples/workflows.mdx create mode 100644 docs/content/docs/java/more/meta.json diff --git a/docs/content/docs/java/more/examples/benchmark.mdx b/docs/content/docs/java/more/examples/benchmark.mdx new file mode 100644 index 00000000..1d6b4eb3 --- /dev/null +++ b/docs/content/docs/java/more/examples/benchmark.mdx @@ -0,0 +1,113 @@ +--- +title: Benchmark +description: "A self-contained throughput program: measure enqueue rate, processing rate, and end-to-end latency on your own hardware." +--- + +Numbers depend heavily on your machine, backend, and task body, so the only +number that matters is the one you measure. This program stages a batch of +no-op jobs, drains them with a worker, and reports the three rates that bound +a queue. + +## Benchmark.java + +```java +import java.util.ArrayList; +import java.util.List; +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.model.Job; +import org.byteveda.taskito.model.JobFilter; +import org.byteveda.taskito.model.JobStatus; +import org.byteveda.taskito.task.Task; +import org.byteveda.taskito.worker.Worker; + +public final class Benchmark { + private static final int N = 50_000; + private static final Task NOOP = Task.of("noop", Integer.class); + + public static void main(String[] args) throws InterruptedException { + try (Taskito taskito = Taskito.builder().sqlite("bench.db").open()) { + + // 1. Enqueue throughput — stage N jobs in batches of 1,000. + long t0 = System.currentTimeMillis(); + for (int i = 0; i < N; i += 1_000) { + List batch = new ArrayList<>(1_000); + for (int k = 0; k < 1_000; k++) { + batch.add(i + k); + } + taskito.enqueueMany(NOOP, batch); + } + long enqueueMs = System.currentTimeMillis() - t0; + + // 2. Processing throughput — drain the backlog, polling stats until empty. + long t1 = System.currentTimeMillis(); + long processMs; + try (Worker worker = taskito.worker() + .handle(NOOP, n -> null) + .concurrency(8) + .batchSize(64) + .channelCapacity(512) + .start()) { + while (taskito.stats().completed < N) { + Thread.sleep(20); + } + processMs = System.currentTimeMillis() - t1; + } + + // 3. End-to-end latency — created → completed across a sample of jobs. + List latencies = taskito + .listJobs(JobFilter.builder().status(JobStatus.COMPLETE).limit(1_000).build()) + .stream() + .filter(job -> job.completedAt != null) + .map(job -> job.completedAt - job.createdAt) + .sorted() + .toList(); + long p50 = latencies.get((int) (latencies.size() * 0.5)); + long p99 = latencies.get((int) (latencies.size() * 0.99)); + + System.out.printf("enqueue: %d jobs/s%n", Math.round(N / (enqueueMs / 1000.0))); + System.out.printf("process: %d jobs/s%n", Math.round(N / (processMs / 1000.0))); + System.out.printf("latency: p50 %dms p99 %dms%n", p50, p99); + } + } +} +``` + +## Running it + +```bash +java -cp app.jar Benchmark +``` + +Sample output (illustrative — measure your own): + +``` +enqueue: 110000 jobs/s +process: 30000 jobs/s +latency: p50 9ms p99 45ms +``` + +## What drives the numbers + +| Lever | Effect | +| --- | --- | +| `enqueueMany` | one storage round-trip per batch instead of per job | +| `batchSize` | jobs claimed per scheduler poll — fewer polls under load | +| `channelCapacity` | in-flight buffer between the scheduler and the handler pool | +| `concurrency` | handler threads draining the channel | +| Backend | SQLite (WAL) for a single node; Postgres/Redis to scale out workers | +| Task body | a real handler's I/O usually dominates — benchmark *your* task too | + + + The hot path is the Rust core — enqueue, dequeue, and result handling happen + natively; your JVM only runs the handler body. Raise `batchSize` and + `channelCapacity` together to keep a busy worker saturated. + + +## Key patterns + +| Pattern | Where | +| --- | --- | +| Batched staging | `taskito.enqueueMany` | +| Drain to a target | poll `taskito.stats().completed` | +| Worker throughput knobs | `batchSize` + `channelCapacity` + `concurrency` | +| Latency percentiles | `listJobs` + `completedAt − createdAt` | diff --git a/docs/content/docs/java/more/examples/bulk-emails.mdx b/docs/content/docs/java/more/examples/bulk-emails.mdx new file mode 100644 index 00000000..b229765d --- /dev/null +++ b/docs/content/docs/java/more/examples/bulk-emails.mdx @@ -0,0 +1,114 @@ +--- +title: Bulk Emails +description: "Fan a large recipient list out with enqueueMany or a Batcher, then bound the send parallelism with worker concurrency." +--- + +Sending to a large list is an enqueue-throughput problem: stage every message +as its own job in as few round-trips as possible, then let the worker's +bounded pool pace the actual sends so you stay within your provider's quota. + +## Tasks.java + +One job per recipient keeps retries and failures isolated — a bad address +never blocks the rest. The retry policy spaces attempts against a flaky +provider. + +```java +import java.time.Duration; +import org.byteveda.taskito.task.RetryPolicy; +import org.byteveda.taskito.task.Task; + +public final class Tasks { + public record Email(String to, String subject, String body) {} + + public static final Task SEND_EMAIL = Task.of("send_email", Email.class) + .maxRetries(5) + .retryPolicy(RetryPolicy.exponential(Duration.ofSeconds(2), Duration.ofMinutes(5))); + + private Tasks() {} +} +``` + +## Send.java + +`enqueueMany` stages a whole chunk in a single storage round-trip. Chunk a +very large list so each call stays a reasonable size. + +```java +import java.util.ArrayList; +import java.util.List; +import org.byteveda.taskito.Taskito; + +public final class Send { + public static List sendCampaign(Taskito taskito, List recipients, + String subject, String body) { + List ids = new ArrayList<>(recipients.size()); + for (int i = 0; i < recipients.size(); i += 1_000) { + List chunk = recipients.subList(i, Math.min(i + 1_000, recipients.size())) + .stream() + .map(to -> new Tasks.Email(to, subject, body)) + .toList(); + ids.addAll(taskito.enqueueMany(Tasks.SEND_EMAIL, chunk)); + } + return ids; // one job id per recipient + } + + private Send() {} +} +``` + +For a *trickle* of sends produced across many threads, a `Batcher` buffers +payloads and flushes them as one `enqueueMany` call when it reaches `maxBatch` +or `maxDelay` elapses: + +```java +try (Batcher batcher = + Batcher.of(taskito, Tasks.SEND_EMAIL, 500, Duration.ofSeconds(2))) { + events.forEach(event -> batcher.add(toEmail(event))); +} // close() flushes what remains +``` + + + `enqueueMany` does not apply `uniqueKey` dedup — the batch path trades + idempotency for throughput. If you must not double-send, dedup the recipient + list before enqueueing, or use single `enqueue` calls with a `uniqueKey`. + + +## WorkerMain.java + +The fixed pool is the pacing knob: at most 20 sends run at once. Swap +`concurrency` for `autoscale(AutoscaleOptions.of(4, 20))` to shrink the pool +when the backlog drains. + +```java +try (Taskito taskito = Taskito.builder().sqlite("bulk-email.db").open(); + Worker worker = taskito.worker() + .handle(Tasks.SEND_EMAIL, email -> provider.send(email)) + .concurrency(20) // at most 20 in flight at once + .start()) { + worker.awaitShutdown(); +} +``` + +## Monitoring the send + +Watch the backlog drain and catch addresses that exhausted their retries. + +```java +QueueStats stats = taskito.statsByQueue("default"); +System.out.printf("pending=%d running=%d dead=%d%n", stats.pending, stats.running, stats.dead); + +for (DeadJob dead : taskito.listDeadByTask("send_email", 50, 0)) { + System.err.printf("gave up on %s: %s%n", dead.taskName, dead.error); +} +``` + +## Key patterns + +| Pattern | Where | +| --- | --- | +| One round-trip per chunk | `taskito.enqueueMany` | +| Buffered trickle staging | `Batcher.of(taskito, task, maxBatch, maxDelay)` | +| Concurrency cap | `worker().concurrency(20)` (or `autoscale`) | +| Isolated failures | one job per recipient | +| Drain + failure visibility | `statsByQueue` + `listDeadByTask` | diff --git a/docs/content/docs/java/more/examples/data-pipeline.mdx b/docs/content/docs/java/more/examples/data-pipeline.mdx new file mode 100644 index 00000000..87bde45c --- /dev/null +++ b/docs/content/docs/java/more/examples/data-pipeline.mdx @@ -0,0 +1,141 @@ +--- +title: ETL Data Pipeline +description: "A diamond-shaped extract → transform → load DAG with a failure branch, error inspection, and structural analysis." +--- + +A classic ETL job: pull rows from a source, run two independent transforms over +the staged data, then load the merged result. Modeling it as a workflow gives +you dependency ordering, per-step retries, and a queryable run you can monitor. + + clean + extract --> enrich + clean --> load:::sink + enrich --> load`} +/> + +## Project structure + +``` +pipeline/ + Pipeline.java # the four tasks + the DAG + Monitor.java # analyze + inspect a run + WorkerMain.java # the worker process +``` + +## Pipeline.java + +Each step coordinates through a staging store (the DB, S3, a temp dir) rather +than passing results down the graph — the standard ETL shape. `load` runs only +after both transforms complete; a `quarantine` branch fires only when a step +fails. + +```java +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.task.Task; +import org.byteveda.taskito.workflows.Step; +import org.byteveda.taskito.workflows.Workflow; +import org.byteveda.taskito.workflows.WorkflowRun; + +public final class Pipeline { + public static final Task EXTRACT = Task.of("extract", String.class).maxRetries(3); + public static final Task CLEAN = Task.of("clean", String.class); + public static final Task ENRICH = Task.of("enrich", String.class).queue("io"); + public static final Task LOAD = Task.of("load", String.class); + public static final Task QUARANTINE = Task.of("quarantine", String.class); + + /** Every step receives the same runId payload and coordinates via staging. */ + public static WorkflowRun run(Taskito taskito, String runId) { + Workflow etl = Workflow.named("etl") + .step("extract", EXTRACT, runId) + .step("clean", CLEAN, runId, "extract") + .step(Step.of("enrich", ENRICH, runId).after("extract").queue("io").build()) + .step(Step.of("load", LOAD, runId).after("clean", "enrich").priority(10).build()) + // Failure branch: fires only when an upstream step fails. + .step(Step.of("quarantine", QUARANTINE, runId).after("load").onFailure().build()); + return taskito.submitWorkflow(etl); + } + + private Pipeline() {} +} +``` + +## Monitor.java + +`WorkflowAnalysis` gives you a structural view of the definition; +`workflowStatus` + `jobErrors` surface the failure history of any step's +underlying job. + +```java +import java.util.List; +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.model.JobError; +import org.byteveda.taskito.workflows.WorkflowAnalysis; +import org.byteveda.taskito.workflows.Workflow; + +public final class Monitor { + + /** Validate + preview the DAG before ever submitting it. */ + public static void preview(Workflow etl) { + WorkflowAnalysis.validate(etl); // throws on cycles / unknown deps + System.out.println("order: " + WorkflowAnalysis.topologicalOrder(etl)); + System.out.println("levels: " + WorkflowAnalysis.levels(etl)); + System.out.println("roots: " + WorkflowAnalysis.roots(etl)); + } + + /** Inspect a run: which step failed, and why, attempt by attempt. */ + public static void report(Taskito taskito, String runId) { + taskito.workflowStatus(runId).ifPresent(status -> { + System.out.println("state: " + status.state.wire()); + status.failedStep().ifPresent(step -> { + status.node(step).ifPresent(node -> { + List errors = taskito.jobErrors(node.jobId); + errors.forEach(err -> + System.out.printf("%s attempt %d: %s%n", step, err.attempt, err.error)); + }); + }); + }); + } + + private Monitor() {} +} +``` + + + If a required step fails, its still-pending dependents are skipped and the + run ends `completed_with_failures` (or `failed`). The `onFailure()` step is + the recovery path — it runs cleanup or alerting exactly when the happy path + didn't. + + +## WorkerMain.java + +The worker consumes both queues and tracks workflows so conditional nodes +(the `quarantine` branch) are evaluated. + +```java +try (Taskito taskito = Taskito.builder().sqlite("pipeline.db").open(); + Worker worker = taskito.worker() + .handle(Pipeline.EXTRACT, runId -> staging.write(runId, "raw", source.read())) + .handle(Pipeline.CLEAN, runId -> staging.transform(runId, "raw", "clean")) + .handle(Pipeline.ENRICH, runId -> staging.transform(runId, "raw", "enrich")) + .handle(Pipeline.LOAD, runId -> warehouse.load(staging.merge(runId))) + .handle(Pipeline.QUARANTINE, runId -> staging.quarantine(runId)) + .queues("default", "io") + .trackWorkflows() + .start()) { + worker.awaitShutdown(); +} +``` + +## Key patterns + +| Pattern | Where | +| --- | --- | +| Diamond dependency DAG | `Step.of(...).after("clean", "enrich")` | +| Per-step queue / priority | the `Step` builder | +| Failure branch | `Step.of(...).onFailure()` | +| Pre-submit validation | `WorkflowAnalysis.validate` / `topologicalOrder` | +| Per-attempt error history | `workflowStatus` → `failedStep` → `jobErrors` | +| Skip-on-failure cascade | required step failure ⇒ pending dependents skipped | diff --git a/docs/content/docs/java/more/examples/index.mdx b/docs/content/docs/java/more/examples/index.mdx new file mode 100644 index 00000000..de88670e --- /dev/null +++ b/docs/content/docs/java/more/examples/index.mdx @@ -0,0 +1,24 @@ +--- +title: Examples +description: "End-to-end, runnable Java examples — each demonstrating a slice of Taskito's Java SDK in a realistic setting." +--- + +Each example is a self-contained project you can copy, run, and adapt. They +lean on the [API reference](/java/api-reference) — read that for the surface, +come here for working code. + +| Example | Demonstrates | +| --- | --- | +| [Spring Boot service](/java/more/examples/spring-service) | REST API on the Spring starter that enqueues jobs, reports status, and cancels in flight | +| [Notification service](/java/more/examples/notifications) | Delayed scheduling, idempotency, priority, periodic tasks, batch enqueue | +| [Web scraper](/java/more/examples/web-scraper) | Retry backoff, named queues, middleware hooks, fan-out workflows | +| [ETL data pipeline](/java/more/examples/data-pipeline) | A multi-stage DAG with error history, failure branches, and structural analysis | +| [DAG workflows](/java/more/examples/workflows) | Fan-out/fan-in, gates, conditions, sub-workflows, caching, canvas shortcuts | +| [Saga — checkout](/java/more/examples/saga-checkout) | Compensating transactions with reverse-order rollback | +| [Bulk emails](/java/more/examples/bulk-emails) | High-volume `enqueueMany` and `Batcher` with bounded worker concurrency | +| [Predicate-gated jobs](/java/more/examples/predicate-gated-jobs) | Enqueue-time gates with `allOf` / `anyOf` / `not` and defer/skip decisions | +| [Benchmark](/java/more/examples/benchmark) | Measuring enqueue, processing, and end-to-end latency and throughput | + +Every snippet targets the standalone Java SDK — `org.byteveda:taskito`, Java 17 — +with the Rust core embedded in the artifact. Pick the one closest to your +problem and start there. diff --git a/docs/content/docs/java/more/examples/meta.json b/docs/content/docs/java/more/examples/meta.json new file mode 100644 index 00000000..e549450a --- /dev/null +++ b/docs/content/docs/java/more/examples/meta.json @@ -0,0 +1,14 @@ +{ + "title": "Examples", + "pages": [ + "spring-service", + "notifications", + "web-scraper", + "data-pipeline", + "workflows", + "saga-checkout", + "bulk-emails", + "predicate-gated-jobs", + "benchmark" + ] +} diff --git a/docs/content/docs/java/more/examples/notifications.mdx b/docs/content/docs/java/more/examples/notifications.mdx new file mode 100644 index 00000000..fde7b38a --- /dev/null +++ b/docs/content/docs/java/more/examples/notifications.mdx @@ -0,0 +1,159 @@ +--- +title: Notification Service +description: "Delayed sends, idempotent enqueues, priority lanes, periodic digests, and batch fan-out for a notification system." +--- + +A notification service is a natural fit for a queue: every send is independent, +some are scheduled for later, duplicates must be suppressed, and urgent alerts +should jump the line. This example wires those requirements onto the Java SDK. + +## Project structure + +``` +notifications/ + Tasks.java # one task per channel + payload records + Service.java # the producer API the rest of the app calls + WorkerMain.java # the worker process +``` + +## Tasks.java + +Each channel is its own task so it retries and scales independently. + +```java +import java.time.Duration; +import org.byteveda.taskito.task.RetryPolicy; +import org.byteveda.taskito.task.Task; + +public final class Tasks { + public record Email(String to, String subject, String body) {} + public record Sms(String to, String text) {} + + public static final Task SEND_EMAIL = Task.of("send_email", Email.class) + .maxRetries(5) + .retryPolicy(RetryPolicy.exponential(Duration.ofSeconds(2), Duration.ofMinutes(5))); + + public static final Task SEND_SMS = Task.of("send_sms", Sms.class) + .maxRetries(3); + + public static final Task SEND_DIGEST = Task.of("send_digest", String.class); + + private Tasks() {} +} +``` + +## Service.java + +The producer-facing helpers map application events onto enqueue options. + +```java +import java.time.Duration; +import java.util.List; +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.model.JobFilter; +import org.byteveda.taskito.model.JobStatus; +import org.byteveda.taskito.task.EnqueueOptions; + +public final class Service { + private final Taskito taskito; + + public Service(Taskito taskito) { + this.taskito = taskito; + } + + // 1. Delayed scheduling — remind 24h from now. + public String scheduleReminder(String email) { + return taskito.enqueue(Tasks.SEND_EMAIL, new Tasks.Email(email, "Reminder", "..."), + EnqueueOptions.builder().delay(Duration.ofHours(24)).build()); + } + + // 2. Idempotency — a duplicate enqueue with the same key is a no-op while + // the first is still pending or running. + public String sendWelcomeOnce(String userId, String email) { + return taskito.enqueue(Tasks.SEND_EMAIL, new Tasks.Email(email, "Welcome", "..."), + EnqueueOptions.builder().uniqueKey("welcome:" + userId).build()); + } + + // 3. Priority — security alerts preempt routine mail. + public String sendSecurityAlert(String phone, String text) { + return taskito.enqueue(Tasks.SEND_SMS, new Tasks.Sms(phone, text), + EnqueueOptions.builder().priority(100).build()); + } + + // 4. Batch fan-out — stage a whole digest run in one storage call. + public List sendDigests(List due) { + return taskito.enqueueMany(Tasks.SEND_EMAIL, due); + } + + // 5. Cancellation — pull a not-yet-started send. + public boolean cancelScheduled(String jobId) { + return taskito.cancel(jobId); // false if it already started + } + + // 6. Inspection — what's still pending? + public List pendingSends() { + return taskito.listJobs(JobFilter.builder() + .status(JobStatus.PENDING) + .task("send_email") + .limit(50) + .build()); + } +} +``` + +## WorkerMain.java + +The worker runs the channels and registers the periodic digest at `08:00` +America/New_York time. + +```java +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.scheduling.PeriodicTask; +import org.byteveda.taskito.worker.Worker; + +public final class WorkerMain { + public static void main(String[] args) throws InterruptedException { + try (Taskito taskito = Taskito.builder().sqlite("notifications.db").open()) { + taskito.registerPeriodic(PeriodicTask.builder("daily-digest", "send_digest", "0 8 * * *") + .timezone("America/New_York") + .build()); + + try (Worker worker = taskito.worker() + .handle(Tasks.SEND_EMAIL, email -> emailProvider.send(email)) + .handle(Tasks.SEND_SMS, sms -> smsProvider.send(sms)) + .handle(Tasks.SEND_DIGEST, ignored -> buildAndFanOutDigests()) + .queues("default") + .start()) { + worker.awaitShutdown(); + } + } + } +} +``` + +## Running it + + + + ```bash + java -cp app.jar WorkerMain + ``` + + + ```bash + jshell> new Service(taskito).sendSecurityAlert("+1...", "Login from a new device") + ``` + + + +## Key patterns + +| Pattern | Where | +| --- | --- | +| Delayed / scheduled send | `EnqueueOptions.builder().delay(...)` | +| Idempotent send | `EnqueueOptions.builder().uniqueKey(...)` | +| Priority lane | `EnqueueOptions.builder().priority(...)` | +| Cancel a pending job | `taskito.cancel` | +| Recurring digest | `registerPeriodic` (cron + timezone) | +| Batch fan-out | `taskito.enqueueMany` | +| Pending-work inspection | `taskito.listJobs(JobFilter)` | diff --git a/docs/content/docs/java/more/examples/predicate-gated-jobs.mdx b/docs/content/docs/java/more/examples/predicate-gated-jobs.mdx new file mode 100644 index 00000000..ce8e5750 --- /dev/null +++ b/docs/content/docs/java/more/examples/predicate-gated-jobs.mdx @@ -0,0 +1,119 @@ +--- +title: Predicate-Gated Jobs +description: "Reject, skip, or defer enqueues that don't meet a policy at submit time, composing business-hours, feature-flag, and quota gates." +--- + +A predicate is a function evaluated when a job is enqueued. If it returns +`false`, the enqueue is rejected with a `PredicateRejectedException` and no job +is created — backpressure at the front door, before any work is staged. The +richer `EnqueueGate` can also *skip* silently or *defer* to a later time. + +## What this gates + +- promotional sends only during business hours, +- an expensive re-index only when its feature flag is on, +- a tenant's exports only while under quota. + +## Policies.java + +A `Predicate` receives a `PredicateContext(taskName, payload)`. Compose them +with `Predicates.allOf` / `anyOf` / `not`; `Recipes` ships the ready-made +time-window and feature-flag gates. + +```java +import org.byteveda.taskito.predicates.Predicate; + +public final class Policies { + public static final Predicate UNDER_QUOTA = context -> { + String tenantId = (String) context.payload(); + return usage.exports(tenantId) < usage.limit(tenantId); + }; + + public static final Predicate FLAG_ON = + context -> flags.isOn("expensive-reindex"); + + private Policies() {} +} +``` + +## App.java + +`taskito.predicate` attaches a boolean gate; `taskito.gate` attaches a +decision gate (allow / skip / defer / reject). Multiple gates on one task run +in registration order and the first non-allow decision wins. + +```java +import java.time.ZoneId; +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.predicates.Predicates; +import org.byteveda.taskito.predicates.Recipes; + +Taskito taskito = Taskito.builder().sqlite("gated.db").open(); + +// Out-of-hours promos are *deferred* to the next opening, not dropped. +taskito.gate("send_promo", Recipes.businessHours(ZoneId.of("America/New_York"))); + +// Re-index only while its flag is on; otherwise skip silently. +taskito.predicate("reindex", Policies.FLAG_ON); + +// Exports: under quota AND outside business hours (off-peak only). +taskito.predicate("export_data", Predicates.allOf( + Policies.UNDER_QUOTA, + Predicates.not(context -> isBusinessHours()))); +``` + +## Enqueuing against a gate + +A rejected enqueue throws synchronously — handle it where you submit. When a +skip is an expected outcome, `tryEnqueue` returns an empty `Optional` instead +of throwing. + +```java +import java.util.Optional; +import org.byteveda.taskito.errors.PredicateRejectedException; + +try { + taskito.enqueue(SEND_PROMO, "newsletter-subscribers"); +} catch (PredicateRejectedException rejected) { + respond(409, "blocked by policy"); +} + +// Skip-tolerant: empty means a gate decided this enqueue shouldn't happen. +Optional id = taskito.tryEnqueue(REINDEX, "catalog"); +id.ifPresentOrElse( + jobId -> log.info("reindex staged as {}", jobId), + () -> log.info("reindex flag is off — skipped")); +``` + +## Custom defer decisions + +An `EnqueueGate` maps a `PredicateContext` to an `EnqueueDecision` — build +your own windows on top of `defer`/`deferUntil`: + +```java +import java.time.Duration; +import org.byteveda.taskito.predicates.EnqueueDecision; + +taskito.gate("send_promo", context -> + marketingPaused() + ? EnqueueDecision.defer(Duration.ofHours(1)) + : EnqueueDecision.allow()); +``` + + + Predicates and gates run at **enqueue time** in the producer process, + synchronously. Keep them cheap and side-effect-free — a slow gate slows + every enqueue. For checks that need the worker (resource state, external + I/O), gate inside the task and fail it instead. + + +## Key patterns + +| Pattern | Where | +| --- | --- | +| Attach a boolean gate | `taskito.predicate(task, predicate)` | +| Allow / skip / defer / reject | `taskito.gate(task, enqueueGate)` | +| Compose policies | `Predicates.allOf` / `anyOf` / `not` | +| Ready-made windows | `Recipes.businessHours` / `timeWindow` / `dayOfWeek` / `featureFlag` | +| Handle rejection | `catch (PredicateRejectedException)` | +| Tolerate skips | `taskito.tryEnqueue` → `Optional` | diff --git a/docs/content/docs/java/more/examples/saga-checkout.mdx b/docs/content/docs/java/more/examples/saga-checkout.mdx new file mode 100644 index 00000000..fba9752b --- /dev/null +++ b/docs/content/docs/java/more/examples/saga-checkout.mdx @@ -0,0 +1,122 @@ +--- +title: Saga — e-commerce checkout +description: "A multi-step checkout with compensating transactions: when a later step fails, completed steps roll back in reverse order." +--- + +A checkout touches several systems — inventory, payments, shipping — with no +distributed transaction across them. The saga pattern models each forward step +with a compensator; if a later step fails, Taskito runs the compensators for +the already-completed steps in reverse-dependency order. + +## Tasks.java + +Declare each forward task and its rollback. A compensator receives the forward +step's *result* as its payload, so it knows exactly what to undo. + +```java +public final class Tasks { + public record Reservation(String reservationId) {} + public record Charge(String chargeId) {} + public record Shipment(String shipmentId) {} + + public static final Task RESERVE = Task.of("reserve_inventory", Order.class); + public static final Task RELEASE = Task.of("release_inventory", Reservation.class); + + public static final Task CHARGE = Task.of("charge_payment", Order.class); + public static final Task REFUND = Task.of("refund_payment", Charge.class); + + public static final Task SHIP = Task.of("create_shipment", Order.class); + public static final Task CANCEL_SHIPMENT = Task.of("cancel_shipment", Shipment.class); + + private Tasks() {} +} +``` + +## Checkout.java + +Each step names its `compensate` task. The steps run in dependency order; the +compensators are wired automatically. + +```java +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.workflows.Step; +import org.byteveda.taskito.workflows.Workflow; +import org.byteveda.taskito.workflows.WorkflowRun; + +public final class Checkout { + public static WorkflowRun submit(Taskito taskito, Order order) { + Workflow checkout = Workflow.named("checkout") + .step(Step.of("reserve", Tasks.RESERVE, order) + .compensate(Tasks.RELEASE) + .build()) + .step(Step.of("charge", Tasks.CHARGE, order) + .after("reserve") + .compensate(Tasks.REFUND) + .build()) + .step(Step.of("ship", Tasks.SHIP, order) + .after("charge") + .compensate(Tasks.CANCEL_SHIPMENT) + .build()); + return taskito.submitWorkflow(checkout); + } + + private Checkout() {} +} +``` + +## Running it + +The worker registers the forward *and* compensation handlers, and tracks +workflows so the saga orchestrator can drive the rollback. + +```java +WorkflowRun run = Checkout.submit(taskito, order); +WorkflowStatus done = run.await(Duration.ofMinutes(1)); + +switch (done.state) { + case COMPLETED -> System.out.println("order placed"); + case COMPENSATED -> System.out.println("rolled back cleanly — customer charged nothing"); + case COMPENSATION_FAILED -> System.err.println("manual intervention needed"); + default -> System.out.println(done.state.wire()); +} +``` + +## What happens on failure + +If `create_shipment` throws, `reserve` and `charge` have already completed. +Taskito runs their compensators in reverse order: + +``` +ship ✗ failed +charge → refund_payment(Charge{chargeId}) (compensate) +reserve → release_inventory(Reservation{reservationId}) (compensate) +``` + +The run lands in `COMPENSATED` if every compensator succeeds, or +`COMPENSATION_FAILED` if one of them throws. Independent rollbacks in the same +wave run in parallel; each compensation job carries an idempotency key, so a +tracker restart never double-dispatches one. + + + Compensators should be idempotent. A refund or release may be retried, and a + partially-applied rollback that re-runs must not double-refund or release + twice. + + +## Run states + +| `WorkflowState` | Meaning | +| --- | --- | +| `COMPLETED` | every forward step succeeded | +| `COMPENSATING` | a step failed; compensators are running | +| `COMPENSATED` | rollback finished cleanly | +| `COMPENSATION_FAILED` | a compensator threw — needs manual repair | + +## Key patterns + +| Pattern | Where | +| --- | --- | +| Forward + rollback pair | `Step.of(...).compensate(task)` | +| Reverse-order rollback | automatic on step failure | +| Result-aware undo | compensator receives the forward result | +| Outcome branching | `switch (done.state)` | diff --git a/docs/content/docs/java/more/examples/spring-service.mdx b/docs/content/docs/java/more/examples/spring-service.mdx new file mode 100644 index 00000000..c10c796d --- /dev/null +++ b/docs/content/docs/java/more/examples/spring-service.mdx @@ -0,0 +1,182 @@ +--- +title: Spring Boot Image Service +description: "A REST API on the Spring Boot 3 starter that enqueues image jobs, reports status, and cancels work in flight." +--- + +A thin Spring Boot front end accepts upload requests, enqueues a Taskito job, +and hands back a job id immediately. The browser then polls the status +endpoint (state, progress, error) and can cancel a job that hasn't started. + +## Project structure + +``` +image-service/ + src/main/java/example/ + ImageServiceApplication.java # Spring Boot app + Taskito config + ImageTasks.java # the processImage task + handler + ImageController.java # the REST surface + WorkerRunner.java # starts the worker with the app + src/main/resources/application.yaml +``` + +## Dependencies + +The starter auto-configures a single `Taskito` bean from `taskito.*` +properties and closes it with the application context. + +```kotlin +dependencies { + implementation("org.byteveda:taskito") + implementation("org.byteveda:taskito-spring") + implementation("org.springframework.boot:spring-boot-starter-web") +} +``` + +```yaml +# application.yaml +taskito: + url: image-service.db # SQLite; or a postgres:// / redis:// URL +``` + +## ImageTasks.java + +The task descriptor and its handler live together. The handler is plain code — +one payload in, one result out. + +```java +package example; + +import java.util.ArrayList; +import java.util.List; +import org.byteveda.taskito.task.Task; +import org.springframework.stereotype.Component; + +@Component +public class ImageTasks { + + public record ProcessImage(String url, List sizes) {} + + public record Variant(int size, String key) {} + + public static final Task PROCESS_IMAGE = + Task.of("process_image", ProcessImage.class) + .maxRetries(2) + .timeoutMs(120_000); + + public List process(ProcessImage payload) throws Exception { + List variants = new ArrayList<>(); + for (int size : payload.sizes()) { + resize(payload.url(), size); // your real work here + variants.add(new Variant(size, payload.url() + "@" + size + ".webp")); + } + return variants; + } + + private void resize(String url, int size) throws Exception { + // ... + } +} +``` + +## ImageController.java + +Enqueue and return the job id with `202 Accepted`; report status from the +`Job` snapshot; cancel a job that hasn't started. + +```java +package example; + +import java.util.Map; +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.task.EnqueueOptions; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +@RestController +@RequestMapping("/images") +public class ImageController { + private final Taskito taskito; + + public ImageController(Taskito taskito) { + this.taskito = taskito; + } + + @PostMapping + public ResponseEntity> submit(@RequestBody ImageTasks.ProcessImage request) { + String jobId = taskito.enqueue( + ImageTasks.PROCESS_IMAGE, + request, + EnqueueOptions.builder().priority(5).build()); + return ResponseEntity.accepted().body(Map.of("jobId", jobId)); + } + + @GetMapping("/{id}") + public ResponseEntity> status(@PathVariable String id) { + return taskito.getJob(id) + .map(job -> ResponseEntity.ok(Map.of( + "status", job.status.wire(), + "progress", job.progress == null ? 0 : job.progress, + "error", job.error == null ? "" : job.error))) + .orElse(ResponseEntity.notFound().build()); + } + + @PostMapping("/{id}/cancel") + public Map cancel(@PathVariable String id) { + // cancel() pulls a pending job; requestCancel() flags a running one. + boolean cancelled = taskito.cancel(id) || taskito.requestCancel(id); + return Map.of("cancelled", cancelled); + } +} +``` + +## WorkerRunner.java + +The worker runs in-process here for a single-service deploy; split it into its +own Spring profile (or process) when API latency matters. `Worker` is closed +by the container on shutdown, draining in-flight jobs. + +```java +package example; + +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.worker.Worker; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class WorkerRunner { + + @Bean(destroyMethod = "close") + public Worker worker(Taskito taskito, ImageTasks tasks) { + return taskito.worker() + .handle(ImageTasks.PROCESS_IMAGE, tasks::process) + .queues("default") + .concurrency(4) + .start(); + } +} +``` + +## Running it + +```bash +./gradlew bootRun +# POST http://localhost:8080/images {"url": "...", "sizes": [256, 512, 1024]} +# GET http://localhost:8080/images/ +``` + + + The starter's `@ConditionalOnMissingBean` means you can define your own + `Taskito` bean (custom serializer, codecs, resources) and the + auto-configuration steps aside. + + +## Key patterns + +| Pattern | Where | +| --- | --- | +| Auto-configured client | `taskito.url` property + `org.byteveda:taskito-spring` | +| Enqueue and return a job id (`202 Accepted`) | `POST /images` | +| Status polling | `taskito.getJob` → status / progress / error | +| Cancel pending or running work | `taskito.cancel` / `taskito.requestCancel` | +| Graceful worker shutdown | `@Bean(destroyMethod = "close")` on the `Worker` | diff --git a/docs/content/docs/java/more/examples/web-scraper.mdx b/docs/content/docs/java/more/examples/web-scraper.mdx new file mode 100644 index 00000000..60ded2aa --- /dev/null +++ b/docs/content/docs/java/more/examples/web-scraper.mdx @@ -0,0 +1,181 @@ +--- +title: Web Scraper Pipeline +description: "A polite scraper: retry backoff, a dedicated network queue with bounded concurrency, middleware logging, and a fan-in workflow that aggregates results." +--- + +Scraping is bursty, failure-prone, and must respect the target's limits. This +example fetches a set of pages with exponential-backoff retries, isolates +network work on its own queue served by a small worker pool, and joins the +results with a fan-out / fan-in workflow. + +## Project structure + +``` +scraper/ + Tasks.java # fetch / list / aggregate + logging middleware + Run.java # submit the workflow: list → fetch (fan-out) → aggregate + WorkerMain.java # two workers + periodic cleanup +``` + +## Tasks.java + +`fetch_page` retries with exponential backoff and lives on a dedicated +`network` queue so a backlog there never starves the default queue. Politeness +comes from the network worker's small fixed pool — at most two fetches run at +once. + +```java +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import org.byteveda.taskito.middleware.Middleware; +import org.byteveda.taskito.middleware.TaskContext; +import org.byteveda.taskito.task.RetryPolicy; +import org.byteveda.taskito.task.Task; + +public final class Tasks { + public static final Task FETCH_PAGE = Task.of("fetch_page", String.class) + .queue("network") + .maxRetries(4) + .retryPolicy(RetryPolicy.exponential(Duration.ofSeconds(1), Duration.ofMinutes(1))); + + // Source step: supplies the URL list the fan-out expands over. + public static final Task> LIST_URLS = + Task.of("list_urls", new com.fasterxml.jackson.core.type.TypeReference>() {}); + + // Fan-in combiner: receives one entry per fetched page (each child's result). + public static final Task> AGGREGATE = + Task.of("aggregate", new com.fasterxml.jackson.core.type.TypeReference>() {}); + + private static final HttpClient HTTP = HttpClient.newHttpClient(); + + public static String fetch(String url) throws Exception { + HttpResponse res = HTTP.send( + HttpRequest.newBuilder(URI.create(url)).GET().build(), + HttpResponse.BodyHandlers.ofString()); + if (res.statusCode() >= 400) { + throw new IllegalStateException("HTTP " + res.statusCode() + " for " + url); + } + return res.body(); + } + + public static Map aggregate(List pages) { + long links = pages.stream() + .flatMap(html -> html.lines().filter(line -> line.contains("href="))) + .count(); + return Map.of("pages", pages.size(), "links", (int) links); + } + + /** Middleware: one log line per execution and per failure. */ + public static Middleware logging() { + return new Middleware() { + @Override + public void before(TaskContext ctx) { + System.out.printf("→ %s %s%n", ctx.taskName, ctx.jobId); + } + + @Override + public void after(TaskContext ctx, Object result) { + System.out.printf("✓ %s %s%n", ctx.taskName, ctx.jobId); + } + + @Override + public void onError(TaskContext ctx, Throwable error) { + System.out.printf("✗ %s %s: %s%n", ctx.taskName, ctx.jobId, error); + } + }; + } + + private Tasks() {} +} +``` + +## Run.java + +A fan-out / fan-in expands the URL list into one `fetch_page` child per URL, +then `aggregate` receives every fetched page as one list. + +```java +import java.time.Duration; +import java.util.List; +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.workflows.FanMode; +import org.byteveda.taskito.workflows.Workflow; +import org.byteveda.taskito.workflows.WorkflowRun; + +public final class Run { + public static void main(String[] args) { + List urls = List.of("https://a.example", "https://b.example", "https://c.example"); + + try (Taskito taskito = Taskito.builder().sqlite("scraper.db").open()) { + Workflow scrape = Workflow.named("scrape") + .step("list", Tasks.LIST_URLS, urls) + .fanOut("fetch", Tasks.FETCH_PAGE, FanMode.EACH, "list") + .fanIn("aggregate", Tasks.AGGREGATE, FanMode.ALL, "fetch"); + + WorkflowRun run = taskito.submitWorkflow(scrape); + var status = run.await(Duration.ofMinutes(2)); + System.out.println(run.id() + " " + status.state.wire()); + } + } +} +``` + + + Fan-in passes the children's results to its task as one list, so `aggregate` + sees every fetched page. A `Canvas.chord` would instead join on *completion* + and call the callback with its own payload — use it when the join doesn't + need the children's results. + + +## WorkerMain.java + +Two workers over the same store: a small fixed pool for the polite network +queue and a wider one for everything else. A periodic sweep drops completed +jobs older than a day. + +```java +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.scheduling.PeriodicTask; +import org.byteveda.taskito.worker.Worker; + +public final class WorkerMain { + public static void main(String[] args) throws InterruptedException { + try (Taskito taskito = Taskito.builder().sqlite("scraper.db").open()) { + taskito.use(Tasks.logging()); + taskito.registerPeriodic(PeriodicTask.builder("cache-sweep", "cleanup_cache", "0 * * * *").build()); + + try (Worker network = taskito.worker() + .handle(Tasks.FETCH_PAGE, Tasks::fetch) + .queues("network") + .concurrency(2) // politeness: at most 2 fetches in flight + .start(); + Worker main = taskito.worker() + .handle(Tasks.LIST_URLS, list -> list) + .handle(Tasks.AGGREGATE, Tasks::aggregate) + .handle("cleanup_cache", Long.class, + olderThanMs -> taskito.purgeCompleted(86_400_000L)) + .queues("default") + .trackWorkflows() + .start()) { + main.awaitShutdown(); + } + } + } +} +``` + +## Key patterns + +| Pattern | Where | +| --- | --- | +| Transient-failure retries | `RetryPolicy.exponential` | +| Workload isolation | `Task.queue("network")` + a dedicated worker | +| Politeness via bounded parallelism | `worker().concurrency(2)` on the network worker | +| Cross-cutting logging | `taskito.use(Middleware)` | +| Expand-then-join | `fanOut(FanMode.EACH)` + `fanIn(FanMode.ALL)` | +| Scheduled maintenance | `registerPeriodic` + `purgeCompleted` | diff --git a/docs/content/docs/java/more/examples/workflows.mdx b/docs/content/docs/java/more/examples/workflows.mdx new file mode 100644 index 00000000..f4c8be82 --- /dev/null +++ b/docs/content/docs/java/more/examples/workflows.mdx @@ -0,0 +1,159 @@ +--- +title: DAG Workflow Examples +description: "Fan-out/fan-in map-reduce, approval gates, error-handling conditions, sub-workflows, caching, and canvas shortcuts." +--- + +These are focused recipes for the `Workflow` builder. Each is independent — +combine them as your orchestration needs grow. See the +[Workflows reference](/java/api-reference/workflows) for the underlying model. + +## Map-reduce with fan-out / fan-in + +`fanOut` reads the list result of its predecessor and expands one child per +item; `fanIn` collects those children's results into a single list for a +combiner. + + map_0["map[0]"] + split --> map_1["map[1]"] + split --> map_2["map[2]"] + map_0 --> reduce + map_1 --> reduce + map_2 --> reduce`} +/> + +```java +Task split = Task.of("split", String.class); // → List +Task scoreChunk = Task.of("score_chunk", String.class); // per item +Task> reduce = + Task.of("reduce", new TypeReference>() {}); // [childResult, …] + +Workflow scoreCorpus = Workflow.named("score-corpus") + .step("split", split, corpus) + .fanOut("map", scoreChunk, FanMode.EACH, "split") + .fanIn("reduce", reduce, FanMode.ALL, "map"); + +WorkflowRun run = taskito.submitWorkflow(scoreCorpus); +WorkflowStatus done = run.await(Duration.ofMinutes(5)); +System.out.println(done.state.wire()); +``` + +## Approval gate + +A gate parks the run (`WAITING_APPROVAL`) until it is resolved — or until its +timeout elapses, when `GateAction` decides the outcome. The worker running the +flow must track it so the tracker holds the downstream payloads. + +```java +Workflow publish = Workflow.named("publish") + .step("build", buildArtifact, releaseId) + .gate("review", + GateConfig.timeout(Duration.ofHours(24), GateAction.REJECT, + "Approve production deploy?"), + "build") + .step("deploy", deployArtifact, releaseId, "review"); + +WorkflowRun run = taskito.submitWorkflow(publish); + +// elsewhere, once a reviewer clicks "approve" (worker built with trackWorkflows(publish)): +worker.approveGate(run.id(), "review"); +// or: worker.rejectGate(run.id(), "review", "failed QA"); +``` + +## Error-handling with conditions + +A step's condition decides when it runs relative to its predecessors' +outcomes. `onFailure()` builds an error path that fires only when an upstream +step fails; `condition(Condition)` gates on the run's state in code. + +```java +Workflow importBatch = Workflow.named("import") + .step("load", loadBatch, batchId) + .step(Step.of("notify_ok", notifySuccess, batchId).after("load").onSuccess().build()) + .step(Step.of("quarantine", quarantineBatch, batchId).after("load").onFailure().build()) + .step(Step.of("escalate", pageOncall, batchId) + .after("load") + .condition(ctx -> ctx.failureCount() > 3) // callable — worker must track this workflow + .build()); +``` + +## Sub-workflows + +Compose a child workflow as a single step. The tracker submits it as a child +run and resolves the parent node when the child finalizes. + +```java +Workflow validate = Workflow.named("validate-region") + .step("schema", checkSchema, region) + .step("dupes", checkDuplicates, region, "schema"); + +Workflow regionalEtl = Workflow.named("regional-etl") + .step("extract", extract, region) + .subWorkflow("validate", validate, "extract") + .step("load", load, region, "validate"); +``` + +## Cacheable, incremental steps + +Mark an expensive step `cache(ttl)` to skip re-running it on a later run of +the same workflow while its task + payload are unchanged and within the TTL. +A cacheable step must have a predecessor, and a cache hit produces no forward +result — so it can't feed a fan-out/fan-in. + +```java +Workflow nightly = Workflow.named("nightly") + .step("fetch", fetchRaw, date) + .step(Step.of("featurize", buildFeatures, date) + .after("fetch") + .cache(Duration.ofHours(24)) + .build()) + .step("train", trainModel, date, "featurize"); +``` + +## Canvas shortcuts + +`Canvas` builds common shapes from a few links — submit the result like any +workflow. + +```java +Workflow seq = Canvas.chain("nightly-chain", + Canvas.link("fetch", fetchRaw, date), + Canvas.link("train", trainModel, date)); + +Workflow join = Canvas.chord("crawl", + Canvas.link("report", buildReport, date), // callback after the group + Canvas.link("a", fetchSite, "https://a.example"), + Canvas.link("b", fetchSite, "https://b.example")); +``` + +## Pre-run analysis + +`WorkflowAnalysis` is pure graph computation over a definition — use it to +validate and preview structure before submitting. + +```java +WorkflowAnalysis.validate(nightly); // throws on a cycle +System.out.println(WorkflowAnalysis.topologicalOrder(nightly)); +System.out.println(WorkflowAnalysis.levels(nightly)); // nodes by dependency depth +System.out.println(WorkflowVisualization.mermaid(nightly)); // render the DAG +``` + + + `run.await(timeout)` returns once the run reaches a terminal state — + `COMPLETED`, `COMPLETED_WITH_FAILURES`, `FAILED`, `CANCELLED`, + `COMPENSATED`, or `COMPENSATION_FAILED`. Check `status.state` to branch on + the outcome. + + +## Key patterns + +| Pattern | Builder | +| --- | --- | +| Map-reduce | `fanOut(FanMode.EACH)` + `fanIn(FanMode.ALL)` | +| Human approval | `gate(GateConfig...)` + `worker.approveGate` / `rejectGate` | +| Error branch | `Step.of(...).onFailure()` / `condition(Condition)` | +| Composition | `subWorkflow(...)` | +| Incremental re-runs | `Step.of(...).cache(ttl)` | +| Shape shortcuts | `Canvas.chain` / `group` / `chord` | +| Inspection | `WorkflowAnalysis` + `WorkflowVisualization` | diff --git a/docs/content/docs/java/more/meta.json b/docs/content/docs/java/more/meta.json new file mode 100644 index 00000000..135f689a --- /dev/null +++ b/docs/content/docs/java/more/meta.json @@ -0,0 +1,5 @@ +{ + "title": "Examples", + "root": true, + "pages": ["examples"] +} From 8cea7ac32b4183b7dc1c687ae772c801947727b5 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Tue, 7 Jul 2026 02:53:56 +0530 Subject: [PATCH 06/16] docs(java): operations and extensibility guides --- .../docs/java/guides/extensibility/events.mdx | 43 +++++++++ .../docs/java/guides/extensibility/index.mdx | 13 +++ .../docs/java/guides/extensibility/meta.json | 1 + .../java/guides/extensibility/middleware.mdx | 76 ++++++++++++++++ .../java/guides/extensibility/serializers.mdx | 87 ++++++++++++++++++ .../java/guides/extensibility/webhooks.mdx | 73 +++++++++++++++ .../docs/java/guides/integrations/index.mdx | 33 +++++++ .../docs/java/guides/integrations/meta.json | 4 + .../java/guides/integrations/micrometer.mdx | 56 ++++++++++++ .../docs/java/guides/integrations/sentry.mdx | 44 ++++++++++ .../docs/java/guides/integrations/spring.mdx | 70 +++++++++++++++ .../docs/java/guides/observability/index.mdx | 19 ++++ .../java/guides/observability/logging.mdx | 87 ++++++++++++++++++ .../docs/java/guides/observability/meta.json | 1 + .../java/guides/observability/monitoring.mdx | 73 +++++++++++++++ .../java/guides/operations/autoscaling.mdx | 80 +++++++++++++++++ .../docs/java/guides/operations/backends.mdx | 32 +++++++ .../docs/java/guides/operations/cli.mdx | 53 +++++++++++ .../docs/java/guides/operations/dashboard.mdx | 66 ++++++++++++++ .../java/guides/operations/deployment.mdx | 88 +++++++++++++++++++ .../docs/java/guides/operations/graalvm.mdx | 57 ++++++++++++ .../docs/java/guides/operations/index.mdx | 20 +++++ .../java/guides/operations/inspection.mdx | 60 +++++++++++++ .../docs/java/guides/operations/mesh.mdx | 39 ++++++++ .../docs/java/guides/operations/meta.json | 1 + .../docs/java/guides/operations/security.mdx | 82 +++++++++++++++++ .../docs/java/guides/operations/testing.mdx | 84 ++++++++++++++++++ .../guides/operations/troubleshooting.mdx | 69 +++++++++++++++ 28 files changed, 1411 insertions(+) create mode 100644 docs/content/docs/java/guides/extensibility/events.mdx create mode 100644 docs/content/docs/java/guides/extensibility/index.mdx create mode 100644 docs/content/docs/java/guides/extensibility/meta.json create mode 100644 docs/content/docs/java/guides/extensibility/middleware.mdx create mode 100644 docs/content/docs/java/guides/extensibility/serializers.mdx create mode 100644 docs/content/docs/java/guides/extensibility/webhooks.mdx create mode 100644 docs/content/docs/java/guides/integrations/index.mdx create mode 100644 docs/content/docs/java/guides/integrations/meta.json create mode 100644 docs/content/docs/java/guides/integrations/micrometer.mdx create mode 100644 docs/content/docs/java/guides/integrations/sentry.mdx create mode 100644 docs/content/docs/java/guides/integrations/spring.mdx create mode 100644 docs/content/docs/java/guides/observability/index.mdx create mode 100644 docs/content/docs/java/guides/observability/logging.mdx create mode 100644 docs/content/docs/java/guides/observability/meta.json create mode 100644 docs/content/docs/java/guides/observability/monitoring.mdx create mode 100644 docs/content/docs/java/guides/operations/autoscaling.mdx create mode 100644 docs/content/docs/java/guides/operations/backends.mdx create mode 100644 docs/content/docs/java/guides/operations/cli.mdx create mode 100644 docs/content/docs/java/guides/operations/dashboard.mdx create mode 100644 docs/content/docs/java/guides/operations/deployment.mdx create mode 100644 docs/content/docs/java/guides/operations/graalvm.mdx create mode 100644 docs/content/docs/java/guides/operations/index.mdx create mode 100644 docs/content/docs/java/guides/operations/inspection.mdx create mode 100644 docs/content/docs/java/guides/operations/mesh.mdx create mode 100644 docs/content/docs/java/guides/operations/meta.json create mode 100644 docs/content/docs/java/guides/operations/security.mdx create mode 100644 docs/content/docs/java/guides/operations/testing.mdx create mode 100644 docs/content/docs/java/guides/operations/troubleshooting.mdx diff --git a/docs/content/docs/java/guides/extensibility/events.mdx b/docs/content/docs/java/guides/extensibility/events.mdx new file mode 100644 index 00000000..cbe1524e --- /dev/null +++ b/docs/content/docs/java/guides/extensibility/events.mdx @@ -0,0 +1,43 @@ +--- +title: Events +description: "Subscribe to job outcome events on the worker builder." +--- + +Subscribe to job outcome events with `Worker.Builder.on(...)`. Events fire in +the worker process after the core decides a job's outcome. + +```java +try (Worker worker = taskito.worker() + .handle(add, payload -> payload.a() + payload.b()) + .on(EventName.SUCCESS, event -> log.info("done " + event.jobId)) + .on(EventName.DEAD, event -> alertOps(event)) + .start()) { + worker.awaitShutdown(); +} +``` + +## Events + +| `EventName` | When | +|---|---| +| `SUCCESS` | A job finished successfully. | +| `RETRY` | A failed job is being retried. | +| `DEAD` | A job exhausted its retries and dead-lettered. | +| `CANCELLED` | A job was cancelled. | + +Each listener receives an `OutcomeEvent` carrying `jobId`, `taskName`, and +outcome details: `error` (null on success/cancel), `retryCount` (-1 when not +applicable), and `timedOut` (which separates a timeout from other failures). A +listener that throws is caught and logged — it never blocks other listeners. + +Listeners are add-only and live for the worker's lifetime; there is no +unsubscribe. Register them before `start()` and let them go away when the +worker closes. + + + Events are fire-and-forget notifications. To *wrap* execution (timing, + context, error transformation) use + [middleware](/java/guides/extensibility/middleware); to deliver events to + external HTTP endpoints use + [webhooks](/java/guides/extensibility/webhooks). + diff --git a/docs/content/docs/java/guides/extensibility/index.mdx b/docs/content/docs/java/guides/extensibility/index.mdx new file mode 100644 index 00000000..ea507a30 --- /dev/null +++ b/docs/content/docs/java/guides/extensibility/index.mdx @@ -0,0 +1,13 @@ +--- +title: Extensibility +description: "Extend the Java SDK at every stage of the task lifecycle." +--- + +Extend taskito with custom behavior at every stage of the task lifecycle. + +| Guide | Description | +|---|---| +| [Events](/java/guides/extensibility/events) | Subscribe to job outcome events on the worker | +| [Middleware](/java/guides/extensibility/middleware) | Wrap task execution with before/after/error and outcome hooks | +| [Webhooks](/java/guides/extensibility/webhooks) | Push signed notifications to external services | +| [Pluggable Serializers](/java/guides/extensibility/serializers) | Custom payload serialization — MessagePack, codecs, encryption | diff --git a/docs/content/docs/java/guides/extensibility/meta.json b/docs/content/docs/java/guides/extensibility/meta.json new file mode 100644 index 00000000..0cdf2157 --- /dev/null +++ b/docs/content/docs/java/guides/extensibility/meta.json @@ -0,0 +1 @@ +{ "title": "Extensibility", "pages": ["events", "middleware", "webhooks", "serializers"] } diff --git a/docs/content/docs/java/guides/extensibility/middleware.mdx b/docs/content/docs/java/guides/extensibility/middleware.mdx new file mode 100644 index 00000000..50e4d1b6 --- /dev/null +++ b/docs/content/docs/java/guides/extensibility/middleware.mdx @@ -0,0 +1,76 @@ +--- +title: Middleware +description: "Wrap task execution with before/after/error and outcome hooks." +--- + +Register middleware with `Taskito.use(...)` to wrap execution and react to +outcomes. `Middleware` is an interface whose hooks are all no-op `default` +methods — override only what you need. The `before`/`after`/`onError` hooks +wrap each attempt; the outcome hooks fire after the core decides the result. + +```java +taskito.use(new Middleware() { + @Override + public void before(TaskContext context) { + log.info("start " + context.taskName); + } + + @Override + public void onError(TaskContext context, Throwable error) { + log.error("threw " + context.taskName, error); + } + + @Override + public void onDeadLetter(OutcomeEvent event) { + alertOps(event); + } +}); +``` + +## Hooks + +| Hook | When | +|---|---| +| `onEnqueue(context)` | Producer-side, inside `enqueue`, before serialization. | +| `before(context)` | Before each execution attempt. | +| `after(context, result)` | After a successful attempt. | +| `onError(context, error)` | When an attempt throws (before the retry/dead decision). | +| `onCompleted(event)` | After a job completes successfully. | +| `onRetry(event)` | After the core schedules a retry. | +| `onDeadLetter(event)` | After a job dead-letters. | +| `onCancel(event)` | After a job is cancelled. | + +The execution hooks receive a `TaskContext` with `taskName`, `jobId`, a +mutable per-execution `attributes()` map shared across a job's hooks, and +`job()` with lazily-loaded metadata. The outcome hooks receive an +`OutcomeEvent` with `taskName`, `jobId`, `error`, `retryCount`, and `timedOut` +(which separates a timeout from other failures). Multiple middlewares run in +registration order for every phase. Execution hooks run inside the attempt — +a `before` or `after` that throws fails the attempt like a handler error; +outcome hooks are isolated — one that throws is caught and logged so it never +starves the others. + +## Rewriting an enqueue + +`onEnqueue` receives a **mutable** `EnqueueContext`: replace `payload(...)` or +`options(...)` to rewrite the job, add to `metadata()` to attach data that +travels with it (readable at execution via `context.job().metadata()`), or +throw to abort the enqueue. + +```java +taskito.use(new Middleware() { + @Override + public void onEnqueue(EnqueueContext context) { + context.metadata().put("tenant", currentTenant()); + } +}); +``` + + + Middleware runs in the hot path — keep hooks light. The contrib + [Micrometer](/java/guides/integrations/micrometer) and + [Sentry](/java/guides/integrations/sentry) integrations are built as + middleware. For typed convert/redirect/reject decisions on the producer, use + an `Interceptor` (`Taskito.intercept(...)`) — interceptors run before + middleware `onEnqueue`. + diff --git a/docs/content/docs/java/guides/extensibility/serializers.mdx b/docs/content/docs/java/guides/extensibility/serializers.mdx new file mode 100644 index 00000000..ef9cf19c --- /dev/null +++ b/docs/content/docs/java/guides/extensibility/serializers.mdx @@ -0,0 +1,87 @@ +--- +title: Pluggable Serializers +description: "Pluggable payload serialization — JSON, MessagePack, and codec chains." +--- + +Arguments and results are serialized with a pluggable `Serializer`. The native +core treats payloads as opaque bytes, so serialization is purely a Java +concern. + +- `JsonSerializer` — default; Jackson-backed, human-readable, debuggable. +- `MsgpackSerializer` — compact binary via MessagePack. Optional: add + `org.msgpack:jackson-dataformat-msgpack` to your build. +- `SignedSerializer(delegate, key)` — wraps another serializer with an + HMAC-SHA256 tag and rejects tampered payloads on read (authentication, not + encryption). +- `EncryptedSerializer(delegate, key)` — wraps another serializer with + AES-GCM for confidentiality **and** integrity (16/24/32-byte key). + +```java +Taskito taskito = Taskito.builder() + .url("taskito.db") + .serializer(new MsgpackSerializer()) + .open(); +``` + +## Custom serializer + +Implement the `Serializer` interface — `serialize` to bytes, `deserialize` +back: + +```java +public final class MySerializer implements Serializer { + @Override + public byte[] serialize(Object value) { /* ... */ } + + @Override + public T deserialize(byte[] bytes, Class type) { /* ... */ } +} +``` + + + Producers and workers must agree on the serializer — they exchange bytes + through storage. A job enqueued with MessagePack can only be decoded by a + worker using a compatible serializer. + + +## Payload codecs + +A `PayloadCodec` is a byte-to-byte transform applied **after** serialization on +the producer and reversed **before** deserialization on the worker — the +cross-SDK contract for compression, encryption, and signing. Built-ins: + +- `GzipCodec` — gzip compression; decompression capped at 64 MiB (configurable) + to guard against zip bombs. +- `AesGcmCodec(key)` — AES-GCM encryption, fresh 12-byte IV per payload. +- `HmacCodec(key)` — HMAC-SHA256 signing with constant-time verification. + +Chain them globally — applied in order on the way out, reversed on the way in: + +```java +Taskito taskito = Taskito.builder() + .url("taskito.db") + .codec(new GzipCodec(), new AesGcmCodec(key)) // compress, then encrypt + .open(); +``` + +Or register **named** codecs and select them per task: + +```java +Taskito taskito = Taskito.builder() + .url("taskito.db") + .codec("secret", new AesGcmCodec(key)) + .open(); + +Task export = Task.of("export", Report.class).codecs("secret"); +``` + +Handlers annotated `@Compressed` / `@Encrypted` get the `"compressed"` / +`"encrypted"` codec names recorded on their generated task constants — register +codecs under those names on both producers and workers. The key is supplied at +runtime, never in the annotation. + + + The same chain (or the same names) must be configured on producers and + workers. Put `GzipCodec` before a signing or encryption codec so integrity + is verified before decompressing. + diff --git a/docs/content/docs/java/guides/extensibility/webhooks.mdx b/docs/content/docs/java/guides/extensibility/webhooks.mdx new file mode 100644 index 00000000..6885ccfd --- /dev/null +++ b/docs/content/docs/java/guides/extensibility/webhooks.mdx @@ -0,0 +1,73 @@ +--- +title: Webhooks +description: "Deliver job events to HTTP endpoints — signed, retried, persisted." +--- + +Deliver job outcomes to HTTP endpoints. Deliveries are HMAC-SHA256 signed, +retried with exponential backoff, and subscriptions are persisted in storage — +they survive restarts. + + + +`WebhookManager.attach(taskito)` registers the manager as middleware on the +client; create subscriptions with the `Webhook` builder: + +```java +WebhookManager webhooks = WebhookManager.attach(taskito); + +Webhook hook = webhooks.create(Webhook.builder("https://hooks.example.com/jobs") + .on(EventName.DEAD, EventName.SUCCESS) // required — no events, no deliveries + .secret(System.getenv("WEBHOOK_SECRET")) + .taskFilter("send_email")); // optional, exact task name + +webhooks.list(); +webhooks.get(hook.id); +webhooks.delete(hook.id); +``` + +| Builder method | Default | Description | +|---|---|---| +| `on(EventName...)` | — | Outcomes to deliver. | +| `secret(String)` | none | Signs each delivery as `X-Taskito-Signature: sha256=`. | +| `taskFilter(String)` | all tasks | Only deliver for this exact task name. | +| `header(String, String)` | — | Extra headers added to every delivery. | +| `maxRetries(int)` | `3` | Retry budget per delivery. | +| `timeoutMs(long)` | `10000` | Per-request HTTP timeout. | +| `enabled(boolean)` | `true` | Disable without deleting. | +| `description(String)` | — | Free-form note. | + +Deliveries fire from the **worker process** (where outcomes originate), as an +async `POST` with a JSON body in snake_case: + +```json +{ + "event": "dead", + "job_id": "…", + "task_name": "send_email", + "error": "…", + "retry_count": 3, + "timed_out": false +} +``` + +A send error or a `5xx` response is retried with exponential backoff (500 ms +base, doubling, capped at 30 s) up to `maxRetries`; `4xx` responses are not +retried. Verify the signature on your endpoint by HMAC-SHA256-ing the raw body +with the shared secret and comparing against the `X-Taskito-Signature` header +in constant time. + + + Delivery is fire-and-forget: there is no persisted per-delivery log to query + — final failures are logged (with the URL's path redacted). Subscriptions + themselves are persisted in the queue's settings store, so every process + sharing the backend sees the same hooks; changes propagate to running + workers within 30 seconds. + diff --git a/docs/content/docs/java/guides/integrations/index.mdx b/docs/content/docs/java/guides/integrations/index.mdx new file mode 100644 index 00000000..03e60cc8 --- /dev/null +++ b/docs/content/docs/java/guides/integrations/index.mdx @@ -0,0 +1,33 @@ +--- +title: Integrations +description: "Spring Boot starter and optional observability helpers for the Java SDK." +--- + +The Java SDK ships a Spring Boot 3 starter as a separate artifact, and two +observability helpers in the `org.byteveda.taskito.contrib` package of the main +artifact. The contrib classes compile against their third-party APIs as +`compileOnly` dependencies — nothing is pulled onto your classpath until you +add the matching runtime dependency yourself. + +```kotlin +// Only add what you use. +implementation("org.byteveda:taskito-spring:0.18.0") // Spring Boot 3 starter +implementation("io.micrometer:micrometer-observation:1.13.6") // contrib.TaskitoObservation +implementation("io.sentry:sentry:7.14.0") // contrib.SentryMiddleware +``` + + + Integrations never auto-initialize. Nothing runs until you construct the + class and register it with `taskito.use(...)` (or, for Spring, until the + starter is on the classpath). If the runtime dependency is missing, using the + contrib class fails with a standard `NoClassDefFoundError`. + + +## Framework + +- [Spring Boot](/java/guides/integrations/spring) — auto-configured `Taskito` bean from `taskito.*` properties. + +## Observability + +- [Micrometer](/java/guides/integrations/micrometer) — one `Observation` per execution: a timer plus a trace span. +- [Sentry](/java/guides/integrations/sentry) — capture handler exceptions and dead-letters. diff --git a/docs/content/docs/java/guides/integrations/meta.json b/docs/content/docs/java/guides/integrations/meta.json new file mode 100644 index 00000000..995a6027 --- /dev/null +++ b/docs/content/docs/java/guides/integrations/meta.json @@ -0,0 +1,4 @@ +{ + "title": "Integrations", + "pages": ["index", "spring", "micrometer", "sentry"] +} diff --git a/docs/content/docs/java/guides/integrations/micrometer.mdx b/docs/content/docs/java/guides/integrations/micrometer.mdx new file mode 100644 index 00000000..8cfb2277 --- /dev/null +++ b/docs/content/docs/java/guides/integrations/micrometer.mdx @@ -0,0 +1,56 @@ +--- +title: Micrometer Observation +description: "One Observation per task execution — a timer and a trace span from a single instrumentation." +--- + +Optional dependency: `io.micrometer:micrometer-observation` (the SDK compiles +against it as `compileOnly`; add it to your build). `TaskitoObservation` is a +[middleware](/java/guides/extensibility/middleware) that wraps each task +execution in a Micrometer `Observation` — one instrumentation that yields both +**metrics** (a timer) and a **trace span**, exported by whatever handlers your +`ObservationRegistry` is configured with. + +```java +import org.byteveda.taskito.contrib.TaskitoObservation; + +taskito.use(new TaskitoObservation(registry)); +``` + +Each execution attempt starts an observation named `taskito.task` with the +low-cardinality key value `taskito.task` = the task name. On success the +observation stops cleanly; when the handler throws, the exception is recorded +on the observation before it stops. A retry is a new attempt, so it produces a +new observation. + +## Options + +The three-argument constructor customizes the observation name and filters +tasks: + +```java +taskito.use(new TaskitoObservation( + registry, + "app.job", // observation name (default "taskito.task") + task -> !task.startsWith("internal."))); // Predicate task filter +``` + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `registry` | `ObservationRegistry` | — | The registry your app configures | +| `name` | `String` | `"taskito.task"` | Observation (and derived timer/span) name | +| `taskFilter` | `Predicate` | all tasks | Return `false` to skip a task | + +## Wiring backends + +`TaskitoObservation` creates observations; it never configures exporters. Wire +the registry to your backends the standard Micrometer way — a +`DefaultMeterObservationHandler` over your `MeterRegistry` (Prometheus, +Datadog, ...) for metrics, and a `micrometer-tracing` bridge (OpenTelemetry, +Brave) for spans. In a Spring Boot app with Actuator, inject the +auto-configured `ObservationRegistry` and both come pre-wired. + + + Because metrics and traces derive from the same Observation, you instrument + once and choose backends per environment — no taskito-specific exporter + configuration exists or is needed. + diff --git a/docs/content/docs/java/guides/integrations/sentry.mdx b/docs/content/docs/java/guides/integrations/sentry.mdx new file mode 100644 index 00000000..ee1cb844 --- /dev/null +++ b/docs/content/docs/java/guides/integrations/sentry.mdx @@ -0,0 +1,44 @@ +--- +title: Sentry Integration +description: "Capture handler exceptions and dead-letters from Java workers in Sentry." +--- + +Optional dependency: `io.sentry:sentry` (the SDK compiles against it as +`compileOnly`; add it to your build). Call `Sentry.init(...)` in your app +before registering the middleware — `SentryMiddleware` never initializes Sentry +itself, and its hooks are safe no-ops when Sentry isn't initialized. + +```java +import io.sentry.Sentry; +import org.byteveda.taskito.contrib.SentryMiddleware; + +Sentry.init(options -> options.setDsn(System.getenv("SENTRY_DSN"))); + +taskito.use(new SentryMiddleware()); +``` + +Two things get reported: + +- **Every failed attempt** — the handler's exception is captured with its + stack trace when it throws (`onError`). A job that recovers on retry still + produced one event per failed attempt. +- **Dead-letters** — when a job exhausts its retries, a `FATAL`-level message + (`task dead-lettered: `) marks the terminal failure. + +Both carry the tags `taskito.task` (task name) and `taskito.job` (job id), so +you can group and search by task. + +## Filtering tasks + +The one-argument constructor takes a `Predicate` over the task name; +return `false` to skip a task entirely: + +```java +taskito.use(new SentryMiddleware(task -> !task.startsWith("noisy."))); +``` + + + Configure DSN, environment, and sampling with the Sentry SDK yourself — + the middleware only captures events onto whatever scope your app has + initialized. + diff --git a/docs/content/docs/java/guides/integrations/spring.mdx b/docs/content/docs/java/guides/integrations/spring.mdx new file mode 100644 index 00000000..5cd5d0d7 --- /dev/null +++ b/docs/content/docs/java/guides/integrations/spring.mdx @@ -0,0 +1,70 @@ +--- +title: Spring Boot Integration +description: "Auto-configure a Taskito bean in Spring Boot 3 from taskito.* properties." +--- + +The `taskito-spring` starter auto-configures a `Taskito` bean for Spring Boot 3 +applications. + +```kotlin +implementation("org.byteveda:taskito-spring:0.18.0") +``` + +When the starter is on the classpath, `TaskitoAutoConfiguration` builds and +opens a client from your configuration and registers it as a singleton bean. +The bean's destroy method is `close`, so the client shuts down with the +application context. + +```yaml +taskito: + url: postgres://localhost/taskito # SQLite path, postgres:// or redis:// URL + pool-size: 8 # optional; omit for the backend default + namespace: my-app # optional; isolates this app's jobs in a shared store +``` + +Inject it anywhere: + +```java +@Service +public class SignupService { + private final Taskito taskito; + + public SignupService(Taskito taskito) { + this.taskito = taskito; + } + + public void register(User user) { + taskito.enqueue("send_welcome_email", user.email()); + } +} +``` + +## Properties + +| Property | Type | Default | Description | +|---|---|---|---| +| `taskito.url` | `String` | SQLite under `.taskito/` | Connection string — SQLite path, `postgres://`, or `redis://` | +| `taskito.pool-size` | `Integer` | backend default | Connection-pool size | +| `taskito.namespace` | `String` | — | Namespace isolating this app's jobs in a shared store | + +## Overriding the bean + +The auto-configuration backs off (`@ConditionalOnMissingBean`) when you define +your own `Taskito` bean — do that when you need middleware, codecs, or a custom +serializer: + +```java +@Bean(destroyMethod = "close") +Taskito taskito(TaskitoProperties properties) { + Taskito taskito = Taskito.builder().url(properties.getUrl()).open(); + taskito.use(new SentryMiddleware()); // configure freely + return taskito; +} +``` + + + The starter configures only the client bean. Workers are yours to build and + start — construct one from the injected bean with `taskito.worker()` (for + example in an `ApplicationRunner` or a `SmartLifecycle` bean) and register + your handlers on it. + diff --git a/docs/content/docs/java/guides/observability/index.mdx b/docs/content/docs/java/guides/observability/index.mdx new file mode 100644 index 00000000..f0cdf56f --- /dev/null +++ b/docs/content/docs/java/guides/observability/index.mdx @@ -0,0 +1,19 @@ +--- +title: Observability +description: "Monitor queue health and emit structured logs from the Java SDK." +--- + +See what the queue is doing — counts, per-execution metrics, live workers, +per-job logs — and emit structured logs, all from the `Taskito` handle with no +extra service. + +| Page | What it covers | +|---|---| +| [Monitoring](/java/guides/observability/monitoring) | Stats, per-execution metrics, worker heartbeats, Micrometer/Sentry pointers | +| [Logging](/java/guides/observability/logging) | The built-in leveled logger and per-job task logs — levels, pluggable sink | + + + Lifecycle [events](/java/guides/extensibility/events) and + [middleware](/java/guides/extensibility/middleware) are the extension points + for pushing these signals to any external system. + diff --git a/docs/content/docs/java/guides/observability/logging.mdx b/docs/content/docs/java/guides/observability/logging.mdx new file mode 100644 index 00000000..9d8b636c --- /dev/null +++ b/docs/content/docs/java/guides/observability/logging.mdx @@ -0,0 +1,87 @@ +--- +title: Structured Task Logging +description: "The built-in leveled logger — levels, namespaces, a pluggable sink — plus per-job task logs." +--- + +The SDK ships a tiny zero-dependency leveled logger, `TaskitoLogger`. It writes +to **stderr** by default, so it never pollutes stdout (the CLI's JSON output and +piped data stay clean). Taskito uses it internally; it is public API for your +own use too. + +```java +import org.byteveda.taskito.logging.TaskitoLogger; + +TaskitoLogger log = TaskitoLogger.create("billing"); // tagged [taskito:billing] + +taskito.worker().handle("charge", ChargeRequest.class, request -> { + log.info("charging " + request.amount()); + return charge(request); +}); +``` + +## Levels + +`DEBUG < INFO < WARN < ERROR < SILENT`. The threshold defaults to `WARN` and is +read once from `TASKITO_LOG_LEVEL`; override it at runtime — globally and +immediately: + +```java +import org.byteveda.taskito.logging.LogLevel; + +TaskitoLogger.setLevel(LogLevel.DEBUG); +``` + +`debug` also takes a `Supplier` that is only evaluated when the level +passes the threshold — so expensive log lines cost nothing when filtered out: + +```java +log.debug(() -> "payload=" + bigObject); +``` + +`warn` and `error` take an optional `Throwable`; its stack trace is appended to +the line. + +```java +log.error("delivery failed", exception); +``` + +## Namespaces + +`TaskitoLogger.create(ns)` tags every line `[taskito:ns]`; `TaskitoLogger.root()` +is the bare `[taskito]` logger. + +## Custom sink + +Replace the output sink to route lines to a file, a JSON transport, or your +logging framework — globally and immediately. `LogSink` is a functional +interface receiving every formatted line that clears the threshold: + +```java +import org.byteveda.taskito.logging.LogSink; + +TaskitoLogger.setSink((level, line) -> slf4jLogger.info(line)); +``` + + + The sink only sees the Java SDK's own lines — logs from the native core do + not flow through it. + + +## Per-job task logs + +Separately from the process logger, jobs can carry **persisted, queryable** +log lines. `writeTaskLog` appends a line to storage; `getTaskLogs` reads a +job's lines back: + +```java +taskito.writeTaskLog(jobId, "import", "info", "processed batch 3/10"); + +for (var entry : taskito.getTaskLogs(jobId)) { + System.out.println(entry.loggedAt + " " + entry.level + " " + entry.message); +} +``` + +Each `TaskLog` carries `id`, `jobId`, `taskName`, `level`, `message`, an +optional `extra` payload, and `loggedAt` (Unix milliseconds). Because they live +in storage, task logs survive restarts and are visible from any process on the +same backend. diff --git a/docs/content/docs/java/guides/observability/meta.json b/docs/content/docs/java/guides/observability/meta.json new file mode 100644 index 00000000..c6d9beae --- /dev/null +++ b/docs/content/docs/java/guides/observability/meta.json @@ -0,0 +1 @@ +{ "title": "Observability", "pages": ["monitoring", "logging"] } diff --git a/docs/content/docs/java/guides/observability/monitoring.mdx b/docs/content/docs/java/guides/observability/monitoring.mdx new file mode 100644 index 00000000..5e01d384 --- /dev/null +++ b/docs/content/docs/java/guides/observability/monitoring.mdx @@ -0,0 +1,73 @@ +--- +title: Monitoring & Hooks +description: "Queue stats, per-execution metrics, and worker heartbeats from the Taskito handle." +--- + +Read live queue health straight from the `Taskito` handle — no extra service. +The same data powers the [dashboard](/java/guides/operations/dashboard) and the +[CLI](/java/guides/operations/cli) read commands. + +## Stats + +Counts by status, globally or per queue: + +```java +QueueStats stats = taskito.stats(); // pending, running, completed, failed, dead, cancelled +taskito.statsByQueue("default"); +taskito.statsAllQueues(); // Map +``` + +## Metrics + +`metrics(taskName, sinceMs)` returns the raw per-execution records within a +trailing window — wall time, memory, and success flag for each finished +attempt. Pass `null` as the task name for all tasks: + +```java +List lastHour = taskito.metrics(null, 3_600_000L); +for (TaskMetric m : taskito.metrics("add", 3_600_000L)) { + System.out.println(m.taskName + " " + m.wallTimeNs / 1_000_000 + "ms ok=" + m.succeeded); +} +``` + +Each `TaskMetric` carries `taskName`, `jobId`, `wallTimeNs` (nanoseconds), +`memoryBytes`, `succeeded`, and `recordedAt`. Aggregation (percentiles, rates) +is up to you — or plug in [Micrometer](/java/guides/integrations/micrometer) +for a ready-made timer per task. + +## Error history + +Per-attempt failures for one job: + +```java +List errors = taskito.jobErrors(jobId); // attempt, error, failedAt +``` + +## Workers + +Each running worker registers and heartbeats; list the live fleet: + +```java +for (WorkerInfo w : taskito.listWorkers()) { + System.out.println(w.workerId + " " + w.hostname + " " + w.status + + " threads=" + w.threads + " beat=" + w.lastHeartbeat); +} +``` + +A worker appears while it heartbeats (`lastHeartbeat` is Unix milliseconds); +`stop()` unregisters it. See [Deployment](/java/guides/operations/deployment) +for the worker lifecycle. + +## Metrics scraping & tracing + +For meters and distributed traces, the contrib integrations wrap the +middleware layer: + +- [Micrometer](/java/guides/integrations/micrometer) — one `Observation` per execution: a timer plus a trace span, exported by whatever backend your registry is wired to. +- [Sentry](/java/guides/integrations/sentry) — error capture. + + + Lifecycle [events](/java/guides/extensibility/events) and + [middleware](/java/guides/extensibility/middleware) hooks let you push the + same signals to any backend. + diff --git a/docs/content/docs/java/guides/operations/autoscaling.mdx b/docs/content/docs/java/guides/operations/autoscaling.mdx new file mode 100644 index 00000000..e0853f06 --- /dev/null +++ b/docs/content/docs/java/guides/operations/autoscaling.mdx @@ -0,0 +1,80 @@ +--- +title: Autoscaling +description: "In-process thread autoscaling, and scaling replicas with KEDA via the scaler endpoint." +--- + +Two layers scale with load: the **in-process autoscaler** resizes a single +worker's handler-thread pool, and the **scaler endpoint** exposes queue depth +over HTTP so an external autoscaler (such as KEDA) adds and removes worker +replicas. + +## In-process: autoscale the handler pool + +`autoscale(...)` replaces the worker's fixed/cached pool with a resizable one +driven by queue depth (pending + running, scoped to the worker's queues): + +```java +Worker worker = taskito.worker() + .handle(task, handler) + .autoscale(AutoscaleOptions.of(2, 16)) // min 2, max 16 threads + .start(); +``` + +`AutoscaleOptions.of(min, max)` targets ~10 outstanding tasks per thread, +re-evaluated every 2 seconds. The full record constructor tunes both: + +```java +new AutoscaleOptions(2, 16, 5, Duration.ofSeconds(1)); // 5 tasks/thread, 1s ticks +``` + +## Cross-process: the scaler endpoint + +`Scaler` serves queue depth over HTTP for an external autoscaler, which divides +it by `targetQueueDepth` to pick a replica count: + +```java +try (Taskito taskito = Taskito.builder().postgres(System.getenv("DATABASE_URL")).open(); + Scaler scaler = Scaler.start(taskito, ScalerOptions.onPort(9090))) { + // serves until closed +} +``` + +`ScalerOptions` is a record — `defaults()` is port `9090`, host `0.0.0.0`, +target depth `10`, all queues; the full constructor sets +`(port, host, targetQueueDepth, queue)`. Port `0` picks an ephemeral port +(`scaler.port()` reports it). + +| Endpoint | Returns | +|---|---| +| `GET /api/scaler[?queue=]` | `{ "metricValue": , "targetValue": , "queueName": ... }` | +| `GET /health` | `{ "status": "ok" }` | + +## KEDA + +Point KEDA's `metrics-api` scaler at the endpoint; it scales toward +`ceil(metricValue / targetValue)` replicas: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: taskito-workers +spec: + scaleTargetRef: + name: taskito-worker + minReplicaCount: 1 + maxReplicaCount: 20 + triggers: + - type: metrics-api + metadata: + url: "http://taskito-scaler:9090/api/scaler" + valueLocation: "metricValue" + targetValue: "10" +``` + + + Run the scaler as its own small Deployment + Service (not in the worker pod) + so scaling to zero workers doesn't take the metric source down with it. The + two layers compose: KEDA sizes the fleet, `autoscale(...)` sizes each + member's threads. + diff --git a/docs/content/docs/java/guides/operations/backends.mdx b/docs/content/docs/java/guides/operations/backends.mdx new file mode 100644 index 00000000..6c66e146 --- /dev/null +++ b/docs/content/docs/java/guides/operations/backends.mdx @@ -0,0 +1,32 @@ +--- +title: Backends +description: "SQLite, PostgreSQL, or Redis — same API, different storage." +--- + +The queue stores everything — jobs, results, schedules, locks, rate-limit state +— in one backend. The API is identical across all three; only the builder call +differs. + +```java +Taskito.builder().open(); // SQLite at .taskito/taskito.db (default) +Taskito.builder().sqlite("/var/lib/app/taskito.db").open(); // SQLite, explicit path +Taskito.builder().postgres(System.getenv("PG_URL")).schema("taskito").open(); +Taskito.builder().redis("redis://localhost").prefix("taskito").open(); +``` + +| Backend | Use when | Notes | +|---|---|---| +| **SQLite** | Single host, simplest deploy | One file, WAL mode, no server. Bundled. | +| **PostgreSQL** | Multiple hosts, durability | `SELECT ... FOR UPDATE SKIP LOCKED` dispatch; isolated `schema`. | +| **Redis** | High throughput, ephemeral OK | JSON values + sorted sets; atomic Lua claims. | + +`backend(name)` + `url(dsn)` are the generic forms behind the shortcuts; +`poolSize(int)` sizes the connection pool and `namespace(String)` isolates one +application's jobs in a shared store. All three backends ship in the bundled +native library — no extra build step. + + + Producers and workers must point at the same backend. SQLite is per-host + (shared file); Postgres/Redis let producers and workers run on different + machines against one shared store. + diff --git a/docs/content/docs/java/guides/operations/cli.mdx b/docs/content/docs/java/guides/operations/cli.mdx new file mode 100644 index 00000000..1b9ad3c5 --- /dev/null +++ b/docs/content/docs/java/guides/operations/cli.mdx @@ -0,0 +1,53 @@ +--- +title: CLI +description: "Operate the queue from the terminal with the bundled taskito command." +--- + +The main artifact bundles a picocli-based CLI (`org.byteveda.taskito.cli.Cli`) +that operates any queue from the terminal. Connect with `--url `, +plus `--backend postgres|redis` for the non-SQLite backends; with no options it +opens the default SQLite store at `.taskito/taskito.db`. + +```bash +alias taskito='java -cp app.jar org.byteveda.taskito.cli.Cli' + +taskito --url taskito.db stats +taskito --url taskito.db enqueue add '[2,3]' +taskito --url taskito.db jobs --status failed --limit 50 +taskito --url taskito.db cancel +taskito --url taskito.db pause default +taskito --url taskito.db resume default +taskito --url taskito.db dlq list +taskito --url taskito.db dlq retry +taskito --url taskito.db dlq delete +``` + +Read commands print pretty JSON, so the output pipes straight into `jq`. +`enqueue` takes the task name and an optional JSON payload; `jobs` filters with +`--status`, `--queue`, and `--limit`. + +## Serving the dashboard + +```bash +taskito --url taskito.db dashboard --port 8080 --token "$DASH_TOKEN" +``` + +Serves the bundled [dashboard](/java/guides/operations/dashboard) until +interrupted. `--static ` overrides the SPA assets; `--token` gates the +API. + +## Other backends + +```bash +taskito --backend postgres --url "$DATABASE_URL" stats +taskito --backend redis --url redis://localhost dlq list +``` + +`--url` is required for every backend except SQLite. + + + The CLI is a class in the main jar, not a separate binary — put the jar (and + its dependencies) on the classpath, or ship a small launcher script like the + alias above. There is no worker subcommand: workers are code, built with + `taskito.worker()` in your application. + diff --git a/docs/content/docs/java/guides/operations/dashboard.mdx b/docs/content/docs/java/guides/operations/dashboard.mdx new file mode 100644 index 00000000..3358b48a --- /dev/null +++ b/docs/content/docs/java/guides/operations/dashboard.mdx @@ -0,0 +1,66 @@ +--- +title: Dashboard +description: "Serve the bundled web dashboard and its REST API from a Java process." +--- + +The jar bundles the web dashboard SPA; `DashboardServer` serves it plus a JSON +REST API over the queue — no separate service, no asset build step. + +```java +try (Taskito taskito = Taskito.builder().sqlite("taskito.db").open(); + DashboardServer server = DashboardServer.start(taskito, 8080)) { + System.out.println("dashboard on http://localhost:" + server.port()); + // ... +} +``` + +Or from the [CLI](/java/guides/operations/cli): + +```bash +taskito --url taskito.db dashboard --port 8080 +``` + +Pass `0` as the port for an ephemeral one (`server.port()` reports what was +bound). The SPA is extracted from the jar to a per-user, content-addressed +directory on first use; `-Dtaskito.dashboard.dir=/path` (or the `staticDir` +argument) overrides it with an unpacked build. Without bundled assets only +`/api/*` responds. + +## REST API + +Everything is JSON, fields in snake_case, timestamps in Unix milliseconds. + +| Method · Path | Effect | +|---|---| +| `GET /api/stats` | Counts by status across all queues. | +| `GET /api/stats/queues` | Counts per queue. | +| `GET /api/queues/paused` | Names of paused queues. | +| `GET /api/jobs` | Job list — `?status=&queue=&task=&limit=&offset=`. | +| `GET /api/jobs/{id}` | A single job. | +| `GET /api/dead-letters` | Dead-letter entries — `?limit=&offset=`. | +| `GET /api/metrics` | Per-execution metrics — `?task=&since=` (ms window). | +| `GET /api/workers` | Registered workers + heartbeats. | +| `GET /api/auth/status` | `{ "auth_required": true\|false }` — never needs a token. | +| `POST /api/jobs/{id}/cancel` | Cancel a job. | +| `POST /api/dead-letters/{id}/retry` | Re-enqueue a dead-letter entry. | +| `POST /api/queues/{name}/pause` · `/resume` | Pause / resume a queue. | + +## Auth + +Auth runs **open** by default. Pass a token to require it on every `/api/*` +request (except `/api/auth/status`): + +```java +DashboardServer.start(taskito, 8080, System.getenv("DASH_TOKEN")); +``` + +Requests authenticate with `?token=`; opening `/?token=` once +sets an httpOnly `taskito_token` cookie so the SPA works for the rest of the +session. This is a single shared token — no per-user login, RBAC, or SSO. For +those, put the server behind a reverse proxy that handles auth. + + + Open mode means anyone who can reach the port has full control. Bind it to + localhost, front it with your own auth, or at minimum set a token — see + [Security](/java/guides/operations/security). + diff --git a/docs/content/docs/java/guides/operations/deployment.mdx b/docs/content/docs/java/guides/operations/deployment.mdx new file mode 100644 index 00000000..10bd9fdf --- /dev/null +++ b/docs/content/docs/java/guides/operations/deployment.mdx @@ -0,0 +1,88 @@ +--- +title: Deployment +description: "Process model, native library packaging, and graceful shutdown in production." +--- + +## Process model + +Producers and workers are separate processes that share storage. A typical +deploy: + +- **Web / API process** — opens a `Taskito` and calls `enqueue`. +- **Worker process(es)** — builds a worker over the same backend, registers + handlers, and blocks: + +```java +try (Taskito taskito = Taskito.builder().postgres(System.getenv("DATABASE_URL")).open(); + Worker worker = taskito.worker() + .handle(sendEmail, handlers::sendEmail) + .queues("default", "emails") + .concurrency(8) // fixed pool; 0 (default) = cached pool + .start()) { + Runtime.getRuntime().addShutdownHook(new Thread(worker::close)); + worker.awaitShutdown(); +} +``` + +With SQLite, processes must share the file (same host / volume). With +[Postgres or Redis](/java/guides/operations/backends), producers and workers +can run on different machines. + +## Native library + +The jar bundles the native engine per platform +(`linux-x86_64`, `linux-aarch64`, `osx-*`, `windows-*`) and extracts it to a +per-user directory at first use — content-addressed and SHA-256-verified, so +concurrent processes and upgrades are safe. Two system properties matter in +containers: + +- `-Dtaskito.native.workdir=/path` — extraction directory, for hosts where + `/tmp` is `noexec`. +- `-Dtaskito.native.lib=/path/to/library` — skip extraction entirely and load + an explicit binary (e.g. one you built for musl). + +Standard glibc-based images (`eclipse-temurin`, `debian`, `ubuntu`) work out +of the box. + +## Scaling + +Run more worker processes (or hosts) against one store to scale throughput — +atomic claims keep jobs from double-running. Within a process, size the pool +with `concurrency(n)` or let it breathe with +[autoscale](/java/guides/operations/autoscaling); across processes, scale +replicas with [KEDA](/java/guides/operations/autoscaling); for locality across +many workers, enable the [mesh](/java/guides/operations/mesh). + +## Producer-side batching + +High-volume producers should enqueue in batches — `enqueueMany(task, payloads)` +writes a whole batch in one storage call. When payloads arrive one at a time, +`Batcher` coalesces them: it buffers until `maxBatch` items or `maxDelay` +elapses, then flushes as one `enqueueMany`: + +```java +try (Batcher batcher = Batcher.of(taskito, ingest, 100, Duration.ofMillis(50))) { + events.forEach(batcher::add); // add() returns job ids when it triggers a flush +} // close() flushes what remains +``` + +## Graceful shutdown + +`worker.close()` stops dispatching, drains in-flight handlers (up to 30 +seconds, then interrupts and waits 30 more), then frees the native worker — +wire it to a shutdown hook as above. `stop()` alone stops dispatching without +tearing down. + +## Performance: the FFM fast path + +On JDK 22+ the hot byte operations automatically use a Java FFM (Panama) fast +path packaged as a multi-release-jar overlay; older JDKs transparently fall +back to JNI — same API either way. FFM calls are "restricted native access": +add `--enable-native-access=ALL-UNNAMED` to your JVM flags to grant access and +silence the warning (future JDKs will deny it by default). + + + Always set a timeout on production tasks — a crashed worker's `RUNNING` jobs + are reaped and retried once their timeout elapses. See the + [failure model](/architecture/failure-model). + diff --git a/docs/content/docs/java/guides/operations/graalvm.mdx b/docs/content/docs/java/guides/operations/graalvm.mdx new file mode 100644 index 00000000..eefdc996 --- /dev/null +++ b/docs/content/docs/java/guides/operations/graalvm.mdx @@ -0,0 +1,57 @@ +--- +title: GraalVM Native Image +description: "Compile taskito applications to a native binary with GraalVM." +--- + +The SDK is built to stay native-image friendly, and CI proves it: every change +compiles a smoke application into a GraalVM native image (with +`--no-fallback`, so missing metadata fails the build) and runs the full +enqueue → execute → result path in the compiled binary. + +Two design choices do the heavy lifting: + +- **No runtime reflection for handlers** — the `taskito-processor` annotation + processor generates task bindings at compile time, so `@TaskHandler` wiring + needs no reflection metadata at all. +- **A narrow native seam** — the Rust engine is reached through a small, fixed + JNI surface, and the model classes are plain Jackson DTOs. + +## Building your app + +Use the standard GraalVM Gradle plugin, and generate reachability metadata for +your application with the tracing agent — it records the JNI and Jackson +paths your code actually exercises: + +```bash +./gradlew -Pagent run # run your app (or its tests) under the tracing agent +./gradlew metadataCopy # copy recorded metadata into src/main/resources/META-INF/native-image +./gradlew nativeCompile # build the native binary +``` + +```kotlin +graalvmNative { + binaries.named("main") { + buildArgs.add("--no-fallback") // fail on missing metadata, don't emit a fallback image + } + agent { + metadataCopy { + inputTaskNames.add("run") + outputDirectories.add("src/main/resources/META-INF/native-image") + mergeWithExisting.set(true) + } + } +} +``` + +Exercise every taskito path your app uses under the agent — enqueue, worker +execution, and any inspection APIs whose DTOs must deserialize — so the +recorded metadata is complete. The bundled native library still extracts and +loads at runtime exactly as on the JVM, so the +[deployment](/java/guides/operations/deployment) properties +(`taskito.native.workdir`, `taskito.native.lib`) apply to native binaries too. + + + The repository's `graalvm-smoke` module is a minimal, working reference: + a task, a worker, an event listener, periodic registration, and dead-letter + listing, compiled and executed as a native image in CI. + diff --git a/docs/content/docs/java/guides/operations/index.mdx b/docs/content/docs/java/guides/operations/index.mdx new file mode 100644 index 00000000..ca36cbf2 --- /dev/null +++ b/docs/content/docs/java/guides/operations/index.mdx @@ -0,0 +1,20 @@ +--- +title: Operations +description: "Run the Java SDK reliably in production — deployment, scaling, inspection." +--- + +Run taskito reliably in production. + +| Guide | Description | +|---|---| +| [Backends](/java/guides/operations/backends) | SQLite, Postgres, and Redis — setup and trade-offs | +| [Job Management](/java/guides/operations/inspection) | Inspect, cancel, replay, and clean up jobs | +| [Dashboard](/java/guides/operations/dashboard) | Serve the bundled web dashboard and its REST API | +| [Mesh Scheduling](/java/guides/operations/mesh) | Decentralized work-stealing across worker nodes | +| [Autoscaling](/java/guides/operations/autoscaling) | In-process thread autoscaling and the KEDA scaler endpoint | +| [CLI](/java/guides/operations/cli) | Operate the queue from the terminal | +| [Testing](/java/guides/operations/testing) | The in-memory backend and integration patterns | +| [Security](/java/guides/operations/security) | Signing, encryption, and dashboard auth | +| [Troubleshooting](/java/guides/operations/troubleshooting) | Diagnose stuck jobs and worker issues | +| [Deployment](/java/guides/operations/deployment) | Process model, packaging, and graceful shutdown | +| [GraalVM Native Image](/java/guides/operations/graalvm) | Compile applications to a native binary | diff --git a/docs/content/docs/java/guides/operations/inspection.mdx b/docs/content/docs/java/guides/operations/inspection.mdx new file mode 100644 index 00000000..c02658b5 --- /dev/null +++ b/docs/content/docs/java/guides/operations/inspection.mdx @@ -0,0 +1,60 @@ +--- +title: Job Management +description: "List and inspect jobs, manage the dead-letter queue, and pause queues." +--- + +Read and manage individual jobs directly from the `Taskito` handle. For +queue-wide health — stats, per-execution metrics, workers — see +[Monitoring](/java/guides/observability/monitoring). + +## Jobs + +```java +List failed = taskito.listJobs( + JobFilter.builder().status(JobStatus.FAILED).limit(50).build()); +taskito.getJob(id); // Optional +taskito.jobErrors(id); // per-attempt error history +taskito.cancel(id); // cancel a pending job +``` + +`JobFilter.builder()` also filters by `queue(name)` and `task(name)` and pages +with `offset(n)`. + +## Dead letters + +```java +taskito.listDead(50, 0); // newest first: limit, offset +taskito.listDeadByTask("send_email", 50, 0); +taskito.retryDead(deadId); // re-enqueue; returns the new job id +taskito.deleteDead(deadId); +taskito.purgeDead(olderThanMs); +taskito.purgeDeadByTask("send_email"); +taskito.purgeCompleted(olderThanMs); +``` + +## Pause / resume + +Per-queue operations live on the `Queue` handle from `taskito.queue(name)`; +the global view stays on the client: + +```java +taskito.queue("default").pause(); +taskito.queue("default").resume(); +taskito.queue("default").isPaused(); +taskito.listPausedQueues(); +``` + +## Periodic tasks + +```java +taskito.registerPeriodic(PeriodicTask.builder("nightly", "report", "0 0 0 * * *").build()); +taskito.listPeriodic(); // every registered task, enabled or paused +taskito.pausePeriodic("nightly"); +taskito.resumePeriodic("nightly"); +taskito.deletePeriodic("nightly"); +``` + + + The same data drives the [dashboard](/java/guides/operations/dashboard) and + the [CLI](/java/guides/operations/cli) read commands. + diff --git a/docs/content/docs/java/guides/operations/mesh.mdx b/docs/content/docs/java/guides/operations/mesh.mdx new file mode 100644 index 00000000..55be0c14 --- /dev/null +++ b/docs/content/docs/java/guides/operations/mesh.mdx @@ -0,0 +1,39 @@ +--- +title: Mesh Scheduling +description: "Form a work-stealing overlay so idle workers pull from busy peers." +--- + +Workers can form a decentralized **mesh** — SWIM gossip for peer discovery, +consistent-hash placement, and TCP work-stealing — so idle nodes pull work from +busy ones. The database stays the source of truth; the mesh only optimizes +dispatch locality, so it is safe to add to any worker. + +```java +Worker worker = taskito.worker() + .handle(task, handler) + .mesh(MeshOptions.builder() + .port(7946) // UDP gossip; TCP steal binds port + 1 + .seed("10.0.0.2:7946") // peers to join (none = standalone) + .encryptionKey(System.getenv("MESH_KEY")) // optional XOR-obfuscated gossip + .build()) + .start(); +``` + +Other builder tunables (with defaults): `bindAddr` (`0.0.0.0`), +`advertiseAddr` (required behind NAT / when binding `0.0.0.0` across hosts), +`enableStealing` (`true`), `affinityWeight` (`0.7`), `localBufferCapacity` +(`64`), `maxStealBatch` (`4`), `stealThreshold` (`2`), `virtualNodes` (`150`), +`stealRateLimit` (`10`/s, `0` = unlimited). + +Inspect the cluster from a running worker: + +```java +worker.meshClusterInfo(); // Optional — empty unless started with mesh(...) +``` + + + Mesh is an optimization, never a correctness dependency — atomic claims in + storage still guarantee exactly-once dispatch. See + [Mesh Scheduling](/architecture/mesh) for the SWIM/ring/work-stealing + internals (shared engine). + diff --git a/docs/content/docs/java/guides/operations/meta.json b/docs/content/docs/java/guides/operations/meta.json new file mode 100644 index 00000000..dbf70c5b --- /dev/null +++ b/docs/content/docs/java/guides/operations/meta.json @@ -0,0 +1 @@ +{ "title": "Operations", "pages": ["backends", "inspection", "dashboard", "mesh", "autoscaling", "cli", "testing", "security", "troubleshooting", "deployment", "graalvm"] } diff --git a/docs/content/docs/java/guides/operations/security.mdx b/docs/content/docs/java/guides/operations/security.mdx new file mode 100644 index 00000000..0fe40684 --- /dev/null +++ b/docs/content/docs/java/guides/operations/security.mdx @@ -0,0 +1,82 @@ +--- +title: Security +description: "Trust model, payload exposure, webhook signing, and dashboard access." +--- + +## Trust model: storage is code execution + +A worker runs whatever task name a job names, with whatever payload it carries. +So **anyone who can write to the storage can run code on your workers.** Treat +the database / Redis credentials as code-execution credentials: + +- Don't expose the storage to untrusted clients — enqueue through your own + authenticated service, not directly from a browser. +- Use least-privilege storage credentials and a private network. + +## Payloads are not encrypted + +Task arguments and results are serialized with [JSON or +MessagePack](/java/guides/extensibility/serializers) and stored as plain bytes +— by default **not encrypted.** Anyone who can read the storage can read them. + +- Don't put secrets (tokens, PII) directly in task payloads. Pass an id and + fetch the secret inside the handler — for example via a + [resource](/java/guides/resources). +- To detect tampering, sign payloads — `HmacCodec` in a codec chain, or + `SignedSerializer` around the serializer. Workers reject anything altered or + forged (integrity, not confidentiality). +- To hide payloads at rest, encrypt them — `AesGcmCodec` or + `EncryptedSerializer` (AES-GCM: confidentiality **and** integrity). Share + the key across producers and workers. + +## Webhooks + +Webhook deliveries are **HMAC-SHA256 signed** when the subscription has a +secret; the signature rides in `X-Taskito-Signature: sha256=`. Verify it +on the receiver — HMAC the raw body with the shared secret and compare in +constant time — before trusting the body. + +The deliverer POSTs the configured URL directly — there is **no SSRF +allowlist.** Only register webhook URLs you control, and validate them before +storing if they come from users. + +## Proxies + +Proxy references are HMAC-signed with the key you pass to +`new Proxies(hmacKey)` and verified on every resolve, so a forged or altered +reference is rejected. `FileProxyHandler` takes an allowlist of root +directories and refuses paths that resolve outside them — an **empty allowlist +permits any path**, so always set roots in production. + +## Dashboard + +The [dashboard](/java/guides/operations/dashboard) runs in **open mode** by +default — anyone who reaches the port has full control. Start it with a token +to gate `/api/*`, bind it to localhost, or front it with a reverse proxy that +enforces your own auth (login / RBAC / SSO). + +## Mesh gossip + +The mesh `encryptionKey` XOR-obfuscates gossip datagrams — it deters casual +sniffing only and is **not cryptographic protection.** Run the mesh on a +trusted private network. + +## Native library loading + +The bundled native library is extracted to a per-user, owner-only +(`rwx------`) directory and verified by SHA-256 before every load — a swapped +or symlinked file in a shared temp directory is rejected. Point +`-Dtaskito.native.workdir` at a private, exec-permitted directory on hardened +hosts with a `noexec` `/tmp`. + +## Hardening checklist + +- [ ] Storage reachable only from trusted services, on a private network. +- [ ] Enqueue behind your own authn/authz — never expose storage to clients. +- [ ] No secrets in task payloads; fetch them inside the handler. +- [ ] Payloads signed or encrypted where the storage host isn't fully trusted. +- [ ] Webhook receivers verify the HMAC signature; webhook URLs are trusted. +- [ ] `FileProxyHandler` configured with allowlisted roots. +- [ ] Dashboard bound to localhost, or token-gated behind your own auth. +- [ ] Logs don't echo sensitive payloads (redact in `onEnqueue` + [middleware](/java/guides/extensibility/middleware)). diff --git a/docs/content/docs/java/guides/operations/testing.mdx b/docs/content/docs/java/guides/operations/testing.mdx new file mode 100644 index 00000000..c6b339b5 --- /dev/null +++ b/docs/content/docs/java/guides/operations/testing.mdx @@ -0,0 +1,84 @@ +--- +title: Testing +description: "Unit-test with the in-memory backend, integration-test against real SQLite." +--- + +Handlers are plain functions — unit-test them directly. One level up, the +`taskito-test` artifact provides a **pure-Java in-memory backend**: the full +client API with no native library and no disk, ideal for fast unit tests of +producers, handlers, retries, and dead-lettering. + +```kotlin +testImplementation("org.byteveda:taskito-test:0.18.0") +``` + +```java +import org.byteveda.taskito.test.InMemoryTaskito; + +@Test +void runsATaskEndToEnd() throws Exception { + try (Taskito taskito = InMemoryTaskito.open()) { + String id = taskito.enqueue("add", List.of(2, 3)); + + try (Worker worker = taskito.worker() + .handle("add", List.class, numbers -> + (int) numbers.get(0) + (int) numbers.get(1)) + .start()) { + Job job = taskito.awaitJob(id, Duration.ofSeconds(5)).orElseThrow(); + assertEquals(JobStatus.COMPLETE, job.status); + } + + assertEquals(5, taskito.getResult(id, Integer.class).orElseThrow()); + } +} +``` + +`InMemoryTaskito.open(serializer)` swaps the JSON default. To construct the +backend explicitly (e.g. to share one across a custom builder), pass it to +`Taskito.builder().open(new InMemoryQueueBackend())`. + +## Synchronization + +`awaitJob(id, timeout)` blocks until the job reaches a terminal state and is +the simplest synchronization point — tests rarely need a sleep. Registering an +event listener with a `CountDownLatch` works too: + +```java +CountDownLatch done = new CountDownLatch(1); +Worker worker = taskito.worker() + .handle(echo, payload -> payload.length()) + .on(EventName.SUCCESS, event -> done.countDown()) + .start(); +``` + +## Integration tests + +For coverage of the real storage engine (scheduling, claims, locks), open a +throwaway SQLite database — the whole enqueue → execute → result path runs +in-process: + +```java +Path dir = Files.createTempDirectory("taskito-test"); +try (Taskito taskito = Taskito.builder().sqlite(dir.resolve("q.db").toString()).open()) { + // real backend, no server +} +``` + +Always close (or try-with-resources) workers — a leaked worker keeps polling +across tests. + +## Faking dependencies + +Register a [resource](/java/guides/resources) with a stub in the test — the +worker injects whatever is registered, so no real connection is opened: + +```java +taskito.resource("db", context -> fakeDb); +``` + + + The in-memory backend covers the queue contract, not the native engine: + workflows are not supported in-memory, and storage-engine behavior (SQL + claims, cron timing) is only exercised against a real backend. Keep a few + SQLite-backed integration tests alongside the fast in-memory suite. + diff --git a/docs/content/docs/java/guides/operations/troubleshooting.mdx b/docs/content/docs/java/guides/operations/troubleshooting.mdx new file mode 100644 index 00000000..4f704cce --- /dev/null +++ b/docs/content/docs/java/guides/operations/troubleshooting.mdx @@ -0,0 +1,69 @@ +--- +title: Troubleshooting +description: "Common Java SDK issues and how to diagnose them." +--- + +Turn on debug logs first — they show the worker claiming, running, and settling +jobs: + +```java +TaskitoLogger.setLevel(LogLevel.DEBUG); +// or: TASKITO_LOG_LEVEL=debug java -jar app.jar +``` + +## A job stays `PENDING` and never runs + +- **No worker running** — a worker must be built with `taskito.worker()` and + `start()`ed in a live process. +- **Queue mismatch** — the worker only serves its `queues(...)` (default + `default`); a job enqueued to another queue is ignored. Match them. +- **Queue paused** — check `taskito.listPausedQueues()`; + `taskito.queue(name).resume()`. +- **Producer and worker use different storage** — they must point at the same + URL. A relative SQLite path resolves per working directory. + +## A job fails with `no handler registered for task '...'` + +The worker dequeued a job whose task name has no handler on it. Register the +handler (`handle(name, payloadType, fn)`) on the **worker** process — handler +functions don't travel through the queue, only task names and payloads do. + +## `SQLite database is locked` + +SQLite allows one writer at a time. Under many concurrent workers this +surfaces as lock contention. Options: + +- Keep workers on one machine and moderate concurrency, or +- Move to [Postgres or Redis](/java/guides/operations/backends) for + multi-process / multi-machine deployments. + +## `UnsatisfiedLinkError: no bundled native library for platform '...'` + +The jar has no native binary for this OS/architecture. Build the native +library for the platform and point `-Dtaskito.native.lib=/path/to/library` at +it. On hardened hosts where `/tmp` is mounted `noexec`, extraction succeeds +but loading fails — set `-Dtaskito.native.workdir=/path/on/exec/volume` +instead. + +## Payload deserialization errors + +- **Serializer mismatch** — producers and workers must use the same + [serializer](/java/guides/extensibility/serializers) and codec chain. +- **`no codec registered named '...'`** — the task was enqueued with named + codecs; register the same names with `Taskito.builder().codec(name, codec)` + on the worker. +- **Generic payload types** — a non-Jackson serializer only supports plain + `Class` payloads; use `JsonSerializer`/`MsgpackSerializer` (or a + non-generic payload type) for `TypeReference` payloads. + +## A handler hangs and blocks shutdown + +`close()` drains in-flight handlers for 30 seconds, then interrupts and waits +30 more before closing the native worker — a stuck handler delays shutdown by +up to a minute. Set a task timeout so a stuck attempt is reaped, and keep +handlers interruptible. + + + Inspect failures with `taskito.jobErrors(id)` and `taskito.listDead(50, 0)`; + see [job management](/java/guides/operations/inspection). + From 6c5e4da6eb485d68ad52b24e337c1d3ea6baec18 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Tue, 7 Jul 2026 03:05:12 +0530 Subject: [PATCH 07/16] docs: register java sdk and wire shared pages --- .github/workflows/docs.yml | 2 + docs/app/components/diagrams/arch-stack.tsx | 6 +++ docs/app/components/diagrams/worker-fork.tsx | 28 +++++++++++- .../components/landing/scenario-finder.tsx | 22 +++++++--- docs/app/lib/sdk-registry.ts | 16 ++++++- docs/content/docs/architecture/resources.mdx | 20 +++++++++ .../docs/architecture/serialization.mdx | 32 ++++++++++++-- .../content/docs/architecture/worker-pool.mdx | 14 ++++++ docs/content/docs/java/guides/meta.json | 12 +++++- docs/content/docs/java/meta.json | 2 +- docs/content/docs/resources/comparison.mdx | 6 +-- docs/content/docs/resources/faq.mdx | 43 ++++++++++++++++--- 12 files changed, 180 insertions(+), 23 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7489207f..99cfa4ef 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -52,6 +52,8 @@ jobs: - name: Build env: DOCS_BASE_PATH: /taskito + # Three SDK trees prerender in one pass — the default heap OOMs. + NODE_OPTIONS: --max-old-space-size=8192 run: pnpm build - name: SPA fallback for client-side routes diff --git a/docs/app/components/diagrams/arch-stack.tsx b/docs/app/components/diagrams/arch-stack.tsx index f0f3b1a7..c4fb9a4a 100644 --- a/docs/app/components/diagrams/arch-stack.tsx +++ b/docs/app/components/diagrams/arch-stack.tsx @@ -87,6 +87,11 @@ export function ArchitectureStack() { .task(), .enqueue() } + java={ + <> + Task.of(), enqueue() + + } /> , results, workflows, resources — the surface you write against. @@ -167,6 +172,7 @@ export function ResourcePipeline() { inject: or useResource() } + java={Resources.use()} /> ). Task-scoped resources come from a semaphore pool. diff --git a/docs/app/components/diagrams/worker-fork.tsx b/docs/app/components/diagrams/worker-fork.tsx index d7118053..195f653a 100644 --- a/docs/app/components/diagrams/worker-fork.tsx +++ b/docs/app/components/diagrams/worker-fork.tsx @@ -16,6 +16,7 @@ export function WorkerDispatch() { async def} node={async} + java="handler" />{" "} functions to the async runtime. @@ -23,7 +24,11 @@ export function WorkerDispatch() { routes by task type
- +
@@ -49,6 +54,14 @@ export function WorkerDispatch() { block. } + java={ + <> + Handlers run on a JVM ExecutorService — a + cached pool by default, fixed with{" "} + concurrency(n) — fed by the JNI dispatch + bridge. + + } />
@@ -60,7 +73,11 @@ export function WorkerDispatch() { Async pool
- +
} + java={ + <> + Every job is an executor task; the handler's return value + (or exception) is bridged back into the Rust scheduler as + the job outcome. + + } />
diff --git a/docs/app/components/landing/scenario-finder.tsx b/docs/app/components/landing/scenario-finder.tsx index c3160295..f4d167dc 100644 --- a/docs/app/components/landing/scenario-finder.tsx +++ b/docs/app/components/landing/scenario-finder.tsx @@ -4,17 +4,25 @@ import { RawHtml } from "@/components/ui"; import { type Sdk, useActiveSdk } from "@/hooks"; import { DemoModal, type DemoTarget } from "./demo-modal"; -// Scenario guides are authored as Python paths; most map to Node by swapping the -// SDK segment, but a few pages live under a different slug in the Node tree. -const NODE_GUIDE_OVERRIDES: Record = { - "/python/guides/advanced-execution/streaming": "/node/guides/core/streaming", - "/python/guides/workflows/sagas": "/node/guides/workflows/saga", +// Scenario guides are authored as Python paths; most map to another SDK by +// swapping the SDK segment, but a few pages live under a different slug there. +const GUIDE_OVERRIDES: Record> = { + "/python/guides/advanced-execution/streaming": { + node: "/node/guides/core/streaming", + java: "/java/guides/core/streaming", + }, + "/python/guides/workflows/sagas": { + node: "/node/guides/workflows/saga", + java: "/java/guides/workflows/saga", + }, }; /** Resolve a scenario's Python guide path to the active SDK's docs. */ function guideHref(pyPath: string, sdk: Sdk): string { - if (sdk !== "node") return pyPath; - return NODE_GUIDE_OVERRIDES[pyPath] ?? pyPath.replace("/python/", "/node/"); + if (sdk === "python") return pyPath; + return ( + GUIDE_OVERRIDES[pyPath]?.[sdk] ?? pyPath.replace("/python/", `/${sdk}/`) + ); } interface Bullet { diff --git a/docs/app/lib/sdk-registry.ts b/docs/app/lib/sdk-registry.ts index 0bfd00e3..3ecbd547 100644 --- a/docs/app/lib/sdk-registry.ts +++ b/docs/app/lib/sdk-registry.ts @@ -3,7 +3,7 @@ // SDK-aware docs all derive from here. Don't hardcode "python"/"node" elsewhere. /** Supported SDK ids in display order; also the URL prefix + `data-sdk` value. */ -export const SDK_IDS = ["python", "node"] as const; +export const SDK_IDS = ["python", "node", "java"] as const; export type Sdk = (typeof SDK_IDS)[number]; @@ -53,6 +53,20 @@ export const SDK_PROFILES: Record = { "resources", ], }, + java: { + id: "java", + label: "Java", + language: "Java", + binding: "JNI", + navSections: [ + "java/getting-started", + "java/guides", + "architecture", + "java/api-reference", + "java/more/examples", + "resources", + ], + }, }; export function isSdk(value: string | null | undefined): value is Sdk { diff --git a/docs/content/docs/architecture/resources.mdx b/docs/content/docs/architecture/resources.mdx index 7f3fa310..978bed9d 100644 --- a/docs/content/docs/architecture/resources.mdx +++ b/docs/content/docs/architecture/resources.mdx @@ -48,3 +48,23 @@ the worker before the handler runs. Recipes are optionally HMAC-signed for tamper detection. + + + +**Layer 1 — Registration**: resources are registered once with +`queue.resource(name, factory)` (plus scope and pool-config overloads). A +factory may depend on another resource through the context it receives +(`ctx.use("other")`), and a cycle fails fast instead of recursing. + +**Layer 2 — Worker Resource Runtime**: each worker leases a `ResourceRuntime` +that builds resources on first use and disposes them LIFO at shutdown. Handlers +reach them with `Resources.use("name")` (or generated `@Resource` bindings); the +scope — worker / thread / task / request / pooled — decides each resource's +lifetime, with pooled instances checked out per task and returned at task end. + +**Layer 3 — Resource Proxies**: `ProxyHandler` implementations deconstruct live +objects (file handles and similar) into a serializable reference and +reconstruct them before use. References are HMAC-SHA256-signed, with optional +expiry and purpose binding folded into the signature. + + diff --git a/docs/content/docs/architecture/serialization.mdx b/docs/content/docs/architecture/serialization.mdx index 65d09b38..bbea27e8 100644 --- a/docs/content/docs/architecture/serialization.mdx +++ b/docs/content/docs/architecture/serialization.mdx @@ -4,8 +4,8 @@ description: "Pluggable serializers for task arguments and results, with a per-S --- taskito uses a pluggable serializer for task arguments and results. The default -is CloudpickleSerializer} node={JsonSerializer} />, -which . +is CloudpickleSerializer} node={JsonSerializer} java={JsonSerializer} />, +which . @@ -29,6 +29,21 @@ new Queue({ dbPath: "taskito.db", serializer: new MsgpackSerializer() }); + + +```java +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.serialization.MsgpackSerializer; + +// Use msgpack for compact, binary payloads +Taskito queue = Taskito.builder() + .sqlite("taskito.db") + .serializer(new MsgpackSerializer()) + .open(); +``` + + + ## Built-in serializers @@ -51,9 +66,20 @@ new Queue({ dbPath: "taskito.db", serializer: new MsgpackSerializer() }); + + +| Serializer | Format | Best for | +|---|---|---| +| `JsonSerializer` (default) | JSON (Jackson) | Simple types, cross-language interop, debugging | +| `MsgpackSerializer` | Binary (msgpack) | Compact payloads, smaller storage | +| `SignedSerializer` | Wraps a serializer + HMAC-SHA256 | Tamper detection | +| `EncryptedSerializer` | Wraps a serializer + AES-GCM | Encrypted, authenticated payloads | + + + ## Custom serializers -Implement the `Serializer` interface with dumps(obj) / loads(data)} node={<>serialize(value) / deserialize(bytes)} /> methods. +Implement the `Serializer` interface with dumps(obj) / loads(data)} node={<>serialize(value) / deserialize(bytes)} java={<>serialize(value) / deserialize(bytes, type)} /> methods. ## What gets serialized diff --git a/docs/content/docs/architecture/worker-pool.mdx b/docs/content/docs/architecture/worker-pool.mdx index 15399066..96a6721c 100644 --- a/docs/content/docs/architecture/worker-pool.mdx +++ b/docs/content/docs/architecture/worker-pool.mdx @@ -43,3 +43,17 @@ functions. between concurrently dispatched jobs. + + + +- **JVM executor threads**: handlers run on a `java.util.concurrent` + `ExecutorService` owned by the worker — a cached pool by default, a fixed pool + with `concurrency(n)`, or a resizable pool under `autoscale(...)`. +- **Native dispatch bridge**: the Rust core hands each claimed job to the JVM + through a JNI dispatch bridge; the handler's return value (or exception) is + bridged back into the Rust scheduler as the job outcome. +- **Context isolation**: each invocation gets its own `TaskScope` for resource + resolution, entered and exited around the handler on its executor thread — + thread-confined, so concurrent jobs never share per-task state. + + diff --git a/docs/content/docs/java/guides/meta.json b/docs/content/docs/java/guides/meta.json index f0535762..af02e81d 100644 --- a/docs/content/docs/java/guides/meta.json +++ b/docs/content/docs/java/guides/meta.json @@ -1,5 +1,15 @@ { "title": "Guides", "root": true, - "pages": ["index", "core"] + "pages": [ + "index", + "core", + "reliability", + "workflows", + "extensibility", + "resources", + "integrations", + "operations", + "observability" + ] } diff --git a/docs/content/docs/java/meta.json b/docs/content/docs/java/meta.json index 12f0a7d6..a6fad7aa 100644 --- a/docs/content/docs/java/meta.json +++ b/docs/content/docs/java/meta.json @@ -1 +1 @@ -{ "title": "Java", "pages": ["getting-started", "guides"] } +{ "title": "Java", "pages": ["getting-started", "guides", "api-reference", "more"] } diff --git a/docs/content/docs/resources/comparison.mdx b/docs/content/docs/resources/comparison.mdx index 3bdcd4a8..37ae4c57 100644 --- a/docs/content/docs/resources/comparison.mdx +++ b/docs/content/docs/resources/comparison.mdx @@ -3,7 +3,7 @@ title: Comparison description: "Taskito vs Celery, RQ, Dramatiq, Huey, TaskIQ — feature matrix and decision guide." --- -**TL;DR**: Taskito is without the +**TL;DR**: Taskito is without the broker. Rust scheduler, no Redis/RabbitMQ, lower latency, better concurrency. Start with SQLite, scale to Postgres when needed. @@ -12,7 +12,7 @@ Start with SQLite, scale to Postgres when needed. | Feature | taskito | Celery | RQ | Dramatiq | Huey | TaskIQ | |---|---|---|---|---|---|---| | Broker required | **No** | Redis / RabbitMQ | Redis | Redis / RabbitMQ | Redis | Redis / RabbitMQ / Nats | -| Core language | **Rust + ** | Python | Python | Python | Python | Python | +| Core language | **Rust + ** | Python | Python | Python | Python | Python | | Priority queues | **Yes** | Yes | No | No | Yes | Yes | | Rate limiting | **Yes** | Yes | No | Yes | No | No | | Dead letter queue | **Yes** | No | Yes | No | No | No | @@ -79,7 +79,7 @@ widely adopted. Looking to switch? See the{" "} - Migrating from + Migrating from {" "} guide for a step-by-step walkthrough with side-by-side code examples. diff --git a/docs/content/docs/resources/faq.mdx b/docs/content/docs/resources/faq.mdx index 378f65be..4ff4ff50 100644 --- a/docs/content/docs/resources/faq.mdx +++ b/docs/content/docs/resources/faq.mdx @@ -126,18 +126,19 @@ SQLite database. Additionally: - taskito's scheduler runs in Rust (faster polling, lower overhead) - Worker threads are OS threads managed by Rust, not{" "} - + - No external dependencies beyond{" "} - cloudpickle} node="the native addon" /> + cloudpickle} node="the native addon" java="the bundled native library" /> ## Can I use async tasks? Yes. Define the task function with{" "} -async def} node={async} /> and the +async def} node={async} java="a handler that runs on the worker's executor" /> and the worker dispatches it natively —{" "} : @@ -180,16 +181,34 @@ const stats = queue.stats(); + + +```java +Task> fetchUrls = Task.of("fetchUrls", new TypeReference<>() {}); + +try (Worker worker = queue.worker() + .handle(fetchUrls, urls -> urls.stream().map(App::fetch).toList()) + .concurrency(8) + .start()) { + String id = queue.enqueue(fetchUrls, urls); + List result = queue.awaitJob(id, Duration.ofSeconds(30)) + .flatMap(job -> queue.getResult(id, List.class)) + .orElseThrow(); +} +``` + + + Sync and async tasks can coexist in the same queue. The worker automatically routes each job to the correct pool based on the task type. See the{" "} Async Tasks guide for details including concurrency tuning and{" "} -current_job} node={currentJob()} />{" "} +current_job} node={currentJob()} java="middleware-provided job context" />{" "} context in async tasks. ## What serialization format does taskito use? -By default, CloudpickleSerializer} node={JsonSerializer} /> — — or provide a custom serializer: +By default, CloudpickleSerializer} node={JsonSerializer} java={JsonSerializer} /> — — or provide a custom serializer: @@ -217,6 +236,20 @@ Custom serializers implement the `Serializer` interface with + + +```java +Taskito queue = Taskito.builder() + .sqlite("taskito.db") + .serializer(new MsgpackSerializer()) + .open(); +``` + +Custom serializers implement the `Serializer` interface with +`serialize(value)` and `deserialize(bytes, type)` methods. + + + Regardless of serializer, avoid passing unserializable objects like open file handles, database connections, or thread locks. From 79b16511b5b1bbb387d49ea10311ad094d8491dc Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:21:01 +0530 Subject: [PATCH 08/16] docs(java): address review feedback --- .../docs/java/api-reference/context.mdx | 4 ++- .../docs/java/api-reference/resources.mdx | 4 ++- .../java/getting-started/capabilities.mdx | 2 +- .../docs/java/getting-started/concepts.mdx | 2 +- .../java/getting-started/installation.mdx | 4 +-- .../docs/java/guides/core/execution-model.mdx | 2 +- .../docs/java/guides/core/predicates.mdx | 3 ++- .../docs/java/guides/operations/dashboard.mdx | 7 +++++ .../docs/java/guides/operations/testing.mdx | 2 +- .../java/guides/reliability/guarantees.mdx | 4 +-- .../docs/java/guides/reliability/retries.mdx | 2 +- .../docs/java/guides/resources/proxies.mdx | 4 ++- .../docs/java/guides/workflows/gates.mdx | 4 +-- .../docs/java/guides/workflows/saga.mdx | 4 +++ .../docs/java/more/examples/benchmark.mdx | 8 ++++-- .../docs/java/more/examples/notifications.mdx | 26 ++++++++++++++++--- .../more/examples/predicate-gated-jobs.mdx | 25 +++++++++++++++--- docs/content/docs/resources/comparison.mdx | 2 +- docs/content/docs/resources/faq.mdx | 9 ++++--- 19 files changed, 89 insertions(+), 29 deletions(-) diff --git a/docs/content/docs/java/api-reference/context.mdx b/docs/content/docs/java/api-reference/context.mdx index 5403bcaa..e304f4e6 100644 --- a/docs/content/docs/java/api-reference/context.mdx +++ b/docs/content/docs/java/api-reference/context.mdx @@ -3,7 +3,9 @@ title: Context description: "Resources.use inside handlers and the middleware TaskContext." --- -A handler receives exactly one argument — its deserialized payload. The +A handler's first argument is always its deserialized payload — a lambda +handler takes nothing else, while a `@TaskHandler` method may declare further +`@Resource(...)` parameters that the generated companion injects. The execution context around it is reached two ways: resource resolution inside the handler, and the `TaskContext` handed to middleware. diff --git a/docs/content/docs/java/api-reference/resources.mdx b/docs/content/docs/java/api-reference/resources.mdx index 4f67c8eb..9e6014b1 100644 --- a/docs/content/docs/java/api-reference/resources.mdx +++ b/docs/content/docs/java/api-reference/resources.mdx @@ -30,7 +30,9 @@ taskito.resource("db", PoolConfig.of(10).withMaxLifetime(Duration.ofMinutes(30)) The factory receives a `ResourceContext` — `scope()` plus `use(name)` to depend on other resources. A factory may only depend on same-or-longer-lived -resources (a `WORKER` factory can't use a `TASK` resource). +resources (a `WORKER` factory can't use a `TASK` resource), and a `POOLED` +factory may only use `WORKER` resources — pooled instances outlive any one +task, so a shorter-lived dependency would dangle after its scope ends. ## Scopes diff --git a/docs/content/docs/java/getting-started/capabilities.mdx b/docs/content/docs/java/getting-started/capabilities.mdx index 57e0d812..78849af6 100644 --- a/docs/content/docs/java/getting-started/capabilities.mdx +++ b/docs/content/docs/java/getting-started/capabilities.mdx @@ -54,7 +54,7 @@ core — pick a backend, describe tasks, run workers. | Stats, metrics, progress, worker heartbeats | [Monitoring](/java/guides/observability/monitoring) | | Micrometer Observation / Sentry middleware | [Integrations](/java/guides/integrations) | | React dashboard + REST API | [Dashboard](/java/guides/operations/dashboard) | -| In-process autoscaling + KEDA scaler endpoint | [KEDA](/java/guides/operations/keda) | +| In-process autoscaling + KEDA scaler endpoint | [Autoscaling](/java/guides/operations/autoscaling) | | Standalone CLI | [CLI](/java/guides/operations/cli) | ## Framework integrations diff --git a/docs/content/docs/java/getting-started/concepts.mdx b/docs/content/docs/java/getting-started/concepts.mdx index 57e12ff8..cb945029 100644 --- a/docs/content/docs/java/getting-started/concepts.mdx +++ b/docs/content/docs/java/getting-started/concepts.mdx @@ -10,7 +10,7 @@ storage are all in the shared Rust engine. Five concepts cover the surface. heads={["Producer", "Worker"]} rows={[ { left: "enqueue(add, payload) → job id" }, - { self: "stored: pending → claimed → running" }, + { self: "stored: pending → running" }, { right: "result written to store", rev: true }, { left: "getResult(id, Integer.class) → 5" }, ]} diff --git a/docs/content/docs/java/getting-started/installation.mdx b/docs/content/docs/java/getting-started/installation.mdx index 13e60ba3..bd05b582 100644 --- a/docs/content/docs/java/getting-started/installation.mdx +++ b/docs/content/docs/java/getting-started/installation.mdx @@ -66,8 +66,8 @@ separate install: At runtime the platform binary is extracted (content-addressed, safe under concurrent processes) and loaded. Two system properties tune this: `-Dtaskito.native.lib=/abs/path` loads an explicit library (local development -against a fresh build), and `-Dtaskito.native.workdir=` relocates extraction on -hardened `noexec /tmp` hosts. +against a fresh build), and `-Dtaskito.native.workdir=/var/lib/myapp/taskito-native` +relocates extraction on hardened `noexec /tmp` hosts. On JDK 22+ hot byte operations automatically take a Project Panama (FFM) fast diff --git a/docs/content/docs/java/guides/core/execution-model.mdx b/docs/content/docs/java/guides/core/execution-model.mdx index 899bbe4f..33dd63d5 100644 --- a/docs/content/docs/java/guides/core/execution-model.mdx +++ b/docs/content/docs/java/guides/core/execution-model.mdx @@ -49,7 +49,7 @@ Horizontal scaling is just more worker processes (or machines) pointed at shared Postgres or Redis storage. The core claims each job for a single worker, so adding workers adds throughput without duplicate execution. Within one process, `autoscale` handles bursts; across processes, a queue-depth scaler -endpoint feeds KEDA — see [KEDA](/java/guides/operations/keda). Workers can +endpoint feeds KEDA — see [Autoscaling](/java/guides/operations/autoscaling). Workers can also join a work-stealing [mesh](/java/guides/operations/mesh) with `mesh(options)`. diff --git a/docs/content/docs/java/guides/core/predicates.mdx b/docs/content/docs/java/guides/core/predicates.mdx index 76e65701..b9027cd6 100644 --- a/docs/content/docs/java/guides/core/predicates.mdx +++ b/docs/content/docs/java/guides/core/predicates.mdx @@ -54,7 +54,8 @@ taskito.gate("charge", ctx -> switch (classify(ctx.payload())) { |---|---|---| | `allow()` | Job created. | Job id present. | | `skip(reason)` | Throws `EnqueueSkippedException`. | Returns empty. | -| `defer(delay)` / `deferUntil(instant)` | Job created, scheduled after `delay` (overrides any delay in the options). | Job id present. | +| `defer(delay)` | Job created, scheduled `delay` from now (overrides any delay in the options). | Job id present. | +| `deferUntil(instant)` | Job created, scheduled at the absolute `instant` (overrides any delay in the options). | Job id present. | | `reject(reason)` | Throws `PredicateRejectedException`. | Throws too. | `tryEnqueue` is the gate-aware form — a skip becomes an empty `Optional` diff --git a/docs/content/docs/java/guides/operations/dashboard.mdx b/docs/content/docs/java/guides/operations/dashboard.mdx index 3358b48a..606a3479 100644 --- a/docs/content/docs/java/guides/operations/dashboard.mdx +++ b/docs/content/docs/java/guides/operations/dashboard.mdx @@ -59,6 +59,13 @@ sets an httpOnly `taskito_token` cookie so the SPA works for the rest of the session. This is a single shared token — no per-user login, RBAC, or SSO. For those, put the server behind a reverse proxy that handles auth. + + `?token=` puts the secret in the URL, where it can leak via browser history, + `Referer` headers, and proxy or access logs. Use it only over HTTPS, redact + query strings from logs, and rely on the cookie afterwards — once the first + request sets it, the token never needs to appear in a URL again. + + Open mode means anyone who can reach the port has full control. Bind it to localhost, front it with your own auth, or at minimum set a token — see diff --git a/docs/content/docs/java/guides/operations/testing.mdx b/docs/content/docs/java/guides/operations/testing.mdx index c6b339b5..9ccfc2c5 100644 --- a/docs/content/docs/java/guides/operations/testing.mdx +++ b/docs/content/docs/java/guides/operations/testing.mdx @@ -46,7 +46,7 @@ event listener with a `CountDownLatch` works too: ```java CountDownLatch done = new CountDownLatch(1); Worker worker = taskito.worker() - .handle(echo, payload -> payload.length()) + .handle("echo", String.class, payload -> payload.length()) .on(EventName.SUCCESS, event -> done.countDown()) .start(); ``` diff --git a/docs/content/docs/java/guides/reliability/guarantees.mdx b/docs/content/docs/java/guides/reliability/guarantees.mdx index 2d9a7666..33582282 100644 --- a/docs/content/docs/java/guides/reliability/guarantees.mdx +++ b/docs/content/docs/java/guides/reliability/guarantees.mdx @@ -3,8 +3,8 @@ title: Delivery Guarantees description: "At-least-once execution, job claiming, and staying correct under retries." --- -Taskito gives **at-least-once** execution: every enqueued job runs to completion -at least once. A job can also run *more* than once — a worker that crashes +Taskito gives **at-least-once** execution: every enqueued job is attempted at +least once. A job can also run *more* than once — a worker that crashes mid-task (after claiming it, before recording the result) leaves the job to be picked up again. Design handlers to tolerate that. diff --git a/docs/content/docs/java/guides/reliability/retries.mdx b/docs/content/docs/java/guides/reliability/retries.mdx index ca019cee..be6fd395 100644 --- a/docs/content/docs/java/guides/reliability/retries.mdx +++ b/docs/content/docs/java/guides/reliability/retries.mdx @@ -34,7 +34,7 @@ fires: ```java Worker worker = queue.worker() .handle(CHARGE, this::charge) - .on(EventName.RETRY, event -> log.info("retrying {0}", event.taskName)) + .on(EventName.RETRY, event -> log.info("retrying {}", event.taskName)) .on(EventName.DEAD, event -> alertOps(event)) .start(); ``` diff --git a/docs/content/docs/java/guides/resources/proxies.mdx b/docs/content/docs/java/guides/resources/proxies.mdx index eeef610c..e1bc44d4 100644 --- a/docs/content/docs/java/guides/resources/proxies.mdx +++ b/docs/content/docs/java/guides/resources/proxies.mdx @@ -13,7 +13,9 @@ This is explicit — you call `deconstruct` and `reconstruct` yourself, and a `ProxyHandler` per resource type does the (de)construction: ```java -byte[] key = System.getenv("TASKITO_PROXY_KEY").getBytes(StandardCharsets.UTF_8); +String secret = Objects.requireNonNull( + System.getenv("TASKITO_PROXY_KEY"), "TASKITO_PROXY_KEY is not set"); +byte[] key = secret.getBytes(StandardCharsets.UTF_8); Proxies proxies = new Proxies(key).register(new FileProxyHandler()); // Producer: reduce the file to a signed ref and enqueue it diff --git a/docs/content/docs/java/guides/workflows/gates.mdx b/docs/content/docs/java/guides/workflows/gates.mdx index 674c4205..a9b6ac81 100644 --- a/docs/content/docs/java/guides/workflows/gates.mdx +++ b/docs/content/docs/java/guides/workflows/gates.mdx @@ -63,10 +63,10 @@ either method on a worker built without `trackWorkflows()` throws a ```java // Approve — the gate completes and downstream steps are enqueued. -worker.approveGate(runId, "approve-deploy"); +worker.approveGate(run.runId(), "approve-deploy"); // Reject with a reason — downstream steps are skipped, run transitions to FAILED. -worker.rejectGate(runId, "approve-deploy", "Artifacts failed QA review."); +worker.rejectGate(run.runId(), "approve-deploy", "Artifacts failed QA review."); ``` ## Timeout behaviour diff --git a/docs/content/docs/java/guides/workflows/saga.mdx b/docs/content/docs/java/guides/workflows/saga.mdx index 517d36a5..6e309bae 100644 --- a/docs/content/docs/java/guides/workflows/saga.mdx +++ b/docs/content/docs/java/guides/workflows/saga.mdx @@ -17,6 +17,8 @@ Task refund = Task.of("refundPayment", Integer.class); Task ship = Task.of("shipOrder", Integer.class); // No compensator for ship — if it never succeeded, there's nothing to undo. +int orderId = 4212; // the workflow input every step receives + Workflow checkout = Workflow.named("checkout") .step(Step.of("reserve", reserve, orderId).compensate(unreserve).build()) .step(Step.of("charge", charge, orderId).after("reserve").compensate(refund).build()) @@ -134,6 +136,8 @@ the run settles toward failure); once the run is failed, the tracker then rolls back the completed compensable steps: ```java +int orderId = 4212; // the workflow input every step receives + Workflow safeCheckout = Workflow.named("safe-checkout") .step(Step.of("reserve", reserve, orderId).compensate(unreserve).build()) .step(Step.of("charge", charge, orderId).after("reserve").compensate(refund).build()) diff --git a/docs/content/docs/java/more/examples/benchmark.mdx b/docs/content/docs/java/more/examples/benchmark.mdx index 1d6b4eb3..7b7ded00 100644 --- a/docs/content/docs/java/more/examples/benchmark.mdx +++ b/docs/content/docs/java/more/examples/benchmark.mdx @@ -11,6 +11,8 @@ a queue. ## Benchmark.java ```java +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import org.byteveda.taskito.Taskito; @@ -24,8 +26,10 @@ public final class Benchmark { private static final int N = 50_000; private static final Task NOOP = Task.of("noop", Integer.class); - public static void main(String[] args) throws InterruptedException { - try (Taskito taskito = Taskito.builder().sqlite("bench.db").open()) { + public static void main(String[] args) throws Exception { + // Fresh database per run — a reused file skews stats with old rows. + Path dir = Files.createTempDirectory("taskito-bench"); + try (Taskito taskito = Taskito.builder().sqlite(dir.resolve("bench.db").toString()).open()) { // 1. Enqueue throughput — stage N jobs in batches of 1,000. long t0 = System.currentTimeMillis(); diff --git a/docs/content/docs/java/more/examples/notifications.mdx b/docs/content/docs/java/more/examples/notifications.mdx index fde7b38a..07f02c28 100644 --- a/docs/content/docs/java/more/examples/notifications.mdx +++ b/docs/content/docs/java/more/examples/notifications.mdx @@ -107,6 +107,7 @@ The worker runs the channels and registers the periodic digest at `08:00` America/New_York time. ```java +import java.util.List; import org.byteveda.taskito.Taskito; import org.byteveda.taskito.scheduling.PeriodicTask; import org.byteveda.taskito.worker.Worker; @@ -119,15 +120,34 @@ public final class WorkerMain { .build()); try (Worker worker = taskito.worker() - .handle(Tasks.SEND_EMAIL, email -> emailProvider.send(email)) - .handle(Tasks.SEND_SMS, sms -> smsProvider.send(sms)) - .handle(Tasks.SEND_DIGEST, ignored -> buildAndFanOutDigests()) + .handle(Tasks.SEND_EMAIL, WorkerMain::deliverEmail) + .handle(Tasks.SEND_SMS, WorkerMain::deliverSms) + .handle(Tasks.SEND_DIGEST, ignored -> buildAndFanOutDigests(taskito)) .queues("default") .start()) { worker.awaitShutdown(); } } } + + private static Void deliverEmail(Tasks.Email email) { + // Call your email provider here. + System.out.println("email -> " + email.to()); + return null; + } + + private static Void deliverSms(Tasks.Sms sms) { + // Call your SMS provider here. + System.out.println("sms -> " + sms.to()); + return null; + } + + private static Void buildAndFanOutDigests(Taskito taskito) { + // Build the due digest emails from your data store, then fan out. + List due = List.of(); + taskito.enqueueMany(Tasks.SEND_EMAIL, due); + return null; + } } ``` diff --git a/docs/content/docs/java/more/examples/predicate-gated-jobs.mdx b/docs/content/docs/java/more/examples/predicate-gated-jobs.mdx index ce8e5750..ef2f1853 100644 --- a/docs/content/docs/java/more/examples/predicate-gated-jobs.mdx +++ b/docs/content/docs/java/more/examples/predicate-gated-jobs.mdx @@ -21,16 +21,24 @@ with `Predicates.allOf` / `anyOf` / `not`; `Recipes` ships the ready-made time-window and feature-flag gates. ```java +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import org.byteveda.taskito.predicates.Predicate; public final class Policies { + // Stand-ins — replace with your usage store and feature-flag client. + private static final Map EXPORTS_USED = new ConcurrentHashMap<>(); + private static final int EXPORT_LIMIT = 100; + private static final Set ENABLED_FLAGS = Set.of("expensive-reindex"); + public static final Predicate UNDER_QUOTA = context -> { String tenantId = (String) context.payload(); - return usage.exports(tenantId) < usage.limit(tenantId); + return EXPORTS_USED.getOrDefault(tenantId, 0) < EXPORT_LIMIT; }; public static final Predicate FLAG_ON = - context -> flags.isOn("expensive-reindex"); + context -> ENABLED_FLAGS.contains("expensive-reindex"); private Policies() {} } @@ -43,23 +51,32 @@ decision gate (allow / skip / defer / reject). Multiple gates on one task run in registration order and the first non-allow decision wins. ```java +import java.time.LocalTime; import java.time.ZoneId; import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.predicates.Predicate; import org.byteveda.taskito.predicates.Predicates; import org.byteveda.taskito.predicates.Recipes; Taskito taskito = Taskito.builder().sqlite("gated.db").open(); +ZoneId newYork = ZoneId.of("America/New_York"); // Out-of-hours promos are *deferred* to the next opening, not dropped. -taskito.gate("send_promo", Recipes.businessHours(ZoneId.of("America/New_York"))); +taskito.gate("send_promo", Recipes.businessHours(newYork)); // Re-index only while its flag is on; otherwise skip silently. taskito.predicate("reindex", Policies.FLAG_ON); +// Stand-in for your own calendar logic. +Predicate businessHours = context -> { + int hour = LocalTime.now(newYork).getHour(); + return hour >= 9 && hour < 17; +}; + // Exports: under quota AND outside business hours (off-peak only). taskito.predicate("export_data", Predicates.allOf( Policies.UNDER_QUOTA, - Predicates.not(context -> isBusinessHours()))); + Predicates.not(businessHours))); ``` ## Enqueuing against a gate diff --git a/docs/content/docs/resources/comparison.mdx b/docs/content/docs/resources/comparison.mdx index 37ae4c57..64b4c8b1 100644 --- a/docs/content/docs/resources/comparison.mdx +++ b/docs/content/docs/resources/comparison.mdx @@ -3,7 +3,7 @@ title: Comparison description: "Taskito vs Celery, RQ, Dramatiq, Huey, TaskIQ — feature matrix and decision guide." --- -**TL;DR**: Taskito is without the +**TL;DR**: Taskito is without the broker. Rust scheduler, no Redis/RabbitMQ, lower latency, better concurrency. Start with SQLite, scale to Postgres when needed. diff --git a/docs/content/docs/resources/faq.mdx b/docs/content/docs/resources/faq.mdx index 4ff4ff50..3dbddb28 100644 --- a/docs/content/docs/resources/faq.mdx +++ b/docs/content/docs/resources/faq.mdx @@ -185,15 +185,16 @@ const stats = queue.stats(); ```java Task> fetchUrls = Task.of("fetchUrls", new TypeReference<>() {}); +List urls = List.of("https://example.com/a", "https://example.com/b"); try (Worker worker = queue.worker() - .handle(fetchUrls, urls -> urls.stream().map(App::fetch).toList()) + .handle(fetchUrls, batch -> batch.stream().map(App::fetch).toList()) .concurrency(8) .start()) { String id = queue.enqueue(fetchUrls, urls); - List result = queue.awaitJob(id, Duration.ofSeconds(30)) - .flatMap(job -> queue.getResult(id, List.class)) - .orElseThrow(); + queue.awaitJob(id, Duration.ofSeconds(30)); + // getResult takes a Class token, so read the JSON array as String[]. + String[] result = queue.getResult(id, String[].class).orElseThrow(); } ``` From ce080070463930f47038965a77be76d20b2edc28 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:00:57 +0530 Subject: [PATCH 09/16] docs: move SDK switcher to topbar dropdown --- docs/app/components/docs/sidebar.tsx | 53 +--------- docs/app/components/ui/site-nav.tsx | 139 ++++++++++++++++++++++++++- docs/app/styles/docs.css | 109 +++++++++++++++------ 3 files changed, 220 insertions(+), 81 deletions(-) diff --git a/docs/app/components/docs/sidebar.tsx b/docs/app/components/docs/sidebar.tsx index a59d7261..f267724c 100644 --- a/docs/app/components/docs/sidebar.tsx +++ b/docs/app/components/docs/sidebar.tsx @@ -1,14 +1,7 @@ import { useEffect, useState } from "react"; -import { Link, useLocation, useNavigate } from "react-router"; -import { useActiveSdk, useSdk } from "@/hooks"; -import { - forcedSdkForPath, - type NavNode, - navForSdk, - type Sdk, - sdkLabels, - sdkSwitchTarget, -} from "@/lib"; +import { Link, useLocation } from "react-router"; +import { useActiveSdk } from "@/hooks"; +import { type NavNode, navForSdk } from "@/lib"; function containsHref(node: NavNode, current: string): boolean { return ( @@ -17,45 +10,6 @@ function containsHref(node: NavNode, current: string): boolean { ); } -// Switcher options come from the SDK registry, so a new language appears here -// automatically. -const SDK_LABELS = sdkLabels(); - -/** Global SDK toggle. Sets the shared store (flips inline variants + this nav); - * on an SDK-specific page it also navigates to the counterpart page, on a shared - * page it stays put. A labelled control — it switches the whole page, not a panel. */ -function SdkSwitch({ sdk, current }: { sdk: Sdk; current: string }) { - const { setSdk } = useSdk(); - const navigate = useNavigate(); - - function select(target: Sdk) { - if (target === sdk) { - return; - } - setSdk(target); - if (forcedSdkForPath(current)) { - navigate(sdkSwitchTarget(current, target)); - } - } - - return ( -
- {SDK_LABELS.map(({ id, label }) => ( - - ))} -
- ); -} - function Caret({ open, onToggle, @@ -228,7 +182,6 @@ export function Sidebar({ K -