Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
ba20205
docs(java): workflow guides
pratyush618 Jul 6, 2026
fba3d54
docs(java): getting started and core guides
pratyush618 Jul 6, 2026
e020139
docs(java): reliability and resources guides
pratyush618 Jul 6, 2026
3bf80a6
docs(java): api reference
pratyush618 Jul 6, 2026
0ced733
docs(java): examples
pratyush618 Jul 6, 2026
8cea7ac
docs(java): operations and extensibility guides
pratyush618 Jul 6, 2026
0fe96e0
Merge branch 'docs/java-foundation' into docs/java-sdk
pratyush618 Jul 6, 2026
4a94c90
Merge branch 'docs/java-reliability' into docs/java-sdk
pratyush618 Jul 6, 2026
3111cc2
Merge branch 'docs/java-workflows' into docs/java-sdk
pratyush618 Jul 6, 2026
599d40d
Merge branch 'docs/java-operations' into docs/java-sdk
pratyush618 Jul 6, 2026
9628ea7
Merge branch 'docs/java-reference' into docs/java-sdk
pratyush618 Jul 6, 2026
6c5e4da
docs: register java sdk and wire shared pages
pratyush618 Jul 6, 2026
79b1651
docs(java): address review feedback
pratyush618 Jul 7, 2026
ce08007
docs: move SDK switcher to topbar dropdown
pratyush618 Jul 7, 2026
66694b7
docs: add Java SDK hero pane and highlighter
pratyush618 Jul 7, 2026
2af3871
docs: fix callout paragraph spacing
pratyush618 Jul 7, 2026
a4f2190
docs: document Java SDK in ARCHITECTURE.md
pratyush618 Jul 7, 2026
18745c5
docs(java): add api-reference pages
pratyush618 Jul 7, 2026
6ee28c3
docs(java): add resources config/observability/testing guides
pratyush618 Jul 7, 2026
076ce60
docs(java): add core batching guide
pratyush618 Jul 7, 2026
4194b8e
docs(java): add workflows building guide
pratyush618 Jul 7, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ jobs:
- name: Build
env:
DOCS_BASE_PATH: /taskito
# Three SDK trees prerender in one pass — the default heap OOMs.
NODE_OPTIONS: --max-old-space-size=8192
run: pnpm build

- name: SPA fallback for client-side routes
Expand Down
89 changes: 55 additions & 34 deletions ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,31 +21,33 @@ It is deliberately *not* a tutorial on running Taskito (see the

Taskito is a polyglot system: **one Rust engine underneath, a thin language SDK shell
on top per host language** — Python via [PyO3](https://pyo3.rs), Node via
[napi-rs](https://napi.rs). Each shell owns ergonomics and extensibility for its
[napi-rs](https://napi.rs), Java via [JNI](https://github.com/jni-rs/jni-rs) (with an
FFM fast-path on JDK 22+). Each shell owns ergonomics and extensibility for its
language; Rust owns storage, scheduling, dispatch, rate limiting, and worker
management. A shell talks to the core only through its compiled binding crate
(`taskito._taskito` for Python, the `.node` addon for Node) — **a shell never reaches
into core internals, and the core never imports a host-language type except at the
binding edge.** The `WorkerDispatcher` trait in `taskito-core` is binding-free, so a
new shell implements one trait against
(`taskito._taskito` for Python, the `.node` addon for Node, the JNI `.so`/FFM for
Java) — **a shell never reaches into core internals, and the core never imports a
host-language type except at the binding edge.** The `WorkerDispatcher` trait in
`taskito-core` is binding-free, so a new shell implements one trait against
[`BINDING_CONTRACT.md`](crates/taskito-core/BINDING_CONTRACT.md).

```text
┌───────────────────────────────────┐ ┌───────────────────────────────────┐
│ PYTHON SDK sdks/python/taskito/│ │ NODE SDK sdks/node/ │
│ Queue (app.py) = 15 mixins · │ │ Queue/Worker · CLI · dashboard │
│ async_support · serializers │ │ events/middleware · webhooks │
│ FEATURE SUBSYSTEMS (pure-Python): │ │ resources · serializers │
│ interception resources proxies │ │ (Node-native equivalents, not │
│ workflows contrib batching … │ │ 1:1 ports of Python idioms) │
└─────────────────┬─────────────────┘ └─────────────────┬─────────────────┘
│ PyO3 (taskito._taskito) │ napi-rs (.node addon)
┌───────────────────────────────────┐ ┌───────────────────────────────────┐
│ crates/taskito-python/ (PyO3) │ │ crates/taskito-node/ (napi-rs) │
│ py_queue/ · async_worker · │ │ JsQueue/JsWorker · dispatcher · │
│ prefork bridge │ │ convert/ │
└─────────────────┬─────────────────┘ └─────────────────┬─────────────────┘
└───both implement ─┬─ WorkerDispatcher─┘
┌───────────────────────┐ ┌───────────────────────┐ ┌───────────────────────┐
│ PYTHON sdks/python │ │ NODE sdks/node │ │ JAVA sdks/java │
│ Queue = 15 mixins · │ │ Queue/Worker · CLI · │ │ Queue→DefaultTaskito │
│ async · serializers │ │ dashboard · events · │ │ workflows · DI · │
│ interception · │ │ middleware · webhooks │ │ proxies · interception│
│ resources · proxies │ │ resources · │ │ codecs · spring · │
│ workflows · contrib │ │ serializers │ │ processor (@Task) │
└───────────────────────┘ └───────────────────────┘ └───────────────────────┘
PyO3 (._taskito) napi-rs (.node) JNI + FFM (.so)
┌───────────────────────┐ ┌───────────────────────┐ ┌───────────────────────┐
│ taskito-python │ │ taskito-node │ │ taskito-java │
│ py_queue/ · │ │ JsQueue/JsWorker · │ │ backend · dispatcher │
│ async_worker · │ │ dispatcher · convert │ │ convert · ffi · jvm │
│ prefork bridge │ │ │ │ worker · mesh │
└───────────┬───────────┘ └───────────┬───────────┘ └───────────┬───────────┘
└─ all three implement ───┬─ WorkerDispatcher ──────┘
┌─────────────────────────────────────┴─────────────────────────────────────┐
│ RUST CORE crates/taskito-core/ (no host language) │
│ scheduler/ (poll · dispatch · retry · reap · wake) │
Expand All @@ -60,24 +62,25 @@ new shell implements one trait against

WORKFLOWS crates/taskito-workflows/ — separate crate, own schema & stores
(SQLite · Postgres · Redis), surfaced per shell (Python:
py_queue/workflow_ops/ · Node: bound in taskito-node)
py_queue/workflow_ops/ · Node & Java: bound in their crates)
MESH crates/taskito-mesh/ — optional decentralized work-stealing
overlay (SWIM gossip · hash ring · TCP steal); `mesh` feature
NATIVE ASYNC taskito-python/src/native_async/ — optional native-async pool
(Python-coupled; behind the `native-async` feature)
```

The dependency arrows point **downward only**. `taskito-core` knows nothing about
Python, Node, PyO3, or napi; each binding crate (`taskito-python`, `taskito-node`)
depends on `taskito-core`; each SDK package depends on its compiled addon. This
acyclic shape is the property that keeps the codebase changeable — guard it.
Python, Node, Java, PyO3, napi, or JNI; each binding crate (`taskito-python`,
`taskito-node`, `taskito-java`) depends on `taskito-core`; each SDK package depends
on its compiled addon. This acyclic shape is the property that keeps the codebase
changeable — guard it.

---

## Layers & responsibilities

> Sections 1–2 describe the **Python SDK** (the original, most complete shell).
> Sections 6–9 cover the shared crates and the **Node SDK**, the second shell.
> Sections 6–10 cover the shared crates and the **Node** and **Java** SDKs.

### 1. Python API surface — `sdks/python/taskito/`

Expand Down Expand Up @@ -136,7 +139,7 @@ feature): `mod.rs`, `inspection.rs`, `worker.rs`, and `workflow_ops/` (lifecycle
nodes, fan_out, gates, queries, saga). `async_worker.rs` drives the
`AsyncWorkerPool` with `spawn_blocking` + GIL management. This layer converts
Python values to Rust and back; it holds **no business logic**. It implements the
core's `WorkerDispatcher` trait — the same contract the Node binding implements.
core's `WorkerDispatcher` trait — the same contract the Node and Java bindings implement.

### 4. Rust core — `crates/taskito-core/`

Expand Down Expand Up @@ -194,7 +197,24 @@ webhooks, and resource DI. Python-idiom features (proxies, interception) get Nod
**equivalents**, not 1:1 ports. The DB stays the source of truth, so a Python and a
Node worker can share one queue.

### 9. Mesh — `crates/taskito-mesh/`
### 9. Java SDK — `sdks/java/` + `crates/taskito-java/`

The third language shell, peer to Python and Node. `crates/taskito-java/` is a
[JNI](https://github.com/jni-rs/jni-rs) binding crate (`backend.rs`, `dispatcher.rs`,
`convert.rs`, `queue/`, `workflows/`, `worker.rs`, `mesh.rs`) that implements the same
core `WorkerDispatcher` trait, with an FFM fast-path (`ffi.rs`, `ffi_c.rs`) that
bypasses JNI on JDK 22+. `sdks/java/` is the Gradle project (package
`org.byteveda.taskito`, baseline JDK 17): a call on `Queue`/`Taskito` routes
`DefaultTaskito → core.CoreFacade → spi.QueueBackend → internal.JniQueueBackend →
JNI`, so the JNI seam is the shell's single point of contact with Rust. Feature
packages mirror the other shells — `workflows/`, `resources/` (DI), `proxies/`,
`interception/`, `serialization/` (payload codecs), `middleware/`, `webhooks/`,
`autoscale/`, `contrib/`. Extra Gradle subprojects: `:processor` (compile-time
`@TaskHandler` processing, GraalVM-clean), `:test-support` (JNI-free
`InMemoryQueueBackend`), `:spring`, `:graalvm-smoke`. Python-idiom features get Java
**equivalents**, not 1:1 ports; the DB stays the shared source of truth.

### 10. Mesh — `crates/taskito-mesh/`

Optional (`mesh` feature). A decentralized work-stealing overlay — SWIM gossip
(UDP), a consistent-hash ring, a local deque, and TCP work-stealing — composed
Expand All @@ -211,11 +231,12 @@ violates one as a design regression, not a style nit.

1. **Dependencies point downward only.** SDK → its binding crate → core → storage.
`taskito-core` must not depend on any binding crate (`taskito-python`,
`taskito-node`) or host-language type.
`taskito-node`, `taskito-java`) or host-language type.
2. **Each binding crate is its language's only seam.** A shell touches Rust solely
via its compiled addon's public surface — no reaching into struct internals. Keep
the surface contract in sync: `_taskito.pyi` for Python, the generated `.d.ts` for
Node. A new language implements `WorkerDispatcher` against `BINDING_CONTRACT.md`.
Node, the `spi.QueueBackend` interface for Java. A new language implements
`WorkerDispatcher` against `BINDING_CONTRACT.md`.
3. **Asyncio is confined to `async_support/`** (plus the narrow, documented
exceptions: `app.py` uses only `iscoroutinefunction`; `contrib/fastapi.py`).
No inline `import asyncio` to dodge a boundary — split the module instead.
Expand All @@ -242,8 +263,8 @@ detailed versions.
4. Implement it for Redis in `redis_backend/`.
5. Wire it through the `delegate!` macro in `storage/mod.rs`.
6. Expose it on each shell that needs it: PyO3 in `crates/taskito-python/src/py_queue/`
(then add the signature to `sdks/python/taskito/_taskito.pyi`) and/or napi in
`crates/taskito-node/src/queue/`.
(then add the signature to `sdks/python/taskito/_taskito.pyi`), napi in
`crates/taskito-node/src/queue/`, and/or JNI in `crates/taskito-java/src/queue/`.
7. Test: a Rust test in `storage/sqlite/tests.rs` + the contract suite (runs against
all three backends in CI).

Expand All @@ -268,9 +289,9 @@ detailed versions.

## Why this scales

The codebase is large (≈25k LOC Python, ≈29k LOC Rust across 5 crates, ≈7k LOC
Node/TS) but not heavy: no god-objects, an acyclic dependency graph, one binding seam
per language, and duplication erased by macros. The biggest files are deduplicating
The codebase is large (≈25k LOC Python, ≈37k LOC Rust across 6 crates, ≈12k LOC Java,
≈7k LOC Node/TS) but not heavy: no god-objects, an acyclic dependency graph, one
binding seam per language, and duplication erased by macros. The biggest files are deduplicating
macros (`diesel_common/jobs.rs`) or inherently complex state machines (saga, scheduler)
— large because the *problem* is, not because concerns are tangled. Drift comes from
eroding the six boundary rules above; hold them and the design scales with the
Expand Down
6 changes: 6 additions & 0 deletions docs/app/components/diagrams/arch-stack.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,11 @@ export function ArchitectureStack() {
<code>.task()</code>, <code>.enqueue()</code>
</>
}
java={
<>
<code>Task.of()</code>, <code>enqueue()</code>
</>
}
/>
, results, workflows, resources — the surface you write against.
</>
Expand Down Expand Up @@ -167,6 +172,7 @@ export function ResourcePipeline() {
<code>inject:</code> or <code>useResource()</code>
</>
}
java={<code>Resources.use()</code>}
/>
). Task-scoped resources come from a semaphore pool.
</>
Expand Down
28 changes: 26 additions & 2 deletions docs/app/components/diagrams/worker-fork.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,19 @@ export function WorkerDispatch() {
<SdkSwap
python={<code>async def</code>}
node={<code>async</code>}
java="handler"
/>{" "}
functions to the async runtime.
</div>
</div>
<span className="lrole">routes by task type</span>
</div>
<div className="fork-route">
<SdkSwap python="sync def · async def" node="sync · async" />
<SdkSwap
python="sync def · async def"
node="sync · async"
java="handlers"
/>
</div>
<div className="archfork">
<div className="forkcol">
Expand All @@ -49,6 +54,14 @@ export function WorkerDispatch() {
block.
</>
}
java={
<>
Handlers run on a JVM <code>ExecutorService</code> — a
cached pool by default, fixed with{" "}
<code>concurrency(n)</code> — fed by the JNI dispatch
bridge.
</>
}
/>
</div>
</div>
Expand All @@ -60,7 +73,11 @@ export function WorkerDispatch() {
<span className="ltag t-py">Async pool</span>
<div className="lbody">
<div className="lt">
<SdkSwap python="NativeAsyncPool" node="Native async pool" />
<SdkSwap
python="NativeAsyncPool"
node="Native async pool"
java="Executor dispatch"
/>
</div>
<div className="ld">
<SdkSwap
Expand All @@ -78,6 +95,13 @@ export function WorkerDispatch() {
its promise is awaited back into the core.
</>
}
java={
<>
Every job is an executor task; the handler's return value
(or exception) is bridged back into the Rust scheduler as
the job outcome.
</>
}
/>
</div>
</div>
Expand Down
53 changes: 3 additions & 50 deletions docs/app/components/docs/sidebar.tsx
Original file line number Diff line number Diff line change
@@ -1,14 +1,7 @@
import { useEffect, useState } from "react";
import { Link, useLocation, useNavigate } from "react-router";
import { useActiveSdk, useSdk } from "@/hooks";
import {
forcedSdkForPath,
type NavNode,
navForSdk,
type Sdk,
sdkLabels,
sdkSwitchTarget,
} from "@/lib";
import { Link, useLocation } from "react-router";
import { useActiveSdk } from "@/hooks";
import { type NavNode, navForSdk } from "@/lib";

function containsHref(node: NavNode, current: string): boolean {
return (
Expand All @@ -17,45 +10,6 @@ function containsHref(node: NavNode, current: string): boolean {
);
}

// Switcher options come from the SDK registry, so a new language appears here
// automatically.
const SDK_LABELS = sdkLabels();

/** Global SDK toggle. Sets the shared store (flips inline variants + this nav);
* on an SDK-specific page it also navigates to the counterpart page, on a shared
* page it stays put. A labelled control — it switches the whole page, not a panel. */
function SdkSwitch({ sdk, current }: { sdk: Sdk; current: string }) {
const { setSdk } = useSdk();
const navigate = useNavigate();

function select(target: Sdk) {
if (target === sdk) {
return;
}
setSdk(target);
if (forcedSdkForPath(current)) {
navigate(sdkSwitchTarget(current, target));
}
}

return (
<div className="sdk-switch">
{SDK_LABELS.map(({ id, label }) => (
<button
key={id}
type="button"
className={`sdk-opt ${sdk === id ? "active" : ""}`.trim()}
aria-pressed={sdk === id}
aria-label={`${label} SDK`}
onClick={() => select(id)}
>
{label}
</button>
))}
</div>
);
}

function Caret({
open,
onToggle,
Expand Down Expand Up @@ -228,7 +182,6 @@ export function Sidebar({
<kbd>K</kbd>
</span>
</button>
<SdkSwitch sdk={sdk} current={current} />
<nav id="sidenav">
{navForSdk(sdk).map((group) => (
<NavGroup key={group.title} group={group} current={current} />
Expand Down
10 changes: 8 additions & 2 deletions docs/app/components/landing/hero.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@ import { Link } from "react-router";
import { RawHtml } from "@/components/ui";
import { useSdk } from "@/hooks";
import { sdkProfile } from "@/lib";
import { highlightPython, highlightTs } from "@/lib/highlight-lite";
import {
highlightJava,
highlightPython,
highlightTs,
} from "@/lib/highlight-lite";
import { HERO_COMING_SOON, HERO_PANES } from "@/lib/landing-content";

function CopyButton({ text }: { text: string }) {
Expand Down Expand Up @@ -32,7 +36,9 @@ export function Hero() {
const codeHtml =
active.lang === "ts"
? highlightTs(active.code)
: highlightPython(active.code);
: active.lang === "java"
? highlightJava(active.code)
: highlightPython(active.code);

return (
<section className="hero">
Expand Down
22 changes: 15 additions & 7 deletions docs/app/components/landing/scenario-finder.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,25 @@ import { RawHtml } from "@/components/ui";
import { type Sdk, useActiveSdk } from "@/hooks";
import { DemoModal, type DemoTarget } from "./demo-modal";

// Scenario guides are authored as Python paths; most map to Node by swapping the
// SDK segment, but a few pages live under a different slug in the Node tree.
const NODE_GUIDE_OVERRIDES: Record<string, string> = {
"/python/guides/advanced-execution/streaming": "/node/guides/core/streaming",
"/python/guides/workflows/sagas": "/node/guides/workflows/saga",
// Scenario guides are authored as Python paths; most map to another SDK by
// swapping the SDK segment, but a few pages live under a different slug there.
const GUIDE_OVERRIDES: Record<string, Record<string, string>> = {
"/python/guides/advanced-execution/streaming": {
node: "/node/guides/core/streaming",
java: "/java/guides/core/streaming",
},
"/python/guides/workflows/sagas": {
node: "/node/guides/workflows/saga",
java: "/java/guides/workflows/saga",
},
};

/** Resolve a scenario's Python guide path to the active SDK's docs. */
function guideHref(pyPath: string, sdk: Sdk): string {
if (sdk !== "node") return pyPath;
return NODE_GUIDE_OVERRIDES[pyPath] ?? pyPath.replace("/python/", "/node/");
if (sdk === "python") return pyPath;
return (
GUIDE_OVERRIDES[pyPath]?.[sdk] ?? pyPath.replace("/python/", `/${sdk}/`)
);
}

interface Bullet {
Expand Down
Loading