diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d207b18 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.go text eol=lf diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 61b7b22..11a730a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,45 +1,51 @@ - + -## 改了什么，为什么 +## What Changed and Why - + -## 关联 issue +## Related Issue - + -## Spec 依据 +## Specification - + -## 破坏性变更与迁移 +## Breaking Changes and Migration - + -## 面向用户的变更 +## User-Visible Change - + ```release-note ``` -## 验证 +## Verification - + -## 变异测试 +## Mutation Test -## 必须检查 +## Required Checks -- [ ] `make ci` 在本地绿 -- [ ] 没有新旧测试并存 -- [ ] 生产代码里没有 `time.Now()` -- [ ] 没有用 `time.Sleep` 做同步 -- [ ] 没有用 mutex 做跨调用等待 -- [ ] 有 spec 支撑；spec 和实装的差距已在文内说明 +- [ ] `make ci` passes locally. +- [ ] Old and new tests do not test the same contract. +- [ ] Production code does not call `time.Now()`. +- [ ] Tests do not use `time.Sleep` for synchronization. +- [ ] Code does not use a mutex to wait across Calls. +- [ ] A specification supports the change. It states each implementation gap. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 778d663..1e88dad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: matrix: go-version: ["1.25.x", "stable"] @@ -20,23 +20,77 @@ jobs: with: go-version: ${{ matrix.go-version }} cache: true - - name: Install staticcheck - run: | - go install honnef.co/go/tools/cmd/staticcheck@v0.6.1 - echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + - name: Verify platform + run: go run ./internal/platformcheck -os linux -arch amd64 - name: Run CI checks run: make ci - name: Run govulncheck if: matrix.go-version == 'stable' - run: go run golang.org/x/vuln/cmd/govulncheck@latest ./... + run: go tool govulncheck ./... + macos: + runs-on: macos-15 + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.25.x" + cache: true + - name: Verify platform + run: go run ./internal/platformcheck -os darwin -arch arm64 + - name: Run default tests + run: go run ./internal/testcheck ./... + - name: Run simulation tests + run: go run ./internal/testcheck -tags sim -run '^TestSim' ./sim/... + - name: Run generator tests + run: go run ./internal/testcheck -tags gen ./cmd/gorgen/... + - name: Check generated test package + run: go tool gorgen -pkg ./cmd/gorgen/testfixture/endtoend/domain -check + - name: Check generated example package + run: go tool gorgen -pkg ./examples/shadow/domain -check + - name: Run transport network tests + run: go run ./internal/testcheck -tags net ./transport/... + - name: Run example network tests + run: go run ./internal/testcheck -tags net ./examples/shadow/... + windows: + runs-on: windows-2025 + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.25.x" + cache: true + - name: Verify platform + run: go run ./internal/platformcheck -os windows -arch amd64 + - name: Run default tests + run: go run ./internal/testcheck ./... + - name: Run simulation tests + run: go run ./internal/testcheck -tags sim -run '^TestSim' ./sim/... + - name: Run generator tests + run: go run ./internal/testcheck -tags gen ./cmd/gorgen/... + - name: Check generated test package + run: go tool gorgen -pkg ./cmd/gorgen/testfixture/endtoend/domain -check + - name: Check generated example package + run: go tool gorgen -pkg ./examples/shadow/domain -check + - name: Run transport network tests + run: go run ./internal/testcheck -tags net ./transport/... + - name: Run example network tests + run: go run ./internal/testcheck -tags net ./examples/shadow/... ci: if: ${{ always() }} - needs: test - runs-on: ubuntu-latest + needs: [test, macos, windows] + runs-on: ubuntu-24.04 steps: - - name: Verify matrix result + - name: Verify required jobs run: | echo "matrix result: ${{ needs.test.result }}" - if [ "${{ needs.test.result }}" != "success" ]; then + echo "macOS result: ${{ needs.macos.result }}" + echo "Windows result: ${{ needs.windows.result }}" + if [ "${{ needs.test.result }}" != "success" ] || \ + [ "${{ needs.macos.result }}" != "success" ] || \ + [ "${{ needs.windows.result }}" != "success" ]; then exit 1 fi diff --git a/CONTEXT.md b/CONTEXT.md index 9f5009e..49a0a0c 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -15,7 +15,8 @@ The stable identity of one Grain. It contains a GrainType and a GrainKey. _Avoid_: identity, address, instance ID, object ID **GrainType**: -The kind of Grain. Grains with different GrainTypes do not share a GrainId. +The stable Application name for one kind of Grain. Grains with different +GrainTypes do not share a GrainId. _Avoid_: class, model, category **GrainKey**: @@ -26,6 +27,11 @@ _Avoid_: ID, name, identifier A typed value that names a Grain without creating it. _Avoid_: proxy, handle, stub, pointer +**Grain Context**: +Runtime data for one Activation. The Application uses it to get State, Grain +References, Grain Timers, Reminders, and lifecycle controls. +_Avoid_: Binder, service provider, runtime handle + **Call**: A request to run one method on a Grain through a Grain Reference. _Avoid_: message, invocation, packet @@ -37,6 +43,7 @@ _Avoid_: header, context value, request property **Call Filter**: Shared policy that runs before or after a Call. +Call Filters are deferred until after 0.1.0. _Avoid_: interceptor, middleware ## Runtime model @@ -56,11 +63,16 @@ The end of an Activation. Deactivation does not delete a Grain's GrainId or State. _Avoid_: destroy, delete, terminate +**Deactivate on Idle**: +A request from a Grain to end its Activation after the current Call. The next +Call uses a new Activation. +_Avoid_: delete Grain, stop Grain, destroy instance + **Lifecycle**: The path from Activation to Deactivation for one Grain. _Avoid_: object lifetime, process lifetime -## State and reminders +## State and time **State**: The current data owned by a Grain. State describes the Grain now. @@ -79,6 +91,21 @@ A future Call that the Grain Runtime remembers for a Grain. A Reminder can happen once or repeat on a period. _Avoid_: timer, wake-up, scheduled task, job +**Invalid Reminder**: +A persisted Reminder whose GrainType or method is not installed. The Grain +Runtime cannot make its Call. +_Avoid_: bad job, dead letter, broken timer + +**Terminal Result**: +The result that removes one unchanged Invalid Reminder. Later scans cannot +return it. A new Set creates a new Reminder setting. +_Avoid_: tombstone, dead letter + +**Grain Timer**: +An Activation-local callback that a Grain schedules. The Grain Runtime does +not save it and discards it when the Activation ends. +_Avoid_: Reminder, job, scheduler + ## Call results **Conflict**: diff --git a/FINDINGS.md b/FINDINGS.md deleted file mode 100644 index 1aec08d..0000000 --- a/FINDINGS.md +++ /dev/null @@ -1,23 +0,0 @@ -# Findings - -After two passes over the device shadow as an external user, the API friction that remains: - -1. **Entity time: resolved.** The entity reads the runtime clock from its binder; the example writes report timestamps with `gor.Now(binder)`. Constructors no longer need to receive a clock, and tests control time. - -2. **Entity-to-entity references: resolved.** An entity gets a typed reference to another entity from its own binder; the example notifies the workshop with `gor.Ref[Workshop](binder, workshopID)`. No factory closure capturing the runtime. - -3. **Cross-entity consistency: still there, and deliberately not provided.** One report writes the device shadow, resets the offline timer, and notifies the workshop in sequence; when a later step fails, an earlier one may have succeeded. `gor` offers no cross-entity transactions, outbox, compensation, or unified retry state; the business must accept this consistency window, or put the state that must be atomic into one entity. - -4. **Invisible scheduled-delivery failures: resolved, but only the visibility.** A failing scheduled method now comes out of the runtime's `OnError` sink; the example's two entry points both install it and verify, with a failing scheduled method, that it actually arrives. The runtime still does not retry; retrying, backing off, and alerting remain the application's call. - -5. **`State[T].Get()` shared-value semantics: the API semantics stay; the docs now say it clearly.** A map or slice from `Get()` is the very instance the activation holds; after mutating it you must still call `Set()` for persistence; without `Set()`, eviction reverts to the old value in the store. This is deliberately kept semantics, not a friction step 5.5 removes. - -6. **Missing lifecycle hooks: resolved.** Entities can implement `OnActivate` and `OnDeactivate`; the example's load generator waits on an `OnDeactivate` channel signal for real idle eviction, then calls the entity to confirm state is restored from the store; tests also use fake clocks to verify automatic eviction, hook calls, and reactivation. Hook errors still go through the unified `OnError` sink — see the new friction below. - -New findings this round: - -7. **Entities must keep their `Binder` themselves.** Both `gor.Now` and entity-to-entity `gor.Ref` need the `Binder`, so the entity stores it in a field for later methods and lifecycle hooks. This is the current design, not an implementation bug: it keeps entities from capturing runtime objects, but it is also the first rule a new user must remember. - -8. **`OnError`'s source information too coarse: resolved.** Scheduled-delivery failures and `OnDeactivate` failures share one structured sink: the event gives the entity, the original error, and a closed source set — scheduled delivery carries the method name, deactivation failure carries the deactivation reason. The source set is closed; nothing outside the package can add to it, so a scheduled method that happens to be named `"OnDeactivate"` cannot be confused with the deactivation hook. It carries no schedule metadata or attempt counts, and reports no scan or claim failures; that information may be stale after claiming, the ETag is not an application decision, and the runtime has no retry model. The scheduled delivery canceled mid-shutdown is not reported either. - -9. **`OnDeactivate` had no deactivation reason: resolved.** Deactivation distinguishes four application-actionable cases — idle, current node lost ownership, normal shutdown, instance untrusted — and the reason is fixed at the first transition out of the active state; no later event rewrites it. Applications can then choose to reclaim local resources, hand back node ownership, teardown before process exit, or alert on fault; reasons are not exposed one by one per internal implementation branch. The deactivation hook's work context has no deadline and is never canceled. An abrupt stop does not start teardown that has not begun; a graceful stop waits for teardown that has begun to return. diff --git a/Makefile b/Makefile index 4f0db8c..a3503d4 100644 --- a/Makefile +++ b/Makefile @@ -1,31 +1,67 @@ -.PHONY: test sim gen lint net fmt fmt-check ci tidy bench +.PHONY: test sim gen generated-check lint net fuzz resource release-command-check external external-tagged tidy-check fmt fmt-check race ci tidy bench + +# Keep release evidence in the specified order, including with make -j. +.NOTPARALLEL: + +# Caller-provided Make variables must not replace release test commands. +FUZZ_TIME ?= 10s test: - go test ./... + go run ./internal/testcheck ./... -# 性能基线单独运行，不进默认 test；GOR_BENCH_DIR 可指定真盘目录 +# Benchmarks run separately. GOR_BENCH_DIR can select a real-disk directory. bench: go test . -run '^$$' -bench . -benchmem -count=1 -# 模拟测试跑得慢，单独一条 target，不进默认 test +# Simulation tests run separately from the default test target. sim: - go test -tags sim -run TestSim ./sim/... + go run ./internal/testcheck -tags sim -run '^TestSim' ./sim/... -# 生成器的端到端测试要起 go list 子进程，同样不进默认 test +# Generator integration tests start Go subprocesses and run separately. gen: - go test -tags gen ./cmd/gorgen/... + go run ./internal/testcheck -tags gen ./cmd/gorgen/... + go tool gorgen -pkg ./cmd/gorgen/testfixture/endtoend/domain -check + go tool gorgen -pkg ./examples/shadow/domain -check + +generated-check: + go tool gorgen -pkg ./cmd/gorgen/testfixture/endtoend/domain -check + go tool gorgen -pkg ./examples/shadow/domain -check -# 真 TCP 的传输测试单独运行，不进默认 test +# Real TCP tests run separately from the default test target. net: - go test -tags net ./transport/... - go test -tags net ./examples/shadow/... + go run ./internal/testcheck -tags net ./transport/... + go run ./internal/testcheck -tags net ./examples/shadow/... + +fuzz: + go run ./internal/testcheck ./transport -run '^$$' -fuzz '^FuzzReadFrame$$' -fuzztime=$(FUZZ_TIME) + go run ./internal/testcheck . -run '^$$' -fuzz '^FuzzDecodeRequestContext$$' -fuzztime=$(FUZZ_TIME) + go run ./internal/testcheck ./internal/codegen -run '^$$' -fuzz '^FuzzParseGrainMarker$$' -fuzztime=$(FUZZ_TIME) + +resource: + go run ./internal/testcheck ./internal/runtime -run '^TestRuntime_ActivationChurnReleasesOwnedResources$$' -count=1 + go run ./internal/testcheck ./internal/timer -run '^TestPoller_LargeBacklogKeepsPagesAndWorkersBounded$$' -count=1 + +release-command-check: + go run ./internal/testcheck -tags release ./internal/testcheck -run '^TestMakefile_ReleaseCommandsKeepTestcheck$$' -count=1 + +# External release proof uses empty Go caches and separate operating system processes. +external: release-command-check + go run ./internal/testcheck -tags release -run '^TestQuickStartHTTPProcess$$' ./examples/shadow/cmd/shadow + go run ./internal/testcheck -tags release -run '^TestExternalModule(Configuration|RestartProof|UpgradeProof)$$' ./examples/shadow/cmd/conformance + +# Published release proof resolves the fixed tag without a local replacement. +external-tagged: release-command-check + go run ./internal/testcheck -tags release -run '^TestExternalModule(Configuration|RestartProof|UpgradeProof)$$' ./examples/shadow/cmd/conformance -args -external-tagged + +tidy-check: + go mod tidy -diff lint: go run ./internal/constraintcheck go vet ./... - staticcheck ./... + go tool staticcheck ./... go vet -tags sim ./sim/... - staticcheck -tags sim ./sim/... + go tool staticcheck -tags sim ./sim/... fmt: gofmt -l -w . @@ -37,14 +73,10 @@ fmt-check: exit 1; \ fi -ci: - $(MAKE) fmt-check - $(MAKE) lint - $(MAKE) test - go test -count=1 -race ./... - $(MAKE) sim - $(MAKE) gen - $(MAKE) net +race: + go run ./internal/testcheck -count=1 -race ./... + +ci: fmt-check tidy-check lint release-command-check test race sim gen net resource fuzz tidy: go mod tidy diff --git a/README.md b/README.md index 9ac3969..e5d6136 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ **A persistent Grain Runtime for Go.** It is embeddable, runs as one Silo, and is designed for deterministic simulation testing. -> **Status:** single-Silo features are implemented and usable. Cluster +> **Status:** Single Silo features are implemented and usable. Cluster > features are an optional preview. Detailed progress: [ROADMAP.md](ROADMAP.md) ## What this is @@ -11,28 +11,40 @@ and is designed for deterministic simulation testing. A Go library that makes a Grain with a GrainId, State, serialized Calls, and restart recovery your programming unit. You write Go interfaces and structs. `gor` handles Activation, Call ordering, persistence, and Reminders. A -future cluster is an optional extension, not the main line. +future Cluster is an optional extension, not the main line. `gor` is a Go port of the Orleans runtime model. The Go API uses Go forms, but the product terms and runtime meaning follow Orleans. The main design rules are in the [design documents](design/README.md): -- The programming model is typed at compile time, not `any` in, `any` out — proxies are generated from Go interfaces ([design/codegen.md](design/codegen.md)). +- The programming model uses compile-time types. +- The generator creates Grain Reference implementations from Go interfaces. + See [design/codegen.md](design/codegen.md). - One Silo is a first-class product. It needs no sidecar or remote service. -- Deterministic simulation testing is an architectural constraint, not a testing technique retrofitted afterwards ([design/testing.md](design/testing.md)). This is the main difference between this project and comparable implementations. +- Deterministic simulation testing is an architectural constraint. + See [design/testing.md](design/testing.md). -## Why it exists +## Quick Start -The Go ecosystem has a gap right in this spot. Put the three conditions — stateful, crash-transparent, embeddable — together: +Requires Go 1.25 or later. Start the device shadow example from the repository +root: -| | Language | Form | Usable single-node | -|---|---|---|---| -| Temporal | Go | separate server + workers | needs a server and a database | -| Restate / Rivet | Rust | single-binary server | yes, but not a Go library | -| Dapr | Go | sidecar process | adds one deployment unit | -| goakt | Go | library | yes, but the API is `any`-based, and maintenance is concentrated on a single person | +```bash +go run ./examples/shadow/cmd/shadow +``` + +In another terminal, report and read one device: + +```bash +curl -i -X POST http://localhost:8080/devices/device-1/reports \ + -H 'Content-Type: application/json' \ + -d '{"workshop_id":"assembly","state":"temperature=20"}' +curl http://localhost:8080/devices/device-1/shadow +``` -Measured details: [research/landscape.md](research/landscape.md) (in Chinese). +The release test builds this command and verifies this HTTP flow. See the +[complete example guide](examples/shadow/README.md) for State, Reminders, and +restart behavior. ## What it does not do @@ -42,16 +54,17 @@ Measured details: [research/landscape.md](research/landscape.md) (in Chinese). - No source or binary compatibility promise with Orleans. - No Call Filters. - No reentrant or interleaved Grain Calls. -- No cluster operation tools or unbounded scale. +- No Cluster operation tools or unbounded scale. ## Documentation -- [docs/vision.md](docs/vision.md) — positioning, principles, non-goals. Read this when judging whether a change is aligned with the direction. -- [docs/programming-model.md](docs/programming-model.md) — the user-facing programming model and API shape. -- [design/](design/README.md) — architecture, subsystem designs, technical trade-offs. -- [research/](research/README.md) (in Chinese) — the measured evidence behind those decisions (Orleans source-code measurements, ecosystem landscape, Go-side capability boundaries). -- [ROADMAP.md](ROADMAP.md) — MVP slicing and acceptance criteria. -- [examples/shadow/](examples/shadow/) — a runnable device-shadow service. The API friction it surfaced is recorded in [FINDINGS.md](FINDINGS.md). +- [docs/vision.md](docs/vision.md) describes the product direction. +- [docs/programming-model.md](docs/programming-model.md) defines the public model + and API. +- [design/](design/README.md) contains technical designs. +- [research/](research/README.md) contains measured evidence. +- [ROADMAP.md](ROADMAP.md) tracks implementation and release work. +- [examples/shadow/](examples/shadow/) is a runnable device shadow service. ## Development @@ -61,9 +74,10 @@ make sim # deterministic simulation tests make gen # generator end-to-end tests make net # transport tests over real TCP make lint # vet + staticcheck +make external # clean consumer and process restart proof ``` -Requires Go 1.25 or later — `testing/synctest` only became GA in 1.25, and it is the foundation of the testing strategy. +`testing/synctest` is the foundation of the deterministic test strategy. ## License diff --git a/ROADMAP.md b/ROADMAP.md index eef087b..5b153fc 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,18 +1,34 @@ # Roadmap -**Direction.** `gor` is for programs that need stateful objects on one machine. The main line is the single-node product; clustering is an optional extension that exists and is shipped, but is not the path the project is built around. See [docs/vision.md](docs/vision.md). - -**Status.** The single-node core (steps 1–5.5) is implemented and usable; for practical purposes, the single-node product is done. Clustering (step 6) is implemented and shipped in 0.0.x as a preview. The pre-announcement checklist is done. The announced 0.1.0 target and its remaining composition and failure-evidence work are defined below. - -Slicing principle: every step runs, is accepted, and delivers value on its own. Dependencies of the form "step 1 cannot be verified until step 6" are not allowed. - -## The single-node core - -Steps 1 through 5.5 form the single-Silo product: a per-Grain runtime, +**Direction.** `gor` runs persistent Grains in one Silo. Cluster support is an +optional preview. See [docs/vision.md](docs/vision.md). + +**Status.** The Single Silo core in steps 1 through 5.5 is implemented and +usable. Runtime setup, stable GrainType, Grain Context, Activation lifecycle, +Grain Timers, State failure recovery, bounded Reminder delivery, the Silo +Activation admission limit, and bounded resource tests are complete for 0.1.0. +Portable benchmark probes are also complete. The fail-closed local release +gate and fuzz checks are also complete. The external restart proof and public +document cleanup are complete. The exact tagged-module command is ready. The +hosted supported-system run and the successful tagged-module proof remain +open. Action recovery confirms Grain State before it commits an Application +receipt. Pending actions recover in save order. An Invalid Reminder now gets +one Terminal Result. Direct lifecycle proof for a mutable State alias and a +Reminder-turn State failure is complete. The exact `v0.0.5` SQLite migration +and external upgrade proof are complete locally. Cluster support remains a +preview. + +Each step must run and deliver value on its own. A later step must not be +needed to verify an earlier step. + +## The Single Silo core + +Steps 1 through 5.5 form the current Single Silo core: a per-Grain Runtime, persistent State, typed Grain References, the simulation harness, Reminders, -and the API fixes the first real example surfaced. They are implemented. +and the API fixes from the first real example. They are implemented. The 0.1.0 +section defines the remaining product work. -### 1. Single-process runtime +### 1. Single Silo Runtime A per-key mailbox (one goroutine + one channel), an activation cache, and idle eviction. No network, no persistence, no cluster. @@ -28,29 +44,37 @@ Method dispatch is handwritten in this step. Besides the factory, `Register` tak State read/write for objects, plus one table with CAS. The storage backend is embedded (one of SQLite / bbolt / pebble; the choice is in [design/persistence.md](design/persistence.md)). -The registration factory signature changes from `func() T` to `func(*gor.Binder) T` in this step. Step 1 has no storage, so no placeholder `Binder` is added early. This is a planned breaking change, not an oversight. +The registration factory first receives an Activation context in this step. +The 0.1.0 public name for this value is `*gor.GrainContext`. Step 1 has no +storage, so no placeholder context is added early. **Acceptance:** state is restored after a process restart; concurrent write conflicts are rejected by CAS instead of silently overwritten. -### 3. Typed proxy code generation +### 3. Typed Grain Reference generation -Generate proxy implementations from user-written Go interfaces, replacing `any` in, `any` out. Design and precedent: [design/codegen.md](design/codegen.md). +Generate Grain Reference implementations from user-written Go interfaces. +The public method signatures keep their Go types. See +[design/codegen.md](design/codegen.md). The `gor.Register` signature changes from `(rt, factory, dispatch)` to `(rt, factory)` in this step — the generator takes over the handwritten dispatch function from step 1. `gor.Ref` is added at the same time. Like the step 2 factory signature, this is a planned breaking change. -**Acceptance:** calling a remote object's method from user code uses exactly the local interface's signature; a wrong argument type is a compile error, not a runtime panic. +**Acceptance:** calling a Grain method uses the interface signature. A wrong +argument type is a compile error. ### 4. Deterministic simulation test skeleton Seed-driven fault injection, `testing/synctest` fake clocks, and porcupine for linearizability checking. Design: [design/simulation.md](design/simulation.md). -This step must come before the cluster. DST cannot be retrofitted — it requires all I/O behind interfaces, all components as explicit state machines, and no direct wall-clock reads anywhere. Adding it after step 6 would mean rewriting step 6. Reasons: [design/testing.md](design/testing.md). +This step must come before Cluster work. DST requires I/O interfaces, explicit +state machines, and injected time. See [design/testing.md](design/testing.md). -The fake network is not in this step. There are no cross-node calls yet, so there is nothing to inject; step 6 hangs a new fault source on the skeleton built here. The skeleton itself must be completed here: seeds, fault injection on the fake store, node crash and restart, event log, invariant assertions. +The fake network is not in this step. There are no cross-Silo Calls yet. This +step adds seeds, Store faults, Silo restart, event logs, and invariants. -Node crashes require `runtime` to gain a stop path that does not drain. This is a planned addition, not a breaking change. +Silo crashes require a Runtime stop path that does not drain. -**Acceptance:** a fixed seed reproduces a sequence of injected store faults and node crashes, and a rerun yields a byte-identical decision sequence; invariants hold under those faults; a double activation created by two nodes sharing one store is blocked by the ETag on write conflict instead of being silently overwritten. +**Acceptance:** a fixed seed reproduces Store faults and Silo crashes. A rerun +produces the same decision sequence. ETags prevent silent State replacement. What is reproduced is the injected decisions, not the whole execution. A fault deactivates the Activation, and later Grain State can depend on @@ -64,18 +88,25 @@ Deliberately not repeating Orleans Reminders v1's design — the in-memory cache The Reminder table does not go through `store.Store`; it is a new interface. It is also a new fault source on the step-4 skeleton — it must be hooked up in this step, not deferred to step 6. -Claiming via CAS must be done right now. In step 6, two nodes' pollers can scan the same row at the same time; fixing it then would mean rewriting all the earlier tests. +Reminder Claim uses CAS. Two Silo pollers can read the same row. Only one can +win that Claim. -**Acceptance:** a task that has come due still fires after a process crash; the same due time is delivered at most once, and this must hold as a simulation-test invariant under injected claim faults and node crashes. +**Acceptance:** a due Reminder can run after a process crash. One due time is +delivered at most once under Claim faults and Silo crashes. ### 5.5 API fixes from the example -Implemented. The example's factory now takes only `*gor.Binder`; the load generator no longer narrates eviction — it first asserts the local activation directory is empty, then reads state back, and that assertion fails when eviction is off. +Implemented. The example's factory takes only the current Activation context. +The load generator no longer narrates eviction. It first asserts that the +local Activation directory is empty. It then reads State back. The assertion +fails when eviction is off. Batch 2 renames the legacy context type to +`*gor.GrainContext`. The friction from writing the first real example was minor, but all of it sat on the main path: -- `gor.Now(b)` — the Binder already holds the injected `Clock`; without it, users would write `time.Now()`. -- `gor.Ref[T](b, key)` — a Grain calling another Grain should not require the +- `gor.Now(g)` - Grain Context holds the injected `Clock`; without it, + Applications would use `time.Now()`. +- `gor.Ref[T](g, key)` - a Grain calling another Grain should not require the factory to capture a Runtime object. - `OnError` — scheduled delivery failures used to be dropped silently; they are now visible to users through the unified error sink. - `OnActivate` / `OnDeactivate` — the lifecycle hooks used to be missing; they are now implemented as optional interfaces, so the example can be notified on activation and eviction. @@ -84,17 +115,17 @@ The first two are in [design/persistence.md](design/persistence.md), the third i Placed before step 6 because it changes the public API. API changes get more expensive the later they come, and the example app is waiting on the new signatures to get the README right. -## The single-node line, going forward +## The Single Silo line -The single-Silo core above is usable. A user who runs `gor` on one machine -has typed Grains, State that survives a crash, serialized Calls, Reminders, -lifecycle hooks, observability, and a stable error contract. The 0.1.0 work -still has to make these capabilities one public experience. Clustering is -optional and is not a prerequisite. +The Single Silo core above is usable. A user who runs `gor` on one machine has +typed Grains, persistent State, serialized Calls, Reminders, lifecycle hooks, +observability, and stable error codes. The 0.1.0 work must close known failure +gaps and add the remaining promised Orleans capabilities. Clustering is not a +prerequisite. ### A durability control for state writes -A State write is the operation single-Silo users care about most. It bounds +A State write is the operation Single Silo users care about most. It bounds how many State changes one Grain can make per second ([design/benchmarks.md](design/benchmarks.md)). The durability control is implemented and applies to Grain State only. Its exact limits are in @@ -105,11 +136,14 @@ Grain State. Reminder and membership data stays at Full durability. The baseline is recorded at both tiers on real disk: Full 1.7 ms/op, Relaxed 14 us/op ([benchmarks.md](benchmarks.md)). -**Acceptance.** A single-node user can pick a durability tier without writing their own store; the benchmark records a number at the relaxed tier alongside the full-durability baseline; the durability trade is stated in product language in the docs. +**Acceptance.** A Single Silo user can select a durability tier. Benchmarks +must record both tiers. Product documents must state the durability trade. ### Defend the reproducible-test foundation -"Reproducible tests, not hope" is one of the two commitments the project rests on ([docs/vision.md](docs/vision.md)). The deterministic harness is what makes every promise — single-node included — checkable, so keeping the decision half pure is not a cluster side-task. The ruling: the decision half reads only liveness the test driver itself caused, and the reproducibility gate covers the batch, not one seed. Design: [design/simulation.md](design/simulation.md). +Deterministic tests make each product promise checkable. The decision model +uses only driver-owned liveness. The gate covers every seed in the batch. See +[design/simulation.md](design/simulation.md). Implemented. The decision encoding reads driver-owned liveness only — `simulationCluster` keeps a liveness model moved only by the driver's crash, leave, and restart decisions — and the reproducibility gate runs every seed in the batch twice and compares decision lines byte for byte. Per-message drop is wired into the fake network: whether a request or a reply is dropped is a seed-drawn decision, and a dropped reply produces the state where the write took effect but the caller saw failure. The transport-failed → unknown mapping is guarded by a history test: a dropped reply is followed by a call that reads the write it left behind, and the history linearizes only under the unknown mapping. @@ -117,38 +151,63 @@ Implemented. The decision encoding reads driver-owned liveness only — `simulat ### Not a step: other storage backends -bbolt and pebble are candidates for a single-node store, and a single-node-first store raises their relevance: the design leaned toward SQLite partly for cluster reasons — SQLite "satisfies both state storage and coordination tables", and coordination tables are a cluster need ([design/persistence.md](design/persistence.md)). Postgres was cluster-only and leaves with clustering. This is a goal, not a gap: the store interface is public, a user can ship their own backend, and no measured `gor`-specific number shows bbolt or pebble beating a relaxed-durability SQLite for this workload. The durability control above is the step with evidence; this becomes a step only when measurement shows a real user pain. They stay deliberately un-milestoned goals. +Other embedded Stores remain possible future work. The public Store interface +lets an Application provide one now. No current measurement requires another +built-in Store. See [design/persistence.md](design/persistence.md). ## Announced release target: 0.1.0 The next announced release is governed by the [0.1.0 product contract](docs/release-0.1.0.md) and delivered in the order specified by [design/release-0.1.0.md](design/release-0.1.0.md). This is a target specification, not a claim that the work is complete. -The release target keeps the product single-Silo and makes the Grain, State, -Reminder, lifecycle, Call, persistence, and observability capabilities -dependable as one public experience. Cluster production is outside this -target. A release item is complete only when its failure behavior is tested -and the conformance Application can use it through the public API. +The release target keeps the product Single Silo. It makes Grain, Grain +Context, Call, State, Grain Timer, Reminder, lifecycle, persistence, and +observability work as one public experience. A release item is complete only +when its failure behavior is tested and the conformance Application uses it +through the public API. The implementation batches are: -1. Freeze the public contract and acceptance matrix. -2. Harden Grain Activation, Calls, lifecycle, shutdown, and errors. -3. Verify State and Reminder recovery under crashes and duplicate attempts. -4. Complete call context, serialization, and application-storage boundaries. -5. Run the conformance application and deterministic failure suite. -6. Pass clean-install and full repository release gates. - -The first batch is documentation-only and must be reviewed before implementation begins. - -## Optional extension: clustering - -Clustering is implemented and shipped in the 0.0.x tags. It is an optional extension, not the main line: it exists for workloads that have outgrown one machine, and single-node users are not asked to pay for it. The boundary is stated in [docs/vision.md](docs/vision.md) and in user terms in [docs/programming-model.md](docs/programming-model.md): during the window while nodes disagree about ownership, a write that always succeeds on a single node can return a conflict to the caller, who must retry. Further cluster work — rolling upgrades and operational cleanup — is deliberately deferred; it is not on the main line. - -### 6. Multiple nodes +1. Freeze the public contract and delivery plan. **Complete.** +2. Add Runtime setup and shutdown, stable GrainType, and Grain Context. + **Complete.** +3. Harden Activation, Call, Deactivate on Idle, callbacks, and shutdown. + **Complete.** +4. Add Grain Timers. **Complete.** +5. Harden State failure and SQLite recovery rules. **Complete.** +6. Bound and observe Reminder work. **Complete.** +7. Harden generation and reduce the public package surface. **Complete.** +8. Add limits, portability, and final release proof. **In progress.** Activation + admission, bounded Activation churn, bounded Reminder backlog proof, and + portable benchmark probes are complete. The fail-closed local gate, fixed + release tools, fuzz checks, and external restart proof are also complete. + Public document cleanup and the exact tagged-module command are complete. + A release audit follow-up now passes each dynamic Reminder name to its typed + method. A second follow-up prevents a caller-provided Make variable from + replacing the internal test command. A third follow-up restores Grain State + before it commits an Application receipt. A fourth follow-up gives an + Invalid Reminder one Terminal Result. A fifth follow-up proves the direct + State failure lifecycle. A sixth follow-up migrates the exact `v0.0.5` + schema and proves the external upgrade path. Hosted platform proof and the + successful tagged-module run are still open. + +The exact scope and proof for each batch are in +[design/release-0.1.0.md](design/release-0.1.0.md). Batches 1 through 7 and +Batch 8a through 8d are complete. Batch 8e owns the final same-commit proof. + +## Cluster preview + +Cluster support ships as an optional preview. It is not part of the 0.1.0 +product contract. Silos can briefly disagree about Grain ownership. A State +write can then return a Conflict. See [docs/programming-model.md](docs/programming-model.md). + +### 6. Multiple Silos A consistent-hash ring, shared-table membership, and death voting. Design: [design/cluster.md](design/cluster.md). -This step must state plainly, in both docs and API: the directory is eventually consistent, a double-activation window exists while the cluster is unstable, and state must therefore carry an ETag. Orleans itself has this semantics (see [research/orleans-internals.md](research/orleans-internals.md) (in Chinese)); do not pretend to do better. +The product docs and API must state the Cluster rule. The Grain Directory is +eventually consistent. Cluster instability can create duplicate Activations. +State must use an ETag to reject stale writes. Orleans has the same semantics +(see [research/orleans-internals.md](research/orleans-internals.md)). Too big; sliced into four segments. The split points are chosen on "does it need the network" — the dividing line is transport, then probing. @@ -158,15 +217,19 @@ GrainIds, or the membership table. #### 6a. Membership table and ring -A membership table, a node state machine (joining / active / dead), view polling, a hash ring, and local routing decisions. No transport: if the computed target is not this node, return an error carrying the owner's address. +A membership table stores each Silo state. View polling and a hash ring select +the owner. This segment has no Transport. A new table and a new fault source, shaped like step 5's Reminder table — deliberately so; step 5 just blazed this trail. 6a's membership-table-and-ring stage only defines member states and the view; the evidence for declaring death is completed by 6c's probe voting. -Implemented. The two boundaries of declaring death are written into [design/cluster.md](design/cluster.md): a failed table read does not make anyone dead, and a death declaration enters the view only after its CAS lands. A node declared dead drops all its activations and closes `Done()`. +Implemented. A failed table read does not declare a Silo dead. A declaration +enters the view only after its CAS succeeds. A dead Silo drops Activations and +closes `Done()`. -**Acceptance:** after a node joins, other nodes can see it; after a crash it is declared dead and removed from the ring; after the view converges, the same key lands on exactly one node; when the view changes, activations the node no longer owns are dropped. All of this must live in the simulation tests and hold under membership-table fault injection. +**Acceptance:** Silos discover a joined Silo. A crashed Silo leaves the ring. +After convergence, one Silo owns each key. Old owners drop their Activations. #### 6t. Transport @@ -192,35 +255,55 @@ This step already changed the generated artifacts. `Invoke`'s argument went from #### 6c. Probing and death voting -Direct probing of ring neighbors, death votes with expiry, and a node's self health check. +Direct probing checks Silo neighbors. Death votes expire. Each Silo also checks +its own health. Replaces 6a's coarse death decision of only looking at `iam_alive_at`. The `suspect_votes` column in the table starts being written only in this step. The design is complete, in [design/cluster.md](design/cluster.md): a single-point probe ring, the `Prober` interface, the parameter table, CAS merging and expiry of votes, the `min(2, n-1)` threshold for declaring death, and giving up the voting right when the self-check fails. The envelope's `kind` field is also introduced in this step. -Implemented. Nodes judge neighbor health by directly probing the ring, using neighbor death votes with expiry and landing death declarations in the membership table via CAS; a node whose self-check fails gives up its voting right, and stale heartbeats, failed table reads, and missing heartbeats are no longer evidence of death. +Implemented. Silos probe ring neighbors and store expiring death votes. A Silo +with a failed self-check cannot vote. Missing heartbeats alone do not prove +death. -The boundary: the two sides of a partition can vote each other dead, even to the point where every node stops serving; recovery requires rejoining the membership table with a new generation — old rows do not resurrect themselves. +A partition can make both sides declare each other dead. Recovery needs a new +membership generation. Old rows do not become active again. -**Acceptance:** a node that is network-isolated but process-healthy is voted dead; old votes left behind by flapping do not wrongly kill a healthy node once they expire. +**Acceptance:** an isolated Silo can be declared dead. Expired votes cannot +later declare a healthy Silo dead. ## Required before an announced release -Not part of any step above, this checklist is the baseline that was completed before the 0.1.0 target was formed. gor currently sits at 0.0.x (publicly visible tags, not announced — see [design/release.md](design/release.md)). The additional 0.1.0 composition, failure-evidence, and usability requirements are tracked in [docs/release-0.1.0.md](docs/release-0.1.0.md). +This checklist was the baseline before the 0.1.0 target. Public 0.0.x tags are +not announced releases. See [design/release.md](design/release.md). The 0.1.0 +requirements are in [docs/release-0.1.0.md](docs/release-0.1.0.md). -- ~~English documentation. Done last — the docs are still changing; translating early means translating twice.~~ **Done.** `README`, `ROADMAP`, `FINDINGS`, `benchmarks.md`, the six `docs/` files, `examples/shadow/README`, and all 17 `design/` files are now English-only, the Chinese originals fully replaced with nothing kept in both languages; `research/`, `CLAUDE.md`, and `.github/PULL_REQUEST_TEMPLATE.md` stay in Chinese as internal evidence and maintainer-facing text — commits and reviews are written in Chinese anyway — and every link to `research/` carries an `(in Chinese)` marker. +- **Public document cleanup complete.** Product and design documents use + English, the current writing rule, and the terms in `CONTEXT.md`. - ~~Public API doc comments. To be completed after step 6c, once the public API is finalized as a release candidate; must meet [design/api-documentation.md](design/api-documentation.md) before `v0.1.0`.~~ **Done.** -- ~~Error and cancellation contract. Stable error codes and the cross-node cancellation boundary must be implemented before `v0.1.0`.~~ **Done.** Spec: [docs/errors.md](docs/errors.md) and [design/errors.md](design/errors.md). A stable code is the only cross-node error identifier. The cancellation boundary is implemented per spec. -- ~~Root runtime shutdown contract. The spec is complete, see [design/runtime.md](design/runtime.md), [design/cluster.md](design/cluster.md), and [docs/programming-model.md](docs/programming-model.md); implementation had not started. Before `v0.1.0`, new calls must stop being admitted during the shutdown window.~~ **Done.** The root runtime's stop state machine and single admission gate are implemented: four transition functions, atomic `admit`/release; the public `Invoke` / inbound handler / scheduled delivery share one gate that sits before the ownership decision and forwarding; `closing→killing` is an escalation, not a no-op; cluster nodes explicitly report their end reason via `DeclaredDead()`; stop coordination uses receive channels only. Transport teardown comes after admitted forwarded requests and inbound replies. Also fixed a real bug where a declared-dead node sent an empty view and triggered a graceful deactivation. -- ~~Deactivation reasons and the background error sink for lifecycle hooks. The spec is complete, see [design/runtime.md](design/runtime.md), [design/timers.md](design/timers.md), and [docs/programming-model.md](docs/programming-model.md); the hooks themselves were implemented, but the deactivation reasons (`DeactivationReason`) and the structured background error sink (`BackgroundError`) were not — both are public API breaking changes. These two must be delivered before `v0.1.0`.~~ **Done.** `OnDeactivate` receives the deactivation reason (idle, ownership lost, normal shutdown, instance untrusted); the reason is fixed at the first transition out of the active state, and the hook gets a work context with no deadline that is never canceled; the background error sink now emits events whose sources are a closed set — scheduled delivery carries the method name, deactivation hook failure carries the deactivation reason, nothing outside the package can add sources, and sources are no longer guessed from method names. Poller scan and claim failures and deliveries canceled mid-shutdown do not enter the sink. Two public API migrations ship with this item (`OnDeactivate` gains a parameter, `OnError` takes an event). -- ~~A real example application, rerun with the new signatures after step 5.5~~ **Done.** See [examples/shadow/](examples/shadow/); design: [docs/example.md](docs/example.md). Its output is [FINDINGS.md](FINDINGS.md) — nine API frictions; the first six went into step 5.5, the README's non-goals, or doc additions; the last three record frictions that still exist. -- ~~Performance baseline numbers and the cross-node forwarding baseline. The numbers in [benchmarks.md](benchmarks.md) were measured before step 6c's probe-config validation landed, and `make bench` failed because `BenchmarkForwardingRoundTrip` did not pass the six probe options that `cluster.New` required (zero values returned `cluster.ErrInvalidConfig`).~~ **Done.** `cluster.New` now implements the default values from the [design/cluster.md](design/cluster.md) parameter table — a zero value means "use the default" (ProbeInterval 1 s, ProbeTimeout 500 ms, ProbeFailures 3, VoteTTL 6 s, MaxTickGap 2 s, MaxTableLatency 500 ms); only negative values are rejected with `ErrInvalidConfig`. `make bench` passes again on a real-disk path, and the documented cluster startup snippet in [docs/programming-model.md](docs/programming-model.md) runs verbatim in a clean module. Re-measured on 2026-08-06: forwarding ns/op values are unchanged code-side — an A/B of the baseline commit under identical machine load overlaps — while allocations moved +1 per local call and +3 per forwarded call; see [benchmarks.md](benchmarks.md). What is measured, what is not, and how conditions are written: [design/benchmarks.md](design/benchmarks.md). -- ~~Observability~~ **Done.** See [design/observability.md](design/observability.md): the runtime provides a snapshot of this node's activations and a completion event per call; no aggregation, export, or alerting. -- Versioning and release. gor uses 0.0.x as a publicly visible but unannounced band: tags are pushed (anyone can `go get` them, the module proxy caches them) but no GitHub Release is created and no release note is assembled. What users can rely on: [docs/compatibility.md](docs/compatibility.md); version numbers, the v1 bar, and the tag/release checklist: [design/release.md](design/release.md). The step-3 install half has been executed on a pseudo-version and passes end to end; the upgrade half was executed for the `v0.0.1` → `v0.0.2` cut (see [design/release.md](design/release.md)). Tagging stays a manual maintainer step. +- **Error and cancellation contract complete.** A stable code is the only + Cross-Silo error identifier. See [docs/errors.md](docs/errors.md) and + [design/errors.md](design/errors.md). +- **Shutdown contract implemented.** The root state machine stops Call + admission and coordinates component shutdown. `Shutdown(ctx)`, `Stopping()`, + and `Done()` are implemented. Activation contexts are Runtime-owned. See + [design/runtime.md](design/runtime.md). +- **Lifecycle and Call error contract implemented.** The API has all five + deactivation reasons, Deactivate on Idle, callback panic containment, and a + structured background error sink. It includes Reminder scan, dispatch, + claim, and Call error sources. See + [design/runtime.md](design/runtime.md) and [design/timers.md](design/timers.md). +- **Example Application complete.** See + [examples/shadow/](examples/shadow/) and [docs/example.md](docs/example.md). +- **Performance baselines complete.** `cluster.New` applies documented defaults. + Local and forwarded Call baselines use real disk. See + [benchmarks.md](benchmarks.md) and + [design/benchmarks.md](design/benchmarks.md). +- **Observability complete.** The Runtime reports local Activations and each + completed Call. See [design/observability.md](design/observability.md). +- **Versioning and release defined.** Public 0.0.x tags are not announced + releases. See [docs/compatibility.md](docs/compatibility.md) and + [design/release.md](design/release.md). Tagging remains a manual step. ## Risks -Only step 6 carries real distributed risk, and it contains no consensus algorithm — Orleans' membership outsources linearizability to a CAS-capable table, and so does `gor`. Steps 1 through 5 contain no distributed invariants; their risk is ordinary engineering risk. - -The real risks are not technical: - -- **Ecosystem risk.** Orbit (EA's JVM virtual-actor implementation, inspired by Orleans) reached 1724 stars and was rewritten in Kotlin once, then was completely abandoned after 2021-06. Projects in this spot have died. -- **Single-person maintenance.** goakt's situation shows how much this hurts credibility. +Cluster preview work has coordination and transport risk. Remaining 0.1.0 work +has release evidence risk. Batch 8 names and tests each release risk. diff --git a/activation_admission_test.go b/activation_admission_test.go new file mode 100644 index 0000000..00f5b6c --- /dev/null +++ b/activation_admission_test.go @@ -0,0 +1,41 @@ +package gor + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "testing/synctest" +) + +func TestRuntime_ActivationLimitUsesPublicOverload(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + factoryCalls := new(atomic.Int32) + rt := mustNew(t, WithMaxActivations(1), WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + installLifecycleAccount(t, rt, factoryCalls, func(*lifecycleAccountGrain) {}) + mustStart(t, rt) + + alice := Ref[lifecycleAccount](rt, "alice") + if _, err := alice.Value(context.Background()); err != nil { + t.Fatalf("first Grain Call error = %v", err) + } + if got := factoryCalls.Load(); got != 1 { + t.Fatalf("factory calls after first Grain = %d, want 1", got) + } + + bob := Ref[lifecycleAccount](rt, "bob") + if _, err := bob.Value(context.Background()); !errors.Is(err, ErrOverloaded) { + t.Fatalf("second Grain Call error = %v, want ErrOverloaded", err) + } else if code, ok := CodeOf(err); !ok || code != ErrOverloaded { + t.Fatalf("CodeOf(second Grain Call) = (%q, %v), want (%q, true)", code, ok, ErrOverloaded) + } + if got := factoryCalls.Load(); got != 1 { + t.Fatalf("factory calls at full limit = %d, want 1", got) + } + + if _, err := alice.Value(context.Background()); err != nil { + t.Fatalf("Call to active Grain error = %v", err) + } + }) +} diff --git a/background_error_test.go b/background_error_test.go index fc5fb3f..6f997eb 100644 --- a/background_error_test.go +++ b/background_error_test.go @@ -13,30 +13,30 @@ import ( "github.com/suraciii/gor/store" ) -// failingReminderStore fails schedule listing or claiming on demand. It exists -// to pin the boundary of the background error exit: list and claim failures -// are scheduler state, not application callback failures. +// failingReminderStore fails the next schedule listing or claim on demand. type failingReminderStore struct { *store.Memory failList atomic.Bool failClaim atomic.Bool + listErr error + claimErr error } -func (s *failingReminderStore) ListDue(ctx context.Context, now time.Time) ([]store.Reminder, error) { - if s.failList.Load() { - return nil, errors.New("simulated list failure") +func (s *failingReminderStore) ListDue(ctx context.Context, now time.Time, after *store.ReminderCursor, limit int) (store.ReminderPage, error) { + if s.failList.Swap(false) { + return store.ReminderPage{}, s.listErr } - return s.Memory.ListDue(ctx, now) + return s.Memory.ListDue(ctx, now, after, limit) } func (s *failingReminderStore) Claim(ctx context.Context, schedule store.Reminder, nextDueAt time.Time) (bool, error) { - if s.failClaim.Load() { - return false, errors.New("simulated claim failure") + if s.failClaim.Swap(false) { + return false, s.claimErr } return s.Memory.Claim(ctx, schedule, nextDueAt) } -// misnamedSchedule is an entity whose interface contains a method literally +// misnamedSchedule is a Grain whose interface contains a method literally // named OnDeactivate. Its signature differs from the Deactivatable hook, so it // does not implement Deactivatable; a scheduled failure of this method must // still be reported as a ReminderInvocation, not as a Deactivation. @@ -52,7 +52,7 @@ type misnamedScheduleDeactivateRequest struct { } type misnamedScheduleDeactivateReply struct{} -type misnamedScheduleEntity struct { +type misnamedScheduleGrain struct { schedule Reminder[misnamedSchedule] wakeErr error } @@ -62,11 +62,11 @@ type misnamedScheduleProxy struct { id GrainId } -func (e *misnamedScheduleEntity) Arm(ctx context.Context) error { +func (e *misnamedScheduleGrain) Arm(ctx context.Context) error { return e.schedule.Set(ctx, "wake", After(12*time.Second), Handle(misnamedSchedule.OnDeactivate)) } -func (e *misnamedScheduleEntity) OnDeactivate(context.Context, TickStatus) error { +func (e *misnamedScheduleGrain) OnDeactivate(context.Context, TickStatus) error { return e.wakeErr } @@ -109,13 +109,13 @@ func newMisnamedScheduleReminderCall(method string, status TickStatus) (any, any func installMisnamedSchedule(t *testing.T, rt *Runtime, wakeErr error) { t.Helper() - if err := InstallType[misnamedSchedule](rt, dispatchMisnamedSchedule, func(invoker Invoker, id GrainId) misnamedSchedule { + if err := InstallType[misnamedSchedule](rt, GeneratedCodeVersion, "gor.misnamedSchedule", dispatchMisnamedSchedule, func(invoker Invoker, id GrainId) misnamedSchedule { return &misnamedScheduleProxy{invoker: invoker, id: id} }, newMisnamedScheduleCall, newMisnamedScheduleReminderCall); err != nil { t.Fatal(err) } - if err := Register[misnamedSchedule](rt, func(b *Binder) misnamedSchedule { - return &misnamedScheduleEntity{schedule: NewReminder[misnamedSchedule](b), wakeErr: wakeErr} + if err := Register[misnamedSchedule](rt, func(b *GrainContext) misnamedSchedule { + return &misnamedScheduleGrain{schedule: NewReminder[misnamedSchedule](b), wakeErr: wakeErr} }); err != nil { t.Fatal(err) } @@ -140,8 +140,9 @@ func TestBackgroundError_ScheduledMethodNamedOnDeactivate(t *testing.T) { errorsSeen <- event }), ) - defer rt.Close() + defer closeRuntime(rt) installMisnamedSchedule(t, rt, wakeErr) + mustStart(t, rt) if err := Ref[misnamedSchedule](rt, "alice").Arm(context.Background()); err != nil { t.Fatalf("Arm: %v", err) @@ -151,7 +152,7 @@ func TestBackgroundError_ScheduledMethodNamedOnDeactivate(t *testing.T) { select { case got := <-errorsSeen: - wantID := GrainId{GrainType: TypeName[misnamedSchedule](), GrainKey: "alice"} + wantID := GrainId{GrainType: GrainType("gor.misnamedSchedule"), GrainKey: "alice"} source, ok := got.Source.(ReminderInvocation) if !ok { t.Fatalf("source = %#v, want ReminderInvocation", got.Source) @@ -186,10 +187,11 @@ func TestBackgroundError_CancelShapedErrorFromLivePoller(t *testing.T) { errorsSeen <- event }), ) - defer rt.Close() + defer closeRuntime(rt) installScheduledAccount(t, rt, new(atomic.Int32), scheduledAccountConfig{cancelShapedErr: true}) + mustStart(t, rt) - if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background()); err != nil { + if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background(), "wake"); err != nil { t.Fatalf("Arm: %v", err) } fakeClock.Advance(12 * time.Second) @@ -224,15 +226,16 @@ func TestBackgroundError_DeactivationCarriesStopReason(t *testing.T) { errorsSeen <- event }), ) - installLifecycleAccount(t, rt, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.deactivateErr = deactivateErr + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateErr = deactivateErr }) + mustStart(t, rt) - id := GrainId{GrainType: TypeName[lifecycleAccount](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: "alice"} if err := rt.Invoke(context.Background(), id, "Value", &lifecycleAccountValueRequest{}, &lifecycleAccountValueReply{}); err != nil { t.Fatalf("initial Value: %v", err) } - rt.Close() + closeRuntime(rt) select { case got := <-errorsSeen: @@ -249,17 +252,79 @@ func TestBackgroundError_DeactivationCarriesStopReason(t *testing.T) { }) } -// TestBackgroundError_ScheduleFaultsAreSilent pins the boundary that list and -// claim failures are scheduler and store state, not application callback -// failures: neither produces an event. -func TestBackgroundError_ScheduleFaultsAreSilent(t *testing.T) { +func TestBackgroundError_DeactivationPanicIsContained(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + errorsSeen := make(chan BackgroundError, 1) + rt := mustNew(t, + WithIdleTimeout(0), + WithEvictionInterval(0), + OnError(func(event BackgroundError) { + errorsSeen <- event + }), + ) + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivatePanic = true + }) + mustStart(t, rt) + + id := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: "alice"} + if err := rt.Invoke(context.Background(), id, "Value", &lifecycleAccountValueRequest{}, &lifecycleAccountValueReply{}); err != nil { + t.Fatalf("initial Value: %v", err) + } + closeRuntime(rt) + + select { + case got := <-errorsSeen: + source, ok := got.Source.(Deactivation) + if !ok || got.GrainId != id || source.Reason != RuntimeClosed || !errors.Is(got.Err, ErrPanic) { + t.Fatalf("event = %#v, want contained RuntimeClosed deactivation panic", got) + } + default: + t.Fatal("deactivation panic did not reach OnError") + } + assertChannelClosed(t, "Done after deactivation panic", rt.Done()) + }) +} + +func TestBackgroundError_OnErrorPanicDoesNotStopCleanup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var observations atomic.Int32 + rt := mustNew(t, + WithIdleTimeout(0), + WithEvictionInterval(0), + OnError(func(BackgroundError) { + observations.Add(1) + panic("error observer failure") + }), + ) + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateErr = errors.New("deactivate failed") + }) + mustStart(t, rt) + + id := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: "alice"} + if err := rt.Invoke(context.Background(), id, "Value", &lifecycleAccountValueRequest{}, &lifecycleAccountValueReply{}); err != nil { + t.Fatalf("initial Value: %v", err) + } + closeRuntime(rt) + if got := observations.Load(); got != 1 { + t.Fatalf("OnError calls = %d, want 1 without recursive reporting", got) + } + assertChannelClosed(t, "Done after OnError panic", rt.Done()) + }) +} + +func TestBackgroundError_ScheduleStoreFailuresAreReported(t *testing.T) { synctest.Test(t, func(t *testing.T) { start := time.Unix(0, 0).UTC() fakeClock := clock.NewFake(start) - backend := &failingReminderStore{Memory: store.NewMemory()} - errorsSeen := make(chan BackgroundError, 1) + listErr := errors.New("simulated list failure") + claimErr := errors.New("simulated claim failure") + backend := &failingReminderStore{Memory: store.NewMemory(), listErr: listErr, claimErr: claimErr} + errorsSeen := make(chan BackgroundError, 2) rt := mustNew(t, WithStore(backend), + WithReminderStore(backend), WithClock(fakeClock), WithIdleTimeout(0), WithEvictionInterval(0), @@ -268,10 +333,11 @@ func TestBackgroundError_ScheduleFaultsAreSilent(t *testing.T) { errorsSeen <- event }), ) - defer rt.Close() + defer closeRuntime(rt) installScheduledAccount(t, rt, new(atomic.Int32), scheduledAccountConfig{}) + mustStart(t, rt) - if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background()); err != nil { + if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background(), "wake"); err != nil { t.Fatalf("Arm: %v", err) } @@ -280,28 +346,29 @@ func TestBackgroundError_ScheduleFaultsAreSilent(t *testing.T) { synctest.Wait() select { case got := <-errorsSeen: - t.Fatalf("OnError received an event for a list failure: %#v", got) + if _, ok := got.Source.(ReminderScan); !ok || got.GrainId != (GrainId{}) || !errors.Is(got.Err, listErr) { + t.Fatalf("list failure event = %#v, want ReminderScan and original error", got) + } default: + t.Fatal("list failure did not reach OnError") } - backend.failList.Store(false) backend.failClaim.Store(true) - fakeClock.Advance(12 * time.Second) + fakeClock.Advance(time.Second) synctest.Wait() select { case got := <-errorsSeen: - t.Fatalf("OnError received an event for a claim failure: %#v", got) + source, ok := got.Source.(ReminderClaim) + wantID := GrainId{GrainType: GrainType("gor.scheduledAccount"), GrainKey: "alice"} + if !ok || got.GrainId != wantID || source.Name != "wake" || !errors.Is(got.Err, claimErr) { + t.Fatalf("claim failure event = %#v, want ReminderClaim for %v and original error", got, wantID) + } default: + t.Fatal("claim failure did not reach OnError") } - backend.failList.Store(false) - backend.failClaim.Store(false) - fakeClock.Advance(12 * time.Second) + fakeClock.Advance(time.Second) synctest.Wait() - // Positive control: with the faults cleared, the same poller must - // deliver the row that stayed due through both silent phases. The - // no-event assertions above are observations of a running poller, not - // of a stopped one. if got, err := Ref[scheduledAccount](rt, "alice").Value(context.Background()); err != nil { t.Fatalf("Value: %v", err) } else if got != 1 { @@ -331,14 +398,15 @@ func TestBackgroundError_CanceledDeliveryNotReported(t *testing.T) { }), ) installScheduledAccount(t, rt, new(atomic.Int32), scheduledAccountConfig{wakeStarted: wakeStarted}) - installLifecycleAccount(t, rt, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.deactivateErr = deactivateErr + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateErr = deactivateErr }) + mustStart(t, rt) - if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background()); err != nil { + if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background(), "wake"); err != nil { t.Fatalf("Arm: %v", err) } - id := GrainId{GrainType: TypeName[lifecycleAccount](), GrainKey: "bob"} + id := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: "bob"} if err := rt.Invoke(context.Background(), id, "Value", &lifecycleAccountValueRequest{}, &lifecycleAccountValueReply{}); err != nil { t.Fatalf("initial Value: %v", err) } @@ -350,7 +418,7 @@ func TestBackgroundError_CanceledDeliveryNotReported(t *testing.T) { t.Fatal("scheduled Wake did not start") } - rt.Close() + closeRuntime(rt) synctest.Wait() // The only event is the hook failure of the graceful close. The diff --git a/benchmark_filesystem_linux_test.go b/benchmark_filesystem_linux_test.go new file mode 100644 index 0000000..6ae7b7f --- /dev/null +++ b/benchmark_filesystem_linux_test.go @@ -0,0 +1,13 @@ +//go:build linux + +package gor + +import "syscall" + +func benchmarkFileSystemMagic(path string) (uint64, bool, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(path, &stat); err != nil { + return 0, false, err + } + return uint64(stat.Type), true, nil +} diff --git a/benchmark_filesystem_other_test.go b/benchmark_filesystem_other_test.go new file mode 100644 index 0000000..edd09f1 --- /dev/null +++ b/benchmark_filesystem_other_test.go @@ -0,0 +1,7 @@ +//go:build !linux + +package gor + +func benchmarkFileSystemMagic(string) (uint64, bool, error) { + return 0, false, nil +} diff --git a/benchmark_test.go b/benchmark_test.go index 0532eb2..21afcd6 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -6,7 +6,6 @@ import ( "os" "path/filepath" "runtime" - "syscall" "testing" "time" @@ -16,16 +15,16 @@ import ( "github.com/suraciii/gor/transport" ) -type benchmarkEntity interface { +type benchmarkGrain interface { Noop(context.Context) error Seed(context.Context) error } -type benchmarkEntityImpl struct { +type benchmarkGrainImpl struct { state State[uint64] } -type benchmarkEntityProxy struct { +type benchmarkGrainProxy struct { invoker Invoker id GrainId } @@ -35,23 +34,23 @@ type benchmarkNoopReply struct{} type benchmarkSeedRequest struct{} type benchmarkSeedReply struct{} -func (e *benchmarkEntityImpl) Noop(context.Context) error { +func (e *benchmarkGrainImpl) Noop(context.Context) error { return nil } -func (e *benchmarkEntityImpl) Seed(ctx context.Context) error { +func (e *benchmarkGrainImpl) Seed(ctx context.Context) error { return e.state.Set(ctx, 1) } -func (p *benchmarkEntityProxy) Noop(ctx context.Context) error { +func (p *benchmarkGrainProxy) Noop(ctx context.Context) error { return p.invoker.Invoke(ctx, p.id, "Noop", nil, nil) } -func (p *benchmarkEntityProxy) Seed(ctx context.Context) error { +func (p *benchmarkGrainProxy) Seed(ctx context.Context) error { return p.invoker.Invoke(ctx, p.id, "Seed", nil, nil) } -func dispatchBenchmarkEntity(ctx context.Context, instance benchmarkEntity, method string, _ any, _ any) error { +func dispatchBenchmarkGrain(ctx context.Context, instance benchmarkGrain, method string, _ any, _ any) error { switch method { case "Noop": return instance.Noop(ctx) @@ -62,7 +61,7 @@ func dispatchBenchmarkEntity(ctx context.Context, instance benchmarkEntity, meth } } -func newBenchmarkEntityCall(method string) (args any, reply any) { +func newBenchmarkGrainCall(method string) (args any, reply any) { switch method { case "Noop": return &benchmarkNoopRequest{}, &benchmarkNoopReply{} @@ -77,6 +76,7 @@ func newBenchmarkRuntime(b *testing.B, backend store.Store, sourceClock clock.Cl b.Helper() options = append([]Option{ WithStore(backend), + WithReminderStore(store.NewMemory()), WithClock(sourceClock), WithIdleTimeout(idleTimeout), WithEvictionInterval(evictionInterval), @@ -85,21 +85,24 @@ func newBenchmarkRuntime(b *testing.B, backend store.Store, sourceClock clock.Cl if err != nil { b.Fatal(err) } - installBenchmarkEntity(b, rt) - b.Cleanup(rt.Close) + installBenchmarkGrain(b, rt) + if err := rt.Start(context.Background()); err != nil { + b.Fatal(err) + } + b.Cleanup(func() { closeRuntime(rt) }) return rt } -func installBenchmarkEntity(b *testing.B, rt *Runtime) { +func installBenchmarkGrain(b *testing.B, rt *Runtime) { b.Helper() - if err := InstallType[benchmarkEntity](rt, dispatchBenchmarkEntity, func(invoker Invoker, id GrainId) benchmarkEntity { - return &benchmarkEntityProxy{invoker: invoker, id: id} - }, newBenchmarkEntityCall, nil); err != nil { - rt.Close() + if err := InstallType[benchmarkGrain](rt, GeneratedCodeVersion, "gor.benchmarkGrain", dispatchBenchmarkGrain, func(invoker Invoker, id GrainId) benchmarkGrain { + return &benchmarkGrainProxy{invoker: invoker, id: id} + }, newBenchmarkGrainCall, noReminderCall); err != nil { + closeRuntime(rt) b.Fatal(err) } - if err := Register[benchmarkEntity](rt, func(binder *Binder) benchmarkEntity { - return &benchmarkEntityImpl{state: NewState[uint64](binder, "value")} + if err := Register[benchmarkGrain](rt, func(grainContext *GrainContext) benchmarkGrain { + return &benchmarkGrainImpl{state: NewState[uint64](grainContext, "value")} }); err != nil { b.Fatal(err) } @@ -107,15 +110,15 @@ func installBenchmarkEntity(b *testing.B, rt *Runtime) { func BenchmarkInvocationRoundTrip(b *testing.B) { rt := newBenchmarkRuntime(b, store.NewMemory(), clock.Real{}, 0, 0) - entity := Ref[benchmarkEntity](rt, "benchmark") - if err := entity.Noop(context.Background()); err != nil { + grain := Ref[benchmarkGrain](rt, "benchmark") + if err := grain.Noop(context.Background()); err != nil { b.Fatal(err) } b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - if err := entity.Noop(context.Background()); err != nil { + if err := grain.Noop(context.Background()); err != nil { b.Fatal(err) } } @@ -123,15 +126,15 @@ func BenchmarkInvocationRoundTrip(b *testing.B) { func BenchmarkInvocationRoundTripWithOnCall(b *testing.B) { rt := newBenchmarkRuntime(b, store.NewMemory(), clock.Real{}, 0, 0, OnCall(func(CallObservation) {})) - entity := Ref[benchmarkEntity](rt, "benchmark") - if err := entity.Noop(context.Background()); err != nil { + grain := Ref[benchmarkGrain](rt, "benchmark") + if err := grain.Noop(context.Background()); err != nil { b.Fatal(err) } b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - if err := entity.Noop(context.Background()); err != nil { + if err := grain.Noop(context.Background()); err != nil { b.Fatal(err) } } @@ -139,8 +142,8 @@ func BenchmarkInvocationRoundTripWithOnCall(b *testing.B) { func BenchmarkForwardingRoundTrip(b *testing.B) { first, _, localID, remoteID := newBenchmarkForwardingRuntimes(b) - local := Ref[benchmarkEntity](first, localID.GrainKey) - remote := Ref[benchmarkEntity](first, remoteID.GrainKey) + local := Ref[benchmarkGrain](first, localID.GrainKey) + remote := Ref[benchmarkGrain](first, remoteID.GrainKey) if err := local.Noop(context.Background()); err != nil { b.Fatal(err) } @@ -148,12 +151,12 @@ func BenchmarkForwardingRoundTrip(b *testing.B) { b.Fatal(err) } - benchmarkInvocation := func(b *testing.B, entity benchmarkEntity) { + benchmarkInvocation := func(b *testing.B, grain benchmarkGrain) { b.Helper() b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - if err := entity.Noop(context.Background()); err != nil { + if err := grain.Noop(context.Background()); err != nil { b.Fatal(err) } } @@ -186,6 +189,7 @@ func newBenchmarkForwardingRuntimes(b *testing.B) (*Runtime, *Runtime, GrainId, newNode := func(nodeTransport *transport.TCP, generation string) *Runtime { rt, err := New( WithStore(backend), + WithReminderStore(backend), WithMemberStore(members), WithNodeAddr(nodeTransport.Addr()), WithGeneration(generation), @@ -200,8 +204,11 @@ func newBenchmarkForwardingRuntimes(b *testing.B) (*Runtime, *Runtime, GrainId, if err != nil { b.Fatal(err) } - installBenchmarkEntity(b, rt) - b.Cleanup(rt.Close) + installBenchmarkGrain(b, rt) + if err := rt.Start(context.Background()); err != nil { + b.Fatal(err) + } + b.Cleanup(func() { closeRuntime(rt) }) return rt } first := newNode(firstTransport, "benchmark-first") @@ -215,8 +222,8 @@ func newBenchmarkForwardingRuntimes(b *testing.B) (*Runtime, *Runtime, GrainId, var localID, remoteID GrainId for index := 0; index < 4096; index++ { - id := GrainId{GrainType: TypeName[benchmarkEntity](), GrainKey: fmt.Sprintf("forward-%04d", index)} - owner, ok := cluster.Owner(view, store.GrainId(id)) + id := GrainId{GrainType: GrainType("gor.benchmarkGrain"), GrainKey: fmt.Sprintf("forward-%04d", index)} + owner, ok := cluster.Owner(view, toStoreGrainID(id)) if !ok { continue } @@ -240,10 +247,10 @@ func BenchmarkStateWrite(b *testing.B) { if err != nil { b.Fatal(err) } - b.Cleanup(func() { rt.Close() }) + b.Cleanup(func() { closeRuntime(rt) }) - binder := newBinder(rt, GrainId{GrainType: "benchmark", GrainKey: "state"}) - state := NewState[uint64](binder, "value") + grainContext := newGrainContext(rt, GrainId{GrainType: "benchmark", GrainKey: "state"}) + state := NewState[uint64](grainContext, "value") if err := state.Set(context.Background(), 0); err != nil { b.Fatal(err) } @@ -261,8 +268,8 @@ func BenchmarkColdActivation(b *testing.B) { database := openBenchmarkSQLite(b, "cold-activation.db") fakeClock := clock.NewFake(time.Unix(0, 0).UTC()) rt := newBenchmarkRuntime(b, database, fakeClock, time.Second, time.Second) - entity := Ref[benchmarkEntity](rt, "benchmark") - if err := entity.Seed(context.Background()); err != nil { + grain := Ref[benchmarkGrain](rt, "benchmark") + if err := grain.Seed(context.Background()); err != nil { b.Fatal(err) } @@ -272,7 +279,7 @@ func BenchmarkColdActivation(b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - if err := entity.Noop(context.Background()); err != nil { + if err := grain.Noop(context.Background()); err != nil { b.Fatal(err) } b.StopTimer() @@ -326,15 +333,18 @@ func benchmarkRealDiskDir(b *testing.B) string { } }) - var stat syscall.Statfs_t - if err := syscall.Statfs(dir, &stat); err != nil { + magic, known, err := benchmarkFileSystemMagic(dir) + if err != nil { b.Fatal(err) } + if !known { + b.Log("benchmark data path: file-system type check is not available on this system") + return dir + } const ( tmpfsSuperMagic = 0x01021994 ramfsSuperMagic = 0x858458f6 ) - magic := uint64(stat.Type) b.Logf("benchmark data path: statfs magic %#x", magic) if magic == tmpfsSuperMagic || magic == ramfsSuperMagic { b.Fatalf("benchmark data path %q is on an in-memory filesystem (statfs magic %#x); use real disk storage", dir, magic) diff --git a/benchmarks.md b/benchmarks.md index 2dda732..76c4cef 100644 --- a/benchmarks.md +++ b/benchmarks.md @@ -22,20 +22,63 @@ Measured on 2026-08-05. The results below come from a single `make bench` run, w With observation disabled, results are about `-0.5%` relative to the existing `0.89 us/op` baseline; an empty callback adds `7.5%` over the disabled state. Forwarding adds about `20513 ns/op` over a same-condition local call — about `24.2` times a local call. The forwarding row uses real loopback TCP to include 6b's framing, JSON encoding/decoding, connection reuse, and handler path; the local and forwarded calls invoke the same `Noop` method on the same type, and the connection and activation are warmed up before timing. This number is not machine-to-machine network latency; it only answers the library-internal extra cost of forwarding over local. -Re-verified on 2026-08-06: the ns/op values are unchanged code-side. The machine was under sustained load (load average ≈ 17, including an inference server at ≈ 1500% CPU), so absolute numbers came out about 2.3 times higher; an A/B comparison of the baseline commit `0678933` with the current HEAD under that identical load shows overlapping distributions (local 2.0–2.2 us, forwarded 41–48 us), so the recorded idle-machine values remain the formal baseline. Allocations did move: +1 per local call and +3 per forwarded call (624 B and 2199 B total before, 640 B and 2250 B now), from the admission gate and the error-envelope work that landed after the original baseline. `BenchmarkStateWrite` reproduces at 1.8 ms/op unchanged; cold activation measured 40 us/op under load against 18 us/op recorded, consistent with its documented load sensitivity. +The 2026-08-06 check ran under sustained load. The load average was about 17, +and an inference server used about 1500% CPU. Absolute results were about 2.3 +times higher. An A/B check compared commit `0678933` with the current commit +under that load. The distributions overlapped: local was 2.0-2.2 us and +forwarded was 41-48 us. Thus, the 2026-08-05 idle values remained the baseline +at that time. + +Allocations changed after that baseline. A local Call added one allocation, +from 624 B to 640 B. A forwarded Call added three allocations, from 2199 B to +2250 B. The later admission gate and error envelope caused these changes. +`BenchmarkStateWrite` stayed at 1.8 ms/op. Cold Activation was 40 us/op under +load, compared with the recorded 18 us/op idle result. + +## 0.1.0 candidate Call check + +Measured on 2026-08-11. The candidate was commit `ab9ec70`. The comparison +commit was `0678933`, which added `OnCall`. Both commits used this command: + +```sh +go test . -run '^$' -bench '^BenchmarkInvocationRoundTrip' -benchmem -count=10 +``` + +The machine had a load average of 5.14 on 32 logical CPUs before the run. +OpenCode, `VBCSCompiler`, and `llama-server` used CPU during the run. Thus, +these values are an A/B check under ambient load. They do not replace the +idle-machine baseline. + +| Commit and path | Median ns/op | Observed range ns/op | B/op | allocs/op | +| --- | ---: | ---: | ---: | ---: | +| `ab9ec70`, disabled | 1,254.5 | 1,147-1,574 | 760 | 14 | +| `ab9ec70`, empty `OnCall` | 1,354.5 | 1,256-3,465 | 760 | 14 | +| `0678933`, disabled | 1,067.0 | 1,004-2,249 | 624 | 10 | +| `0678933`, empty `OnCall` | 1,148.0 | 1,128-1,212 | 624 | 10 | + +The 8.0% value compares the two candidate medians. The empty `OnCall` callback +added no allocation. It meets the 20% `OnCall` cost limit. The complete +disabled Call path was 17.6% slower than the comparison commit and used four +more allocations. The candidate contains the later 0.1.0 reliability work. +This A/B run cannot assign the total increase to one change. + +This table is the current candidate Call record under its stated ambient +conditions. The 2026-08-05 result remains the idle comparison. Re-measure +final `master` on an idle machine when one is available. This is performance +maintenance, not a Batch 8e release gate. ## Measurement conditions - CPU: AMD Ryzen 9 9955HX 16-Core Processor, 16 physical cores, 32 logical CPUs. - Disk: CT2000P310SSD8 NVMe SSD, non-rotating; the benchmark directory is on ext4 at `/dev/mapper/ubuntu--vg-ubuntu--lv`. - Go: `go1.26.5 linux/amd64`. -- Concurrency: one entity, one key, single-goroutine serial calls per benchmark; no concurrent workers. +- Concurrency: one Grain, one key, and serial Calls in one goroutine per benchmark. There are no concurrent workers. - For formal reproduction, take numbers on an idle machine; do not run builds or tests at the same time. Background load raises cold activation noticeably — with concurrent build and test activity it can approach doubling. At the time of measurement no Go/.NET build or test commands were active, but an independent long-running `VBCSCompiler` service was present; it was not terminated. - State write and cold activation use the same real-disk SQLite configuration: WAL. The Full tier and cold activation run `synchronous=FULL`; the Relaxed tier runs `synchronous=NORMAL`, which in WAL mode syncs at checkpoint instead of on every commit. Re-measured on 2026-08-07 (store-level, `store/benchmark_test.go`): the State write rows are re-measurements on this machine under its usual background load — long-running services are present, so these are not clean-idle numbers. At run time the load average was ≈ 1.0 on 32 logical CPUs and no Go build or test was active. Full measured 1.74 ms/op, inside the band of the frozen 1.8 ms/op baseline (three 2 s runs the same day span 1.71–2.09 ms/op); it replaces the baseline as a re-measurement under ambient load, not as a cleaner one. Relaxed measured 13.8 us/op. Both rows were measured with the default `benchtime` on ext4 via `GOR_BENCH_DIR=/home/szf/gor-bench` (statfs magic 0xef53, not tmpfs). -- The observability benchmarks use one entity, one key, single-goroutine serial calls; the forwarding benchmarks use two in-process runtimes, in-memory stores, and two real loopback TCP listeners. -- Before cold activation starts, the entity's `Seed` writes a `gor.State`; after the entity is evicted, the first timed call reads this state back from disk in the factory's `binder.load(ctx)`. That disk read-back is included in the cold-activation numbers; the method itself writes no additional state. +- The observability benchmarks use one Grain, one key, and serial Calls. The forwarding benchmarks use two in-process Runtimes, in-memory Stores, and two real loopback TCP listeners. +- Before cold activation starts, the Grain's `Seed` method writes a `gor.State`. After the Grain is evicted, the first timed Call reads this State from disk during Activation. The cold-activation result includes this disk read. The method does not write more State. - Each benchmark uses a `.gor-bench-*` temp directory under the working directory. `GOR_BENCH_DIR` must point at a real-disk directory; tmpfs and ramfs are rejected by a guard. ## Reproduction commands diff --git a/cluster/cluster.go b/cluster/cluster.go index ceeb48e..841a8e3 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -3,7 +3,7 @@ // // It is an implementation package, not an application dependency. Configure // clustering through gor.New and its cluster options, and use the root gor -// package for entity access. +// package for Grain access. package cluster import ( diff --git a/cluster/node.go b/cluster/node.go index cd06d95..0758ae6 100644 --- a/cluster/node.go +++ b/cluster/node.go @@ -76,6 +76,8 @@ type Node struct { ctx context.Context cancel context.CancelFunc + leaveCtx context.Context + cancelLeave context.CancelFunc done chan struct{} declaredDead chan struct{} views chan View @@ -87,6 +89,15 @@ type Node struct { } func New(config Config) (*Node, error) { + return NewContext(context.Background(), config) +} + +// NewContext joins the cluster within ctx. Work after a successful join uses +// the Node-owned context and does not retain the startup deadline. +func NewContext(ctx context.Context, config Config) (*Node, error) { + if ctx == nil { + ctx = context.Background() + } if config.Prober == nil { return nil, ErrProberRequired } @@ -111,7 +122,8 @@ func New(config Config) (*Node, error) { if config.MaxTableLatency == 0 { config.MaxTableLatency = defaultMaxTableLatency } - ctx, cancel := context.WithCancel(context.Background()) + runCtx, cancel := context.WithCancel(context.Background()) + leaveCtx, cancelLeave := context.WithCancel(context.Background()) node := &Node{ table: config.Table, clock: config.Clock, @@ -126,8 +138,10 @@ func New(config Config) (*Node, error) { voteTTL: config.VoteTTL, maxTickGap: config.MaxTickGap, maxTableLatency: config.MaxTableLatency, - ctx: ctx, + ctx: runCtx, cancel: cancel, + leaveCtx: leaveCtx, + cancelLeave: cancelLeave, done: make(chan struct{}), declaredDead: make(chan struct{}), views: make(chan View, 1), @@ -135,9 +149,10 @@ func New(config Config) (*Node, error) { } node.state.Store(uint32(StateJoining)) - self, members, err := node.join() + self, members, err := node.join(ctx) if err != nil { cancel() + cancelLeave() return nil, err } node.state.Store(uint32(StateActive)) @@ -185,24 +200,53 @@ func (n *Node) Close() { <-n.done } +// CancelStart stops a Node that joined but whose owning Runtime did not finish +// startup. It gives the clean leave write one table-latency interval, then +// interrupts that write and stops abruptly. +func (n *Node) CancelStart() { + n.cancel() + ticker := n.clock.NewTicker(n.maxTableLatency) + defer ticker.Stop() + select { + case <-n.done: + return + case <-ticker.C(): + n.Kill() + } +} + func (n *Node) Kill() { n.state.Store(uint32(StateDead)) + n.cancelLeave() n.cancel() <-n.done } -func (n *Node) join() (store.Member, []store.Member, error) { +func (n *Node) join(ctx context.Context) (_ store.Member, _ []store.Member, resultErr error) { self := store.Member{ NodeAddr: n.nodeAddr, Generation: n.generation, Status: store.MemberJoining, IamAliveAt: n.clock.Now(), } - if _, err := n.table.WriteMember(context.Background(), self); err != nil { + etag, err := n.table.WriteMember(ctx, self) + if err != nil { + if cleanupErr := n.cleanupInitialJoinWrite(self); cleanupErr != nil { + err = errors.Join(err, cleanupErr) + } return store.Member{}, nil, err } + self.ETag = etag + defer func() { + if resultErr == nil { + return + } + if cleanupErr := n.cleanupJoinMember(self); cleanupErr != nil { + resultErr = errors.Join(resultErr, cleanupErr) + } + }() - snapshot, err := n.table.ListMembers(context.Background()) + snapshot, err := n.table.ListMembers(ctx) if err != nil { return store.Member{}, nil, err } @@ -215,7 +259,7 @@ func (n *Node) join() (store.Member, []store.Member, error) { self = members[index] self.Status = store.MemberActive self.IamAliveAt = n.clock.Now() - etag, err := n.table.WriteMember(context.Background(), self) + etag, err = n.table.WriteMember(ctx, self) if err != nil { if errors.Is(err, store.ErrConflict) { return store.Member{}, nil, ErrNodeDead @@ -227,12 +271,46 @@ func (n *Node) join() (store.Member, []store.Member, error) { return self, members, nil } +func (n *Node) cleanupInitialJoinWrite(attempt store.Member) error { + snapshot, err := n.listMembersForJoinCleanup() + if err != nil { + return err + } + index := memberIndex(snapshot.Members, attempt) + if index < 0 { + return nil + } + current := snapshot.Members[index] + if current.Status != store.MemberJoining || !current.IamAliveAt.Equal(attempt.IamAliveAt) { + return nil + } + current.Status = store.MemberDead + _, err = n.writeMemberForJoinCleanup(current) + return err +} + +func (n *Node) cleanupJoinMember(self store.Member) error { + snapshot, err := n.listMembersForJoinCleanup() + if err != nil { + return err + } + index := memberIndex(snapshot.Members, self) + if index < 0 || snapshot.Members[index].Status == store.MemberDead { + return nil + } + current := snapshot.Members[index] + current.Status = store.MemberDead + _, err = n.writeMemberForJoinCleanup(current) + return err +} + func (n *Node) run(self store.Member, view View, heartbeat, viewTicker, probeTicker clock.Ticker) { defer heartbeat.Stop() defer viewTicker.Stop() defer probeTicker.Stop() defer close(n.views) defer close(n.done) + defer n.cancelLeave() var externalDeath bool defer func() { if externalDeath { @@ -426,6 +504,58 @@ func (n *Node) writeMemberForCheck(member store.Member) (store.ETag, error) { } } +func (n *Node) writeMemberForJoinCleanup(member store.Member) (store.ETag, error) { + ctx, cancel := context.WithCancelCause(context.Background()) + finished := make(chan struct{}) + watchDone := make(chan struct{}) + ticker := n.clock.NewTicker(n.maxTableLatency) + go func() { + defer close(watchDone) + select { + case <-ticker.C(): + cancel(errMemberCheckTimeout) + case <-finished: + } + }() + + etag, err := n.table.WriteMember(ctx, member) + close(finished) + <-watchDone + ticker.Stop() + cause := context.Cause(ctx) + cancel(nil) + if err != nil && errors.Is(cause, errMemberCheckTimeout) { + return 0, errMemberCheckTimeout + } + return etag, err +} + +func (n *Node) listMembersForJoinCleanup() (store.MemberSnapshot, error) { + ctx, cancel := context.WithCancelCause(context.Background()) + finished := make(chan struct{}) + watchDone := make(chan struct{}) + ticker := n.clock.NewTicker(n.maxTableLatency) + go func() { + defer close(watchDone) + select { + case <-ticker.C(): + cancel(errMemberCheckTimeout) + case <-finished: + } + }() + + snapshot, err := n.table.ListMembers(ctx) + close(finished) + <-watchDone + ticker.Stop() + cause := context.Cause(ctx) + cancel(nil) + if err != nil && errors.Is(cause, errMemberCheckTimeout) { + return store.MemberSnapshot{}, errMemberCheckTimeout + } + return snapshot, err +} + func (n *Node) checkProbeInterval(probeAt time.Time) bool { previous := n.lastProbeAt n.lastProbeAt = probeAt @@ -507,7 +637,7 @@ func (n *Node) leave(self store.Member) { return } self.Status = store.MemberDead - _, _ = n.table.WriteMember(context.Background(), self) + _, _ = n.table.WriteMember(n.leaveCtx, self) n.state.Store(uint32(StateDead)) } diff --git a/cluster/node_test.go b/cluster/node_test.go index 03ff274..63ceffa 100644 --- a/cluster/node_test.go +++ b/cluster/node_test.go @@ -55,6 +55,137 @@ func TestNodeJoinWritesJoiningReadsThenActivates(t *testing.T) { }) } +func TestNodeJoinUsesStartupContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + backend := &startupContextMemberStore{} + + _, err := NewContext(ctx, testNodeConfig(backend, clock.NewFake(time.Unix(150, 0).UTC()), "node-a", "generation-a")) + if !errors.Is(err, context.Canceled) { + t.Fatalf("NewContext error = %v, want context.Canceled", err) + } + if got := backend.writes.Load(); got != 1 { + t.Fatalf("join writes = %d, want one canceled attempt", got) + } + if got := backend.lists.Load(); got != 1 { + t.Fatalf("member lists = %d, want one cleanup check after canceled write", got) + } +} + +func TestNodeCanceledJoinMarksJoiningRowDead(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + backend := &cancelAfterJoiningMemberStore{ + backend: store.NewMemory(), + cancel: cancel, + } + + _, err := NewContext(ctx, testNodeConfig(backend, clock.NewFake(time.Unix(175, 0).UTC()), "node-a", "generation-a")) + if !errors.Is(err, context.Canceled) { + t.Fatalf("NewContext error = %v, want context.Canceled", err) + } + member := findTestMember(t, backend.backend, "node-a", "generation-a") + if member.Status != store.MemberDead { + t.Fatalf("member status after canceled join = %q, want %q", member.Status, store.MemberDead) + } +} + +func TestNodeAppliedJoinWriteConflictMarksRowDead(t *testing.T) { + tests := []struct { + name string + conflictStatus store.MemberStatus + wantError error + }{ + {name: "joining write", conflictStatus: store.MemberJoining, wantError: store.ErrConflict}, + {name: "active write", conflictStatus: store.MemberActive, wantError: ErrNodeDead}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + backend := &joinWriteAppliedConflictMemberStore{ + backend: store.NewMemory(), + conflictStatus: test.conflictStatus, + } + + _, err := NewContext(context.Background(), testNodeConfig(backend, clock.NewFake(time.Unix(177, 0).UTC()), "node-a", "generation-a")) + if !errors.Is(err, test.wantError) { + t.Fatalf("NewContext error = %v, want %v", err, test.wantError) + } + member := findTestMember(t, backend.backend, "node-a", "generation-a") + if member.Status != store.MemberDead { + t.Fatalf("member status after applied %s conflict = %q, want %q", test.conflictStatus, member.Status, store.MemberDead) + } + }) + } +} + +func TestNodeKillInterruptsBlockedLeave(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + backend := &blockingLeaveMemberStore{ + backend: store.NewMemory(), + leaveStarted: make(chan struct{}), + } + node, err := New(testNodeConfig(backend, clock.NewFake(time.Unix(180, 0).UTC()), "node-a", "generation-a")) + if err != nil { + t.Fatal(err) + } + + closeDone := make(chan struct{}) + go func() { + node.Close() + close(closeDone) + }() + <-backend.leaveStarted + killDone := make(chan struct{}) + go func() { + node.Kill() + close(killDone) + }() + synctest.Wait() + + select { + case <-closeDone: + default: + t.Fatal("Close remained blocked after Kill") + } + select { + case <-killDone: + default: + t.Fatal("Kill remained blocked on the leave write") + } + }) +} + +func TestNodeCancelStartEscalatesBlockedLeave(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + fakeClock := clock.NewFake(time.Unix(190, 0).UTC()) + backend := &blockingLeaveMemberStore{ + backend: store.NewMemory(), + leaveStarted: make(chan struct{}), + } + node, err := New(testNodeConfig(backend, fakeClock, "node-a", "generation-a")) + if err != nil { + t.Fatal(err) + } + + cancelDone := make(chan struct{}) + go func() { + node.CancelStart() + close(cancelDone) + }() + <-backend.leaveStarted + fakeClock.Advance(500 * time.Millisecond) + synctest.Wait() + + select { + case <-cancelDone: + default: + t.Fatal("CancelStart remained blocked after the table-latency limit") + } + if node.State() != StateDead { + t.Fatalf("state after CancelStart escalation = %v, want dead", node.State()) + } + }) +} + func TestNodeHeartbeatUpdatesAliveTimeWithCAS(t *testing.T) { synctest.Test(t, func(t *testing.T) { start := time.Unix(200, 0).UTC() @@ -308,11 +439,77 @@ type appliedConflictMemberStore struct { nextAppliedConflict atomic.Bool } +type joinWriteAppliedConflictMemberStore struct { + backend *store.Memory + conflictStatus store.MemberStatus + conflicted atomic.Bool +} + type failingListMemberStore struct { backend store.MemberStore failNext atomic.Bool } +type startupContextMemberStore struct { + writes atomic.Int32 + lists atomic.Int32 +} + +type cancelAfterJoiningMemberStore struct { + backend *store.Memory + cancel context.CancelFunc + canceled atomic.Bool +} + +func (s *cancelAfterJoiningMemberStore) WriteMember(ctx context.Context, member store.Member) (store.ETag, error) { + etag, err := s.backend.WriteMember(ctx, member) + if err == nil && member.Status == store.MemberJoining && !s.canceled.Swap(true) { + s.cancel() + } + return etag, err +} + +func (s *cancelAfterJoiningMemberStore) ListMembers(ctx context.Context) (store.MemberSnapshot, error) { + return s.backend.ListMembers(ctx) +} + +type blockingLeaveMemberStore struct { + backend *store.Memory + leaveStarted chan struct{} + leaveSeen atomic.Bool +} + +func (s *blockingLeaveMemberStore) WriteMember(ctx context.Context, member store.Member) (store.ETag, error) { + if member.Status == store.MemberDead { + if !s.leaveSeen.Swap(true) { + close(s.leaveStarted) + } + <-ctx.Done() + return 0, ctx.Err() + } + return s.backend.WriteMember(ctx, member) +} + +func (s *blockingLeaveMemberStore) ListMembers(ctx context.Context) (store.MemberSnapshot, error) { + return s.backend.ListMembers(ctx) +} + +func (s *startupContextMemberStore) WriteMember(ctx context.Context, _ store.Member) (store.ETag, error) { + s.writes.Add(1) + if err := ctx.Err(); err != nil { + return 0, err + } + return 0, errors.New("cluster join did not use the startup context") +} + +func (s *startupContextMemberStore) ListMembers(ctx context.Context) (store.MemberSnapshot, error) { + s.lists.Add(1) + if err := ctx.Err(); err != nil { + return store.MemberSnapshot{}, err + } + return store.MemberSnapshot{}, errors.New("cluster join did not use the startup context") +} + func (s *appliedConflictMemberStore) WriteMember(ctx context.Context, member store.Member) (store.ETag, error) { if s.nextAppliedConflict.CompareAndSwap(true, false) { if _, err := s.backend.WriteMember(ctx, member); err != nil { @@ -327,6 +524,18 @@ func (s *appliedConflictMemberStore) ListMembers(ctx context.Context) (store.Mem return s.backend.ListMembers(ctx) } +func (s *joinWriteAppliedConflictMemberStore) WriteMember(ctx context.Context, member store.Member) (store.ETag, error) { + etag, err := s.backend.WriteMember(ctx, member) + if err == nil && member.Status == s.conflictStatus && !s.conflicted.Swap(true) { + return 0, store.ErrConflict + } + return etag, err +} + +func (s *joinWriteAppliedConflictMemberStore) ListMembers(ctx context.Context) (store.MemberSnapshot, error) { + return s.backend.ListMembers(ctx) +} + func (s *failingListMemberStore) WriteMember(ctx context.Context, member store.Member) (store.ETag, error) { return s.backend.WriteMember(ctx, member) } @@ -391,4 +600,7 @@ func findTestMember(t *testing.T, backend store.MemberStore, nodeAddr, generatio var _ store.MemberStore = (*recordingMemberStore)(nil) var _ store.MemberStore = (*appliedConflictMemberStore)(nil) +var _ store.MemberStore = (*joinWriteAppliedConflictMemberStore)(nil) var _ store.MemberStore = (*failingListMemberStore)(nil) +var _ store.MemberStore = (*cancelAfterJoiningMemberStore)(nil) +var _ store.MemberStore = (*blockingLeaveMemberStore)(nil) diff --git a/cluster_runtime_test.go b/cluster_runtime_test.go index 4e9a174..b6face9 100644 --- a/cluster_runtime_test.go +++ b/cluster_runtime_test.go @@ -14,30 +14,92 @@ import ( "github.com/suraciii/gor/clock" "github.com/suraciii/gor/cluster" - runtimepkg "github.com/suraciii/gor/runtime" + runtimepkg "github.com/suraciii/gor/internal/runtime" "github.com/suraciii/gor/store" "github.com/suraciii/gor/transport" ) func TestNew_ClusterJoinErrorIsReturned(t *testing.T) { - _, err := New(WithMemberStore(failingMemberStore{})) - if err == nil { - t.Fatal("New returned nil error for a failed cluster join") + backend := store.NewMemory() + network := newTestTransportNetwork() + rt := mustNew(t, WithMemberStore(failingMemberStore{}), WithTransport(network.add("node-a")), WithStore(backend), WithReminderStore(backend)) + if err := rt.Start(context.Background()); err == nil { + t.Fatal("Start returned nil error for a failed cluster join") + } +} + +func TestRuntime_CanceledClusterStartMarksJoiningRowDead(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + members := &runtimeCancelAfterJoiningMemberStore{ + backend: store.NewMemory(), + cancel: cancel, + } + backend := store.NewMemory() + network := newTestTransportNetwork() + rt := mustNew(t, clusterRuntimeOptions( + backend, + members, + clock.NewFake(time.Unix(625, 0).UTC()), + "node-a", + "generation-a", + network.add("node-a"), + )...) + + err := rt.Start(ctx) + if !errors.Is(err, context.Canceled) { + t.Fatalf("Start error = %v, want context.Canceled", err) + } + member := findClusterMember(t, members, "node-a", "generation-a") + if member.Status != store.MemberDead { + t.Fatalf("member status after canceled Start = %q, want %q", member.Status, store.MemberDead) + } + if err := rt.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRuntime_CanceledClusterStartMarksActiveRowDead(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + members := &runtimeCancelAfterActiveMemberStore{ + backend: store.NewMemory(), + cancel: cancel, + } + backend := store.NewMemory() + network := newTestTransportNetwork() + rt := mustNew(t, clusterRuntimeOptions( + backend, + members, + clock.NewFake(time.Unix(635, 0).UTC()), + "node-a", + "generation-a", + network.add("node-a"), + )...) + + err := rt.Start(ctx) + if !errors.Is(err, context.Canceled) { + t.Fatalf("Start error = %v, want context.Canceled", err) + } + member := findClusterMember(t, members, "node-a", "generation-a") + if member.Status != store.MemberDead { + t.Fatalf("member status after canceled Start = %q, want %q", member.Status, store.MemberDead) + } + if err := rt.Shutdown(context.Background()); err != nil { + t.Fatal(err) } } func TestNew_ClusterRequiresTransport(t *testing.T) { - _, err := New(WithMemberStore(store.NewMemory())) - if err == nil { - t.Fatal("New returned nil error for a cluster without transport") + rt := mustNew(t, WithMemberStore(store.NewMemory())) + if err := rt.Start(context.Background()); !errors.Is(err, ErrInvalidSetup) { + t.Fatalf("Start error = %v, want ErrInvalidSetup", err) } } func TestNew_TransportRequiresMemberStore(t *testing.T) { network := newTestTransportNetwork() - _, err := New(WithTransport(network.add("node-a"))) - if err == nil { - t.Fatal("New returned nil error for a transport without member store") + rt := mustNew(t, WithTransport(network.add("node-a"))) + if err := rt.Start(context.Background()); !errors.Is(err, ErrInvalidSetup) { + t.Fatalf("Start error = %v, want ErrInvalidSetup", err) } } @@ -60,7 +122,44 @@ func TestNew_ClusterProbeDefaults(t *testing.T) { WithEvictionInterval(0), WithTransport(network.add("node-a")), ) - rt.Close() + mustStart(t, rt) + closeRuntime(rt) + }) +} + +func TestRuntime_ShutdownDeadlineInterruptsClusterLeave(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + members := &runtimeBlockingLeaveMemberStore{ + backend: store.NewMemory(), + leaveStarted: make(chan struct{}), + } + backend := store.NewMemory() + network := newTestTransportNetwork() + rt := mustNew(t, clusterRuntimeOptions( + backend, + members, + clock.NewFake(time.Unix(650, 0).UTC()), + "node-a", + "generation-a", + network.add("node-a"), + )...) + mustStart(t, rt) + + ctx, cancel := context.WithCancel(context.Background()) + shutdownDone := make(chan error, 1) + go func() { shutdownDone <- rt.Shutdown(ctx) }() + <-members.leaveStarted + cancel() + synctest.Wait() + + if err := <-shutdownDone; !errors.Is(err, context.Canceled) { + t.Fatalf("Shutdown error = %v, want context.Canceled", err) + } + select { + case <-rt.Done(): + default: + t.Fatal("Runtime Done remained open after abrupt escalation") + } }) } @@ -75,6 +174,8 @@ func TestRuntime_ClusterForwardsInvocationToAnotherOwner(t *testing.T) { second := mustNew(t, clusterRuntimeOptions(backend, members, fakeClock, "node-b", "generation-b", network.add("node-b"))...) registerAccount(t, first) registerAccount(t, second) + mustStart(t, first) + mustStart(t, second) synctest.Wait() fakeClock.Advance(time.Second) @@ -82,8 +183,8 @@ func TestRuntime_ClusterForwardsInvocationToAnotherOwner(t *testing.T) { var target GrainId for index := 0; index < 4096; index++ { - candidate := GrainId{GrainType: TypeName[Account](), GrainKey: strconv.Itoa(index)} - owner, ok := cluster.Owner(*first.clusterView.Load(), store.GrainId(candidate)) + candidate := GrainId{GrainType: GrainType("gor.Account"), GrainKey: strconv.Itoa(index)} + owner, ok := cluster.Owner(*first.clusterView.Load(), toStoreGrainID(candidate)) if ok && owner == "node-b" { target = candidate break @@ -97,8 +198,8 @@ func TestRuntime_ClusterForwardsInvocationToAnotherOwner(t *testing.T) { if err := first.Invoke(context.Background(), target, "Balance", &accountBalanceRequest{}, &balance); err != nil { t.Fatalf("forwarded invocation error = %v", err) } - first.Close() - second.Close() + closeRuntime(first) + closeRuntime(second) }) } @@ -112,12 +213,13 @@ func TestRuntime_ClusterDeactivatesMovedActivation(t *testing.T) { first := mustNew(t, clusterRuntimeOptions(backend, members, fakeClock, "node-a", "generation-a", network.add("node-a"))...) registerFactoryCalls := atomic.Int32{} installAccount(t, first) - if err := Register[Account](first, func(b *Binder) Account { + if err := Register[Account](first, func(b *GrainContext) Account { registerFactoryCalls.Add(1) return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, first) before := cluster.NewView([]store.Member{{ NodeAddr: "node-a", @@ -138,9 +240,9 @@ func TestRuntime_ClusterDeactivatesMovedActivation(t *testing.T) { }) var target GrainId for index := 0; index < 4096; index++ { - candidate := GrainId{GrainType: TypeName[Account](), GrainKey: strconv.Itoa(index)} - beforeOwner, beforeOK := cluster.Owner(before, store.GrainId(candidate)) - afterOwner, afterOK := cluster.Owner(after, store.GrainId(candidate)) + candidate := GrainId{GrainType: GrainType("gor.Account"), GrainKey: strconv.Itoa(index)} + beforeOwner, beforeOK := cluster.Owner(before, toStoreGrainID(candidate)) + afterOwner, afterOK := cluster.Owner(after, toStoreGrainID(candidate)) if beforeOK && afterOK && beforeOwner == "node-a" && afterOwner == "node-b" { target = candidate break @@ -159,19 +261,20 @@ func TestRuntime_ClusterDeactivatesMovedActivation(t *testing.T) { second := mustNew(t, clusterRuntimeOptions(backend, members, fakeClock, "node-b", "generation-b", network.add("node-b"))...) registerAccount(t, second) + mustStart(t, second) synctest.Wait() fakeClock.Advance(time.Second) synctest.Wait() var balance accountBalanceReply - if err := first.engine.Invoke(context.Background(), target, "Balance", &accountBalanceRequest{}, &balance); err != nil { + if err := first.engine.Invoke(context.Background(), toRuntimeGrainID(target), "Balance", &accountBalanceRequest{}, &balance); err != nil { t.Fatalf("direct runtime invocation after ownership change = %v", err) } if got := registerFactoryCalls.Load(); got != 2 { t.Fatalf("factory calls after ownership change = %d, want 2", got) } - first.Close() - second.Close() + closeRuntime(first) + closeRuntime(second) }) } @@ -182,7 +285,8 @@ func TestRuntime_ClusterKillLeavesMemberForFailureDetection(t *testing.T) { members := store.NewMemory() network := newTestTransportNetwork() rt := mustNew(t, clusterRuntimeOptions(store.NewMemory(), members, fakeClock, "node-a", "generation-a", network.add("node-a"))...) - rt.Kill() + mustStart(t, rt) + killRuntime(rt) member := findClusterMember(t, members, "node-a", "generation-a") if member.Status != store.MemberActive { @@ -196,8 +300,8 @@ func TestRuntime_DoneClosesForCloseAndKill(t *testing.T) { name string stop func(*Runtime) }{ - {name: "close", stop: (*Runtime).Close}, - {name: "kill", stop: (*Runtime).Kill}, + {name: "close", stop: closeRuntime}, + {name: "kill", stop: killRuntime}, } { t.Run(test.name, func(t *testing.T) { synctest.Test(t, func(t *testing.T) { @@ -205,6 +309,7 @@ func TestRuntime_DoneClosesForCloseAndKill(t *testing.T) { members := store.NewMemory() network := newTestTransportNetwork() rt := mustNew(t, clusterRuntimeOptions(store.NewMemory(), members, fakeClock, "node-a", "generation-a", network.add("node-a"))...) + mustStart(t, rt) test.stop(rt) select { @@ -232,6 +337,7 @@ func TestRuntime_ClusterDeathStopsAndDeactivates(t *testing.T) { ) first := mustNew(t, firstOptions...) registerAccount(t, first) + mustStart(t, first) secondOptions := clusterRuntimeOptions(backend, members, fakeClock, "node-b", "generation-b") secondOptions = append(secondOptions, WithHeartbeatInterval(time.Hour), @@ -239,8 +345,9 @@ func TestRuntime_ClusterDeathStopsAndDeactivates(t *testing.T) { WithTransport(network.add("node-b")), ) second := mustNew(t, secondOptions...) + mustStart(t, second) - id := GrainId{GrainType: TypeName[Account](), GrainKey: "self-death"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "self-death"} if err := first.Invoke(context.Background(), id, "Balance", &accountBalanceRequest{}, &accountBalanceReply{}); err != nil { t.Fatalf("initial invocation error = %v", err) } @@ -263,12 +370,12 @@ func TestRuntime_ClusterDeathStopsAndDeactivates(t *testing.T) { if err := first.Invoke(context.Background(), id, "Balance", &accountBalanceRequest{}, &accountBalanceReply{}); err == nil { t.Fatal("invocation after cluster death unexpectedly succeeded") } - if err := first.engine.Invoke(context.Background(), id, "Balance", &accountBalanceRequest{}, &accountBalanceReply{}); !errors.Is(err, runtimepkg.ErrRuntimeClosed) { + if err := first.engine.Invoke(context.Background(), toRuntimeGrainID(id), "Balance", &accountBalanceRequest{}, &accountBalanceReply{}); !errors.Is(err, runtimepkg.ErrRuntimeClosed) { t.Fatalf("direct runtime invocation after cluster death error = %v, want %v", err, runtimepkg.ErrRuntimeClosed) } - first.Close() - second.Close() + closeRuntime(first) + closeRuntime(second) }) } @@ -281,6 +388,8 @@ func TestRuntime_HandleRejectsAfterClusterDeath(t *testing.T) { first := mustNew(t, clusterRuntimeOptions(store.NewMemory(), members, fakeClock, "node-a", "generation-a", network.add("node-a"))...) second := mustNew(t, clusterRuntimeOptions(store.NewMemory(), members, fakeClock, "node-b", "generation-b", network.add("node-b"))...) registerAccount(t, first) + mustStart(t, first) + mustStart(t, second) self := findClusterMember(t, members, "node-a", "generation-a") self.Status = store.MemberDead @@ -308,8 +417,8 @@ func TestRuntime_HandleRejectsAfterClusterDeath(t *testing.T) { t.Fatalf("response error = %#v, want node-dead code after cluster death", response.Error) } - first.Close() - second.Close() + closeRuntime(first) + closeRuntime(second) }) } @@ -332,6 +441,8 @@ func TestRuntime_ClusterDeathClosesTransportAndStops(t *testing.T) { ) first := mustNew(t, firstOptions...) second := mustNew(t, clusterRuntimeOptions(store.NewMemory(), members, fakeClock, "node-b", "generation-b", network.add("node-b"))...) + mustStart(t, first) + mustStart(t, second) self := findClusterMember(t, members, "node-a", "generation-a") self.Status = store.MemberDead @@ -358,8 +469,8 @@ func TestRuntime_ClusterDeathClosesTransportAndStops(t *testing.T) { t.Fatalf("root state after cluster death = %v, want stopped", state) } - first.Close() - second.Close() + closeRuntime(first) + closeRuntime(second) }) } @@ -383,12 +494,13 @@ func TestRuntime_ClusterDeathSkipsOnDeactivate(t *testing.T) { }), )...) deactivateCalls := new(atomic.Int32) - installLifecycleAccount(t, first, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.deactivateCalls = deactivateCalls - entity.deactivateErr = errors.New("deactivate failed") + installLifecycleAccount(t, first, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateCalls = deactivateCalls + grain.deactivateErr = errors.New("deactivate failed") }) + mustStart(t, first) - id := GrainId{GrainType: TypeName[lifecycleAccount](), GrainKey: "self-death"} + id := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: "self-death"} if err := first.Invoke(context.Background(), id, "Value", &lifecycleAccountValueRequest{}, &lifecycleAccountValueReply{}); err != nil { t.Fatalf("initial invocation error = %v", err) } @@ -415,60 +527,60 @@ func TestRuntime_ClusterDeathSkipsOnDeactivate(t *testing.T) { t.Fatalf("OnError received an event after cluster death: %#v", got) default: } - first.Close() + closeRuntime(first) }) } -// sideEffectEntity records every Touch as a side effect and carries no state. +// sideEffectGrain records every Touch as a side effect and carries no state. // It exists so a test can tell whether a method body ran on a particular node. -type sideEffectEntity interface { +type sideEffectGrain interface { Touch(context.Context) error } type sideEffectTouchRequest struct{} type sideEffectTouchReply struct{} -type sideEffectEntityProxy struct { +type sideEffectGrainProxy struct { invoker Invoker id GrainId } -func (p *sideEffectEntityProxy) Touch(ctx context.Context) error { +func (p *sideEffectGrainProxy) Touch(ctx context.Context) error { return p.invoker.Invoke(ctx, p.id, "Touch", &sideEffectTouchRequest{}, &sideEffectTouchReply{}) } -type sideEffectEntityImpl struct { +type sideEffectGrainImpl struct { calls *atomic.Int32 } -func (e *sideEffectEntityImpl) Touch(context.Context) error { +func (e *sideEffectGrainImpl) Touch(context.Context) error { e.calls.Add(1) return nil } -func dispatchSideEffectEntity(ctx context.Context, instance sideEffectEntity, method string, _ any, _ any) error { +func dispatchSideEffectGrain(ctx context.Context, instance sideEffectGrain, method string, _ any, _ any) error { if method != "Touch" { return fmt.Errorf("unknown method %q", method) } return instance.Touch(ctx) } -func newSideEffectEntityCall(method string) (args any, reply any) { +func newSideEffectGrainCall(method string) (args any, reply any) { if method != "Touch" { return nil, nil } return &sideEffectTouchRequest{}, &sideEffectTouchReply{} } -func installSideEffectEntity(t *testing.T, rt *Runtime, calls *atomic.Int32) { +func installSideEffectGrain(t *testing.T, rt *Runtime, calls *atomic.Int32) { t.Helper() - if err := InstallType[sideEffectEntity](rt, dispatchSideEffectEntity, func(invoker Invoker, id GrainId) sideEffectEntity { - return &sideEffectEntityProxy{invoker: invoker, id: id} - }, newSideEffectEntityCall, nil); err != nil { + if err := InstallType[sideEffectGrain](rt, GeneratedCodeVersion, "gor.sideEffectGrain", dispatchSideEffectGrain, func(invoker Invoker, id GrainId) sideEffectGrain { + return &sideEffectGrainProxy{invoker: invoker, id: id} + }, newSideEffectGrainCall, noReminderCall); err != nil { t.Fatal(err) } - if err := Register[sideEffectEntity](rt, func(b *Binder) sideEffectEntity { - return &sideEffectEntityImpl{calls: calls} + if err := Register[sideEffectGrain](rt, func(b *GrainContext) sideEffectGrain { + return &sideEffectGrainImpl{calls: calls} }); err != nil { t.Fatal(err) } @@ -496,7 +608,7 @@ func (s *barrierMemberStore) WriteMember(ctx context.Context, m store.Member) (s // TestScenario_ClusterDeathStopsNodeAndHandsoff is the cluster-death scenario: // once the cluster declares this node dead, its Done signal closes, a direct // call on it is rejected without running the method, and another node that has -// converged can execute the same entity. A write barrier on the dead row pins +// converged can execute the same grain. A write barrier on the dead row pins // the moment the row is visible. func TestScenario_ClusterDeathStopsNodeAndHandsoff(t *testing.T) { synctest.Test(t, func(t *testing.T) { @@ -512,16 +624,18 @@ func TestScenario_ClusterDeathStopsNodeAndHandsoff(t *testing.T) { nodeA := mustNew(t, clusterRuntimeOptions(backend, members, fakeClock, "node-a", "generation-a", network.add("node-a"))...) nodeB := mustNew(t, clusterRuntimeOptions(backend, members, fakeClock, "node-b", "generation-b", network.add("node-b"))...) calls := new(atomic.Int32) - installSideEffectEntity(t, nodeA, calls) - installSideEffectEntity(t, nodeB, calls) + installSideEffectGrain(t, nodeA, calls) + installSideEffectGrain(t, nodeB, calls) + mustStart(t, nodeA) + mustStart(t, nodeB) synctest.Wait() fakeClock.Advance(time.Second) synctest.Wait() var id GrainId for index := 0; index < 4096; index++ { - candidate := GrainId{GrainType: TypeName[sideEffectEntity](), GrainKey: strconv.Itoa(index)} - if owner, ok := cluster.Owner(*nodeA.clusterView.Load(), store.GrainId(candidate)); ok && owner == "node-a" { + candidate := GrainId{GrainType: GrainType("gor.sideEffectGrain"), GrainKey: strconv.Itoa(index)} + if owner, ok := cluster.Owner(*nodeA.clusterView.Load(), toStoreGrainID(candidate)); ok && owner == "node-a" { id = candidate break } @@ -565,7 +679,7 @@ func TestScenario_ClusterDeathStopsNodeAndHandsoff(t *testing.T) { t.Fatal("node-a Done is still open after cluster death") } - // The node that converged can execute the same entity. + // The node that converged can execute the same grain. if err := nodeB.Invoke(context.Background(), id, "Touch", &sideEffectTouchRequest{}, &sideEffectTouchReply{}); err != nil { t.Fatalf("touch on node-b after handoff = %v, want nil", err) } @@ -574,8 +688,8 @@ func TestScenario_ClusterDeathStopsNodeAndHandsoff(t *testing.T) { } close(members.release) - nodeA.Close() - nodeB.Close() + closeRuntime(nodeA) + closeRuntime(nodeB) }) } @@ -620,6 +734,67 @@ func findClusterMember(t *testing.T, backend store.MemberStore, nodeAddr, genera type failingMemberStore struct{} +type runtimeBlockingLeaveMemberStore struct { + backend *store.Memory + leaveStarted chan struct{} + leaveSeen atomic.Bool +} + +type runtimeCancelAfterJoiningMemberStore struct { + backend *store.Memory + cancel context.CancelFunc + canceled atomic.Bool +} + +type runtimeCancelAfterActiveMemberStore struct { + backend *store.Memory + cancel context.CancelFunc + canceled atomic.Bool +} + +func (s *runtimeCancelAfterJoiningMemberStore) WriteMember(ctx context.Context, member store.Member) (store.ETag, error) { + etag, err := s.backend.WriteMember(ctx, member) + if err == nil && member.Status == store.MemberJoining && !s.canceled.Swap(true) { + s.cancel() + } + return etag, err +} + +func (s *runtimeCancelAfterJoiningMemberStore) ListMembers(ctx context.Context) (store.MemberSnapshot, error) { + return s.backend.ListMembers(ctx) +} + +func (s *runtimeCancelAfterActiveMemberStore) WriteMember(ctx context.Context, member store.Member) (store.ETag, error) { + etag, err := s.backend.WriteMember(ctx, member) + if err == nil && member.Status == store.MemberActive && !s.canceled.Swap(true) { + s.cancel() + } + return etag, err +} + +func (s *runtimeCancelAfterActiveMemberStore) ListMembers(ctx context.Context) (store.MemberSnapshot, error) { + return s.backend.ListMembers(ctx) +} + +func (s *runtimeBlockingLeaveMemberStore) WriteMember(ctx context.Context, member store.Member) (store.ETag, error) { + if member.Status == store.MemberDead { + if !s.leaveSeen.Swap(true) { + close(s.leaveStarted) + } + <-ctx.Done() + return 0, ctx.Err() + } + return s.backend.WriteMember(ctx, member) +} + +func (s *runtimeBlockingLeaveMemberStore) ListMembers(ctx context.Context) (store.MemberSnapshot, error) { + return s.backend.ListMembers(ctx) +} + +var _ store.MemberStore = (*runtimeBlockingLeaveMemberStore)(nil) +var _ store.MemberStore = (*runtimeCancelAfterJoiningMemberStore)(nil) +var _ store.MemberStore = (*runtimeCancelAfterActiveMemberStore)(nil) + func (failingMemberStore) WriteMember(context.Context, store.Member) (store.ETag, error) { return 0, errors.New("member store unavailable") } diff --git a/cmd/gorgen/external_module_gen_test.go b/cmd/gorgen/external_module_gen_test.go new file mode 100644 index 0000000..a68c2b6 --- /dev/null +++ b/cmd/gorgen/external_module_gen_test.go @@ -0,0 +1,177 @@ +//go:build gen + +package main + +import ( + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestExternalModuleGeneratesAndUsesOnlyPublicPackages(t *testing.T) { + root := moduleRoot(t) + moduleDir := t.TempDir() + goMod := "module example.com/gorclean\n\ngo 1.25.0\n\nreplace github.com/suraciii/gor => " + filepath.ToSlash(root) + "\n" + writeExternalFile(t, filepath.Join(moduleDir, "go.mod"), goMod) + runExternalGo(t, moduleDir, "get", "github.com/suraciii/gor") + runExternalGo(t, moduleDir, "get", "-tool", "github.com/suraciii/gor/cmd/gorgen") + + domainDir := filepath.Join(moduleDir, "domain") + if err := os.MkdirAll(domainDir, 0o755); err != nil { + t.Fatal(err) + } + writeExternalFile(t, filepath.Join(domainDir, "domain.go"), `package domain + +//go:generate go tool gorgen -pkg . + +import ( + "context" + + "github.com/suraciii/gor" +) + +//gor:grain +type Counter interface { + Add(context.Context, int64) (int64, error) + Value(context.Context) (int64, error) +} + +type counter struct { + value gor.State[int64] +} + +func NewCounter(grain *gor.GrainContext) Counter { + return &counter{value: gor.NewState[int64](grain, "value")} +} + +func (c *counter) Add(ctx context.Context, amount int64) (int64, error) { + value := c.value.Get() + amount + if err := c.value.Set(ctx, value); err != nil { + return 0, err + } + return value, nil +} + +func (c *counter) Value(context.Context) (int64, error) { + return c.value.Get(), nil +} +`) + + runExternalGo(t, moduleDir, "generate", "./...") + writeExternalFile(t, filepath.Join(moduleDir, "application_test.go"), `package gorclean + +import ( + "context" + "reflect" + "testing" + + "example.com/gorclean/domain" + "example.com/gorclean/domain/gorgen" + "github.com/suraciii/gor" + "github.com/suraciii/gor/store" +) + +func TestPublicModule(t *testing.T) { + if got := reflect.TypeOf(gor.Idle).PkgPath(); got != "github.com/suraciii/gor" { + t.Fatalf("DeactivationReason package = %q", got) + } + backend := store.NewMemory() + newRuntime := func() *gor.Runtime { + rt, err := gor.New( + gor.WithStore(backend), + gor.WithReminderStore(backend), + gor.Option(func(config *gor.Config) { config.MailboxCapacity = 4 }), + gor.WithIdleTimeout(0), + gor.WithEvictionInterval(0), + ) + if err != nil { + t.Fatal(err) + } + if err := gorgen.Install(rt); err != nil { + t.Fatal(err) + } + if err := gor.Register[domain.Counter](rt, domain.NewCounter); err != nil { + t.Fatal(err) + } + if err := rt.Start(context.Background()); err != nil { + t.Fatal(err) + } + return rt + } + + first := newRuntime() + if value, err := gor.Ref[domain.Counter](first, "one").Add(context.Background(), 7); err != nil || value != 7 { + t.Fatalf("Add = (%d, %v)", value, err) + } + if err := first.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } + + second := newRuntime() + defer second.Shutdown(context.Background()) + if value, err := gor.Ref[domain.Counter](second, "one").Value(context.Background()); err != nil || value != 7 { + t.Fatalf("Value after restart = (%d, %v)", value, err) + } +} +`) + + runExternalGo(t, moduleDir, "tool", "gorgen", "-pkg", "./domain", "-check") + runExternalGo(t, moduleDir, "test", "./...") + assertExternalImportsArePublic(t, moduleDir) +} + +func assertExternalImportsArePublic(t *testing.T, moduleDir string) { + t.Helper() + forbidden := map[string]bool{ + "github.com/suraciii/gor/mail": true, + "github.com/suraciii/gor/runtime": true, + "github.com/suraciii/gor/timer": true, + } + err := filepath.WalkDir(moduleDir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || filepath.Ext(path) != ".go" { + return nil + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly) + if err != nil { + return err + } + for _, imported := range file.Imports { + value, err := strconv.Unquote(imported.Path.Value) + if err != nil { + return err + } + if forbidden[value] || strings.Contains(value, "github.com/suraciii/gor/internal/") { + t.Errorf("%s imports non-public package %s", path, value) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +func runExternalGo(t *testing.T, dir string, args ...string) { + t.Helper() + command := exec.Command("go", args...) + command.Dir = dir + command.Env = append(os.Environ(), "GOWORK=off") + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("go %s: %v\n%s", strings.Join(args, " "), err, output) + } +} + +func writeExternalFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/gorgen/generator_gen_test.go b/cmd/gorgen/generator_gen_test.go index c7b42ef..19ab116 100644 --- a/cmd/gorgen/generator_gen_test.go +++ b/cmd/gorgen/generator_gen_test.go @@ -40,7 +40,7 @@ func TestGenerateFixtureBuilds(t *testing.T) { func TestGeneratedFixtureMatchesCommittedOutput(t *testing.T) { root := moduleRoot(t) - fixtureRoot := filepath.Join(root, "cmd", "gorgen", "testfixture", "endtoend") + fixtureRoot := filepath.Join(root, "cmd", "gorgen", "testfixture", "endtoend", "domain") outputDir, err := os.MkdirTemp(fixtureRoot, "generated-") if err != nil { t.Fatal(err) @@ -61,6 +61,63 @@ func TestGeneratedFixtureMatchesCommittedOutput(t *testing.T) { } } +func TestGenerateDefaultOutputIsCurrent(t *testing.T) { + root := moduleRoot(t) + command := exec.Command("go", "run", "./cmd/gorgen", "-pkg", "./cmd/gorgen/testfixture/endtoend/domain", "-check") + command.Dir = root + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("check default generated output: %v\n%s", err, output) + } +} + +func TestGenerateCheckRejectsStaleOutputWithoutWriting(t *testing.T) { + root := moduleRoot(t) + outputDir := t.TempDir() + path := filepath.Join(outputDir, "generated.go") + stale := []byte("package gorgen\n") + if err := os.WriteFile(path, stale, 0o600); err != nil { + t.Fatal(err) + } + + command := exec.Command("go", "run", "./cmd/gorgen", "-pkg", "./cmd/gorgen/testfixture/endtoend/domain", "-out", outputDir, "-check") + command.Dir = root + output, err := command.CombinedOutput() + if err == nil { + t.Fatal("gorgen -check accepted stale output") + } + if !strings.Contains(string(output), "is stale") { + t.Fatalf("check output = %s, want stale error", output) + } + got, readErr := os.ReadFile(path) + if readErr != nil { + t.Fatal(readErr) + } + if !bytes.Equal(got, stale) { + t.Fatalf("stale output changed to %q", got) + } +} + +func TestGenerateCollisionFixtureBuilds(t *testing.T) { + root := moduleRoot(t) + fixtureRoot := filepath.Join(root, "cmd", "gorgen", "testfixture", "collisions") + outputDir, err := os.MkdirTemp(fixtureRoot, "generated-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(outputDir) }) + + runGorgen(t, root, "./cmd/gorgen/testfixture/collisions/domain", outputDir) + buildPath, err := filepath.Rel(root, outputDir) + if err != nil { + t.Fatal(err) + } + build := exec.Command("go", "build", "./"+filepath.ToSlash(buildPath)) + build.Dir = root + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("go build collision fixture: %v\n%s", err, output) + } +} + func TestGenerateResolvesImportAliasCollision(t *testing.T) { root := moduleRoot(t) fixtureRoot := filepath.Join(root, "cmd", "gorgen", "testfixture", "aliasconflict") @@ -160,7 +217,7 @@ func TestGenerateReportsContractLine(t *testing.T) { } } -func TestGenerateRejectsPackageWithoutEntity(t *testing.T) { +func TestGenerateRejectsPackageWithoutGrain(t *testing.T) { root := moduleRoot(t) outputDir, err := os.MkdirTemp(filepath.Join(root, "cmd", "gorgen", "testfixture"), "empty-generated-") if err != nil { diff --git a/cmd/gorgen/main.go b/cmd/gorgen/main.go index 95ecc23..dcdc915 100644 --- a/cmd/gorgen/main.go +++ b/cmd/gorgen/main.go @@ -1,11 +1,15 @@ // Command gorgen generates Go support code for gor Grain interfaces. // // It loads the package named by -pkg, reads interfaces marked with -// //gor:grain, and writes generated proxies, dispatch functions, and an -// Install function to generated.go. Run it with, for example: +// //gor:grain, and writes Grain Reference implementations, dispatch functions, +// and an Install function to generated.go. Run it with, for example: // // go tool gorgen -pkg ./domain // +// Use -check to compare generated.go with the current contract without +// writing. The command returns a nonzero exit status when output is missing or +// stale. +// // (Add the generator to the module once with // `go get -tool github.com/suraciii/gor/cmd/gorgen`; inside the gor // repository itself, `go run ./cmd/gorgen` works the same.) @@ -18,6 +22,7 @@ package main import ( + "bytes" "errors" "flag" "fmt" @@ -40,6 +45,7 @@ func run(args []string, stderr io.Writer) error { flags.SetOutput(stderr) packagePattern := flags.String("pkg", "", "package containing gor:grain interfaces") output := flags.String("out", "", "output directory (default: /gorgen)") + check := flags.Bool("check", false, "check that generated.go is current without writing") if err := flags.Parse(args); err != nil { return err } @@ -62,12 +68,69 @@ func run(args []string, stderr io.Writer) error { if outputDir == "" { outputDir = filepath.Join(loaded.Dir, "gorgen") } + outputPath := filepath.Join(outputDir, "generated.go") + if *check { + return checkGenerated(outputPath, source) + } if err := os.MkdirAll(outputDir, 0o755); err != nil { return fmt.Errorf("create output directory %s: %w", outputDir, err) } - outputPath := filepath.Join(outputDir, "generated.go") - if err := os.WriteFile(outputPath, source, 0o644); err != nil { + if err := writeFileAtomically(outputPath, source); err != nil { return fmt.Errorf("write %s: %w", outputPath, err) } return nil } + +func checkGenerated(path string, want []byte) error { + got, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("generated output %s is missing; run gorgen without -check", path) + } + return fmt.Errorf("read generated output %s: %w", path, err) + } + if !bytes.Equal(got, want) { + return fmt.Errorf("generated output %s is stale; run gorgen without -check", path) + } + return nil +} + +func writeFileAtomically(path string, source []byte) error { + return writeFileAtomicallyWith(path, 0o644, func(writer io.Writer) error { + _, err := writer.Write(source) + return err + }, replacePath) +} + +func writeFileAtomicallyWith(path string, mode os.FileMode, populate func(io.Writer) error, replace func(string, string) error) error { + temporary, err := os.CreateTemp(filepath.Dir(path), ".gorgen-*") + if err != nil { + return fmt.Errorf("create temporary file: %w", err) + } + temporaryPath := temporary.Name() + closed := false + defer func() { + if !closed { + _ = temporary.Close() + } + _ = os.Remove(temporaryPath) + }() + + if err := temporary.Chmod(mode); err != nil { + return fmt.Errorf("set temporary file mode: %w", err) + } + if err := populate(temporary); err != nil { + return fmt.Errorf("populate temporary file: %w", err) + } + if err := temporary.Sync(); err != nil { + return fmt.Errorf("sync temporary file: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close temporary file: %w", err) + } + closed = true + if err := replace(temporaryPath, path); err != nil { + return fmt.Errorf("replace generated file: %w", err) + } + return nil +} diff --git a/cmd/gorgen/main_test.go b/cmd/gorgen/main_test.go new file mode 100644 index 0000000..1b30b84 --- /dev/null +++ b/cmd/gorgen/main_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "errors" + "io" + "os" + "path/filepath" + "testing" +) + +func TestWriteFileAtomically_PopulateFailureKeepsOldFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "generated.go") + old := []byte("old generated file\n") + if err := os.WriteFile(path, old, 0o644); err != nil { + t.Fatal(err) + } + + wantErr := errors.New("generation interrupted") + err := writeFileAtomicallyWith(path, 0o644, func(writer io.Writer) error { + if _, err := writer.Write([]byte("partial new file")); err != nil { + return err + } + return wantErr + }, replacePath) + if !errors.Is(err, wantErr) { + t.Fatalf("writeFileAtomicallyWith error = %v, want %v", err, wantErr) + } + assertFileEquals(t, path, old) + assertNoGeneratorTempFiles(t, dir) +} + +func TestWriteFileAtomically_ReplaceFailureKeepsOldFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "generated.go") + old := []byte("old generated file\n") + if err := os.WriteFile(path, old, 0o644); err != nil { + t.Fatal(err) + } + + wantErr := errors.New("replace failed") + err := writeFileAtomicallyWith(path, 0o644, func(writer io.Writer) error { + _, err := writer.Write([]byte("complete new file\n")) + return err + }, func(string, string) error { return wantErr }) + if !errors.Is(err, wantErr) { + t.Fatalf("writeFileAtomicallyWith error = %v, want %v", err, wantErr) + } + assertFileEquals(t, path, old) + assertNoGeneratorTempFiles(t, dir) +} + +func TestWriteFileAtomically_ReplacesCompleteFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "generated.go") + if err := os.WriteFile(path, []byte("old\n"), 0o644); err != nil { + t.Fatal(err) + } + newSource := []byte("new complete file\n") + if err := writeFileAtomically(path, newSource); err != nil { + t.Fatal(err) + } + assertFileEquals(t, path, newSource) + assertNoGeneratorTempFiles(t, dir) +} + +func TestCheckGenerated_DoesNotChangeOutput(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "generated.go") + old := []byte("old generated file\n") + if err := os.WriteFile(path, old, 0o600); err != nil { + t.Fatal(err) + } + before, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + + if err := checkGenerated(path, old); err != nil { + t.Fatalf("matching output: %v", err) + } + if err := checkGenerated(path, []byte("new generated file\n")); err == nil { + t.Fatal("stale output passed check") + } + assertFileEquals(t, path, old) + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got, want := info.Mode().Perm(), before.Mode().Perm(); got != want { + t.Fatalf("mode after checks = %o, want unchanged mode %o", got, want) + } + + missing := filepath.Join(dir, "missing.go") + if err := checkGenerated(missing, old); err == nil { + t.Fatal("missing output passed check") + } + if _, err := os.Stat(missing); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing output was created: %v", err) + } +} + +func assertFileEquals(t *testing.T, path string, want []byte) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != string(want) { + t.Fatalf("%s = %q, want %q", path, got, want) + } +} + +func assertNoGeneratorTempFiles(t *testing.T, dir string) { + t.Helper() + matches, err := filepath.Glob(filepath.Join(dir, ".gorgen-*")) + if err != nil { + t.Fatal(err) + } + if len(matches) != 0 { + t.Fatalf("temporary files remain: %v", matches) + } +} diff --git a/cmd/gorgen/replace_nonwindows.go b/cmd/gorgen/replace_nonwindows.go new file mode 100644 index 0000000..b465422 --- /dev/null +++ b/cmd/gorgen/replace_nonwindows.go @@ -0,0 +1,9 @@ +//go:build !windows + +package main + +import "os" + +func replacePath(oldPath, newPath string) error { + return os.Rename(oldPath, newPath) +} diff --git a/cmd/gorgen/replace_windows.go b/cmd/gorgen/replace_windows.go new file mode 100644 index 0000000..2f6f5da --- /dev/null +++ b/cmd/gorgen/replace_windows.go @@ -0,0 +1,17 @@ +//go:build windows + +package main + +import "golang.org/x/sys/windows" + +func replacePath(oldPath, newPath string) error { + oldName, err := windows.UTF16PtrFromString(oldPath) + if err != nil { + return err + } + newName, err := windows.UTF16PtrFromString(newPath) + if err != nil { + return err + } + return windows.MoveFileEx(oldName, newName, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) +} diff --git a/cmd/gorgen/testfixture/collisions/conflict/value.go b/cmd/gorgen/testfixture/collisions/conflict/value.go new file mode 100644 index 0000000..004ca1a --- /dev/null +++ b/cmd/gorgen/testfixture/collisions/conflict/value.go @@ -0,0 +1,3 @@ +package gorgen1_AProxy + +type Value struct{} diff --git a/cmd/gorgen/testfixture/collisions/domain/domain.go b/cmd/gorgen/testfixture/collisions/domain/domain.go new file mode 100644 index 0000000..ecdd1fa --- /dev/null +++ b/cmd/gorgen/testfixture/collisions/domain/domain.go @@ -0,0 +1,39 @@ +package domain + +import ( + "context" + + "github.com/suraciii/gor/cmd/gorgen/testfixture/collisions/conflict" + predeclarederror "github.com/suraciii/gor/cmd/gorgen/testfixture/collisions/error" + keywordcontext "github.com/suraciii/gor/cmd/gorgen/testfixture/collisions/type" +) + +type privatePayload struct{} + +type Payload = privatePayload + +type Record struct { + private int +} + +var _ = Record{}.private + +//gor:grain +type A interface { + BC(_ context.Context, p int, reply string) error + Conflict(context.Context, gorgen1_AProxy.Value) error +} + +//gor:grain +type AB interface { + C(context.Context) error +} + +//gor:grain +type Names interface { + Values(context.Context, int, string) error + Alias(context.Context, Payload) error + Record(context.Context, Record) error + Predeclared(context.Context, predeclarederror.Value) error + KeywordCandidate(context.Context, keywordcontext.Value) error +} diff --git a/cmd/gorgen/testfixture/collisions/error/value.go b/cmd/gorgen/testfixture/collisions/error/value.go new file mode 100644 index 0000000..b907e81 --- /dev/null +++ b/cmd/gorgen/testfixture/collisions/error/value.go @@ -0,0 +1,3 @@ +package error + +type Value struct{} diff --git a/cmd/gorgen/testfixture/collisions/type/value.go b/cmd/gorgen/testfixture/collisions/type/value.go new file mode 100644 index 0000000..e5b7b7e --- /dev/null +++ b/cmd/gorgen/testfixture/collisions/type/value.go @@ -0,0 +1,3 @@ +package context + +type Value struct{} diff --git a/cmd/gorgen/testfixture/domain/domain.go b/cmd/gorgen/testfixture/domain/domain.go index 26bd94f..9fe0f37 100644 --- a/cmd/gorgen/testfixture/domain/domain.go +++ b/cmd/gorgen/testfixture/domain/domain.go @@ -9,5 +9,5 @@ type Account interface { } type Helper interface { - NotAnEntity(string) error + NotAnGrain(string) error } diff --git a/cmd/gorgen/testfixture/empty/domain.go b/cmd/gorgen/testfixture/empty/domain.go index e0ff3fd..6c7dc88 100644 --- a/cmd/gorgen/testfixture/empty/domain.go +++ b/cmd/gorgen/testfixture/empty/domain.go @@ -1,5 +1,5 @@ package empty type Helper interface { - NotAnEntity(string) error + NotAnGrain(string) error } diff --git a/cmd/gorgen/testfixture/endtoend/domain/domain.go b/cmd/gorgen/testfixture/endtoend/domain/domain.go index b39b985..43bca7e 100644 --- a/cmd/gorgen/testfixture/endtoend/domain/domain.go +++ b/cmd/gorgen/testfixture/endtoend/domain/domain.go @@ -1,5 +1,7 @@ package domain +//go:generate go tool gorgen -pkg . + import ( "context" @@ -18,7 +20,7 @@ type account struct { balance gor.State[int64] } -func NewAccount(b *gor.Binder) Account { +func NewAccount(b *gor.GrainContext) Account { return &account{balance: gor.NewState[int64](b, "balance")} } diff --git a/cmd/gorgen/testfixture/endtoend/domain/gorgen/generated.go b/cmd/gorgen/testfixture/endtoend/domain/gorgen/generated.go new file mode 100644 index 0000000..ce23a5b --- /dev/null +++ b/cmd/gorgen/testfixture/endtoend/domain/gorgen/generated.go @@ -0,0 +1,135 @@ +// Code generated by gorgen. DO NOT EDIT. + +package gorgen + +import ( + "context" + "fmt" + + "github.com/suraciii/gor" + "github.com/suraciii/gor/cmd/gorgen/testfixture/endtoend/domain" +) + +const generatedCodeVersion = 1 + +const gorgen7_AccountGrainType gor.GrainType = "domain.Account" + +type gorgen7_AccountProxy struct { + id gor.GrainId + rt gor.Invoker +} + +type gorgen7_Account7_DepositRequest struct { + A0 int64 +} +type gorgen7_Account7_DepositReply struct { + R0 int64 +} + +func (p *gorgen7_AccountProxy) Deposit(ctx context.Context, arg0 int64) (int64, error) { + var reply gorgen7_Account7_DepositReply + err := p.rt.Invoke(ctx, p.id, "Deposit", &gorgen7_Account7_DepositRequest{A0: arg0}, &reply) + return reply.R0, err +} + +type gorgen7_Account5_ResetRequest struct{} +type gorgen7_Account5_ResetReply struct{} + +func (p *gorgen7_AccountProxy) Reset(ctx context.Context) error { + var reply gorgen7_Account5_ResetReply + err := p.rt.Invoke(ctx, p.id, "Reset", &gorgen7_Account5_ResetRequest{}, &reply) + return err +} + +type gorgen7_Account8_SnapshotRequest struct{} +type gorgen7_Account8_SnapshotReply struct { + R0 int64 + R1 string +} + +func (p *gorgen7_AccountProxy) Snapshot(ctx context.Context) (int64, string, error) { + var reply gorgen7_Account8_SnapshotReply + err := p.rt.Invoke(ctx, p.id, "Snapshot", &gorgen7_Account8_SnapshotRequest{}, &reply) + return reply.R0, reply.R1, err +} + +type gorgen7_Account4_TickRequest struct { + A0 gor.TickStatus +} +type gorgen7_Account4_TickReply struct{} + +func (p *gorgen7_AccountProxy) Tick(ctx context.Context, arg0 gor.TickStatus) error { + var reply gorgen7_Account4_TickReply + err := p.rt.Invoke(ctx, p.id, "Tick", &gorgen7_Account4_TickRequest{A0: arg0}, &reply) + return err +} + +func gorgen7_AccountDispatch(ctx context.Context, instance domain.Account, method string, args any, reply any) error { + switch method { + case "Deposit": + typedArgs := args.(*gorgen7_Account7_DepositRequest) + typedReply := reply.(*gorgen7_Account7_DepositReply) + r0, err := instance.Deposit(ctx, typedArgs.A0) + typedReply.R0 = r0 + return err + case "Reset": + err := instance.Reset(ctx) + return err + case "Snapshot": + typedReply := reply.(*gorgen7_Account8_SnapshotReply) + r0, r1, err := instance.Snapshot(ctx) + typedReply.R0 = r0 + typedReply.R1 = r1 + return err + case "Tick": + typedArgs := args.(*gorgen7_Account4_TickRequest) + err := instance.Tick(ctx, typedArgs.A0) + return err + default: + return fmt.Errorf("unknown method %q", method) + } +} + +func gorgen7_AccountNewCall(method string) (args any, reply any) { + switch method { + case "Deposit": + return &gorgen7_Account7_DepositRequest{}, &gorgen7_Account7_DepositReply{} + case "Reset": + return &gorgen7_Account5_ResetRequest{}, &gorgen7_Account5_ResetReply{} + case "Snapshot": + return &gorgen7_Account8_SnapshotRequest{}, &gorgen7_Account8_SnapshotReply{} + case "Tick": + return &gorgen7_Account4_TickRequest{}, &gorgen7_Account4_TickReply{} + default: + return nil, nil + } +} + +func gorgen7_AccountNewReminderCall(method string, status gor.TickStatus) (args any, reply any) { + switch method { + case "Tick": + return &gorgen7_Account4_TickRequest{A0: status}, &gorgen7_Account4_TickReply{} + default: + return nil, nil + } +} + +func gorgen7_AccountNewProxy(rt gor.Invoker, id gor.GrainId) domain.Account { + return &gorgen7_AccountProxy{id: id, rt: rt} +} + +// InstallAccount installs the generated bindings for Account in rt. +func InstallAccount(rt *gor.Runtime) error { + return gor.InstallType[domain.Account](rt, generatedCodeVersion, gorgen7_AccountGrainType, gorgen7_AccountDispatch, gorgen7_AccountNewProxy, gorgen7_AccountNewCall, gorgen7_AccountNewReminderCall) +} + +// Install installs the generated Grain bindings in rt. +// Call it once after creating rt and before registering or referencing any of +// the generated Grain types. After it returns nil, gor.Register and gor.Ref +// can use those types with rt. +func Install(rt *gor.Runtime) error { + if err := InstallAccount(rt); err != nil { + return err + } + return nil +} diff --git a/cmd/gorgen/testfixture/endtoend/domain/gorgen/runtime_test.go b/cmd/gorgen/testfixture/endtoend/domain/gorgen/runtime_test.go new file mode 100644 index 0000000..4b16506 --- /dev/null +++ b/cmd/gorgen/testfixture/endtoend/domain/gorgen/runtime_test.go @@ -0,0 +1,122 @@ +package gorgen + +import ( + "context" + "testing" + "time" + + "github.com/suraciii/gor" + "github.com/suraciii/gor/cmd/gorgen/testfixture/endtoend/domain" + "github.com/suraciii/gor/store" +) + +func TestGeneratedAccountPersistsAcrossRestart(t *testing.T) { + backend := store.NewMemory() + + first := newRuntime(t, backend) + account := gor.Ref[domain.Account](first, "alice") + if value, err := account.Deposit(context.Background(), 7); err != nil || value != 7 { + t.Fatalf("Deposit = (%d, %v), want (7, nil)", value, err) + } + stopRuntime(t, first) + + second := newRuntime(t, backend) + account = gor.Ref[domain.Account](second, "alice") + if value, label, err := account.Snapshot(context.Background()); err != nil || value != 7 || label != "account" { + t.Fatalf("Snapshot = (%d, %q, %v), want (7, account, nil)", value, label, err) + } + if err := account.Reset(context.Background()); err != nil { + t.Fatalf("Reset = %v, want nil", err) + } + stopRuntime(t, second) + + third := newRuntime(t, backend) + account = gor.Ref[domain.Account](third, "alice") + defer stopRuntime(t, third) + if value, label, err := account.Snapshot(context.Background()); err != nil || value != 0 || label != "account" { + t.Fatalf("Snapshot after reset = (%d, %q, %v), want (0, account, nil)", value, label, err) + } +} + +func TestGeneratedDefaultGrainTypeReadsExistingState(t *testing.T) { + backend := store.NewMemory() + if _, err := backend.Write(context.Background(), store.GrainId{ + GrainType: "domain.Account", + GrainKey: "alice", + }, []byte(`{"balance":7}`), 0); err != nil { + t.Fatal(err) + } + rt := newRuntime(t, backend) + defer stopRuntime(t, rt) + + value, label, err := gor.Ref[domain.Account](rt, "alice").Snapshot(context.Background()) + if err != nil || value != 7 || label != "account" { + t.Fatalf("Snapshot existing State = (%d, %q, %v), want (7, account, nil)", value, label, err) + } +} + +func TestNewAccountReminderCallBuildsTypedRequest(t *testing.T) { + status := gor.TickStatus{ + ReminderName: "dynamic/wake", + FirstTickTime: time.Unix(10, 0).UTC(), + Period: time.Minute, + CurrentTickTime: time.Unix(20, 0).UTC(), + } + args, reply := gorgen7_AccountNewReminderCall("Tick", status) + typedArgs, ok := args.(*gorgen7_Account4_TickRequest) + if !ok || typedArgs.A0 != status { + t.Fatalf("Reminder factory Tick args = %#v, want TickStatus %#v", args, status) + } + if _, ok := reply.(*gorgen7_Account4_TickReply); !ok { + t.Fatalf("Reminder factory Tick reply = %T, want generated Tick reply", reply) + } + args, reply = gorgen7_AccountNewReminderCall("Missing", status) + if args != nil || reply != nil { + t.Fatalf("Reminder factory Missing = (%T, %T), want (nil, nil)", args, reply) + } +} + +func TestNewAccountCallUnknownMethodReturnsNil(t *testing.T) { + args, reply := gorgen7_AccountNewCall("Missing") + if args != nil || reply != nil { + t.Fatalf("Call factory Missing = (%T, %T), want (nil, nil)", args, reply) + } + + args, reply = gorgen7_AccountNewCall("Reset") + if _, ok := args.(*gorgen7_Account5_ResetRequest); !ok { + t.Fatalf("Call factory Reset args = %T, want generated Reset request", args) + } + if _, ok := reply.(*gorgen7_Account5_ResetReply); !ok { + t.Fatalf("Call factory Reset reply = %T, want generated Reset reply", reply) + } +} + +func newRuntime(t *testing.T, backend *store.Memory) *gor.Runtime { + t.Helper() + rt, err := gor.New( + gor.WithStore(backend), + gor.WithReminderStore(backend), + gor.WithIdleTimeout(0), + gor.WithEvictionInterval(0), + ) + if err != nil { + t.Fatal(err) + } + if err := Install(rt); err != nil { + t.Fatal(err) + } + if err := gor.Register[domain.Account](rt, domain.NewAccount); err != nil { + t.Fatal(err) + } + if err := rt.Start(context.Background()); err != nil { + t.Fatal(err) + } + return rt +} + +func stopRuntime(t *testing.T, rt *gor.Runtime) { + t.Helper() + if err := rt.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown runtime: %v", err) + } +} diff --git a/cmd/gorgen/testfixture/endtoend/gorgen/generated.go b/cmd/gorgen/testfixture/endtoend/gorgen/generated.go deleted file mode 100644 index 39c1980..0000000 --- a/cmd/gorgen/testfixture/endtoend/gorgen/generated.go +++ /dev/null @@ -1,124 +0,0 @@ -package gorgen - -import ( - "context" - "fmt" - - "github.com/suraciii/gor" - "github.com/suraciii/gor/cmd/gorgen/testfixture/endtoend/domain" -) - -type accountProxy struct { - id gor.GrainId - rt gor.Invoker -} - -type accountDepositRequest struct { - A0 int64 -} -type accountDepositReply struct { - R0 int64 -} - -func (p *accountProxy) Deposit(ctx context.Context, amount int64) (int64, error) { - var reply accountDepositReply - err := p.rt.Invoke(ctx, p.id, "Deposit", &accountDepositRequest{A0: amount}, &reply) - return reply.R0, err -} - -type accountResetRequest struct{} -type accountResetReply struct{} - -func (p *accountProxy) Reset(ctx context.Context) error { - var reply accountResetReply - err := p.rt.Invoke(ctx, p.id, "Reset", &accountResetRequest{}, &reply) - return err -} - -type accountSnapshotRequest struct{} -type accountSnapshotReply struct { - R0 int64 - R1 string -} - -func (p *accountProxy) Snapshot(ctx context.Context) (int64, string, error) { - var reply accountSnapshotReply - err := p.rt.Invoke(ctx, p.id, "Snapshot", &accountSnapshotRequest{}, &reply) - return reply.R0, reply.R1, err -} - -type accountTickRequest struct { - A0 gor.TickStatus -} -type accountTickReply struct{} - -func (p *accountProxy) Tick(ctx context.Context, status gor.TickStatus) error { - var reply accountTickReply - err := p.rt.Invoke(ctx, p.id, "Tick", &accountTickRequest{A0: status}, &reply) - return err -} - -func dispatchAccount(ctx context.Context, instance domain.Account, method string, args any, reply any) error { - switch method { - case "Deposit": - typedArgs := args.(*accountDepositRequest) - typedReply := reply.(*accountDepositReply) - r0, err := instance.Deposit(ctx, typedArgs.A0) - typedReply.R0 = r0 - return err - case "Reset": - err := instance.Reset(ctx) - return err - case "Snapshot": - typedReply := reply.(*accountSnapshotReply) - r0, r1, err := instance.Snapshot(ctx) - typedReply.R0 = r0 - typedReply.R1 = r1 - return err - case "Tick": - typedArgs := args.(*accountTickRequest) - err := instance.Tick(ctx, typedArgs.A0) - return err - default: - return fmt.Errorf("unknown method %q", method) - } -} - -func newAccountCall(method string) (args any, reply any) { - switch method { - case "Deposit": - return &accountDepositRequest{}, &accountDepositReply{} - case "Reset": - return &accountResetRequest{}, &accountResetReply{} - case "Snapshot": - return &accountSnapshotRequest{}, &accountSnapshotReply{} - case "Tick": - return &accountTickRequest{}, &accountTickReply{} - default: - return nil, nil - } -} - -func newAccountReminderCall(method string, status gor.TickStatus) (args any, reply any) { - switch method { - case "Tick": - return &accountTickRequest{A0: status}, &accountTickReply{} - default: - return nil, nil - } -} - -func newAccountProxy(rt gor.Invoker, id gor.GrainId) domain.Account { - return &accountProxy{id: id, rt: rt} -} - -// Install installs the generated Grain bindings in rt. -// Call it once after creating rt and before registering or referencing any of -// the generated Grain types. After it returns nil, gor.Register and gor.Ref -// can use those types with rt. -func Install(rt *gor.Runtime) error { - if err := gor.InstallType[domain.Account](rt, dispatchAccount, newAccountProxy, newAccountCall, newAccountReminderCall); err != nil { - return err - } - return nil -} diff --git a/cmd/gorgen/testfixture/endtoend/gorgen/runtime_test.go b/cmd/gorgen/testfixture/endtoend/gorgen/runtime_test.go deleted file mode 100644 index 5fe3edb..0000000 --- a/cmd/gorgen/testfixture/endtoend/gorgen/runtime_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package gorgen - -import ( - "context" - "testing" - "time" - - "github.com/suraciii/gor" - "github.com/suraciii/gor/cmd/gorgen/testfixture/endtoend/domain" - "github.com/suraciii/gor/store" -) - -func TestGeneratedAccountPersistsAcrossRestart(t *testing.T) { - backend := store.NewMemory() - - first := newRuntime(t, backend) - account := gor.Ref[domain.Account](first, "alice") - if value, err := account.Deposit(context.Background(), 7); err != nil || value != 7 { - t.Fatalf("Deposit = (%d, %v), want (7, nil)", value, err) - } - first.Close() - - second := newRuntime(t, backend) - account = gor.Ref[domain.Account](second, "alice") - if value, label, err := account.Snapshot(context.Background()); err != nil || value != 7 || label != "account" { - t.Fatalf("Snapshot = (%d, %q, %v), want (7, account, nil)", value, label, err) - } - if err := account.Reset(context.Background()); err != nil { - t.Fatalf("Reset = %v, want nil", err) - } - second.Close() - - third := newRuntime(t, backend) - account = gor.Ref[domain.Account](third, "alice") - defer third.Close() - if value, label, err := account.Snapshot(context.Background()); err != nil || value != 0 || label != "account" { - t.Fatalf("Snapshot after reset = (%d, %q, %v), want (0, account, nil)", value, label, err) - } -} - -func TestNewAccountReminderCallBuildsTypedRequest(t *testing.T) { - status := gor.TickStatus{ - FirstTickTime: time.Unix(10, 0).UTC(), - Period: time.Minute, - CurrentTickTime: time.Unix(20, 0).UTC(), - } - args, reply := newAccountReminderCall("Tick", status) - typedArgs, ok := args.(*accountTickRequest) - if !ok || typedArgs.A0 != status { - t.Fatalf("newAccountReminderCall(Tick) args = %#v, want TickStatus %#v", args, status) - } - if _, ok := reply.(*accountTickReply); !ok { - t.Fatalf("newAccountReminderCall(Tick) reply = %T, want *accountTickReply", reply) - } - args, reply = newAccountReminderCall("Missing", status) - if args != nil || reply != nil { - t.Fatalf("newAccountReminderCall(Missing) = (%T, %T), want (nil, nil)", args, reply) - } -} - -func TestNewAccountCallUnknownMethodReturnsNil(t *testing.T) { - args, reply := newAccountCall("Missing") - if args != nil || reply != nil { - t.Fatalf("newAccountCall(Missing) = (%T, %T), want (nil, nil)", args, reply) - } - - args, reply = newAccountCall("Reset") - if _, ok := args.(*accountResetRequest); !ok { - t.Fatalf("newAccountCall(Reset) args = %T, want *accountResetRequest", args) - } - if _, ok := reply.(*accountResetReply); !ok { - t.Fatalf("newAccountCall(Reset) reply = %T, want *accountResetReply", reply) - } -} - -func newRuntime(t *testing.T, backend store.Store) *gor.Runtime { - t.Helper() - rt, err := gor.New(gor.WithStore(backend), gor.WithIdleTimeout(0), gor.WithEvictionInterval(0)) - if err != nil { - t.Fatal(err) - } - if err := Install(rt); err != nil { - t.Fatal(err) - } - if err := gor.Register[domain.Account](rt, domain.NewAccount); err != nil { - t.Fatal(err) - } - return rt -} diff --git a/cycle_test.go b/cycle_test.go index 094b36c..57652e8 100644 --- a/cycle_test.go +++ b/cycle_test.go @@ -14,28 +14,28 @@ import ( "github.com/suraciii/gor/store" ) -// chainEntity is a test entity whose Chain method walks a ring of entity +// chainGrain is a test grain whose Chain method walks a ring of grain // keys: it calls the first key's Chain with the rest of the ring. A test // constructs a cycle by closing the ring on a key already walked. -type chainEntity interface { +type chainGrain interface { Chain(context.Context, []string) error Block(context.Context) error } -type chainEntityImpl struct { - b *Binder +type chainGrainImpl struct { + b *GrainContext blockStarted chan struct{} blockRelease chan struct{} } -func (e *chainEntityImpl) Chain(ctx context.Context, ring []string) error { +func (e *chainGrainImpl) Chain(ctx context.Context, ring []string) error { if len(ring) == 0 { return nil } - return Ref[chainEntity](e.b, ring[0]).Chain(ctx, ring[1:]) + return Ref[chainGrain](e.b, ring[0]).Chain(ctx, ring[1:]) } -func (e *chainEntityImpl) Block(ctx context.Context) error { +func (e *chainGrainImpl) Block(ctx context.Context) error { if e.blockStarted != nil { close(e.blockStarted) } @@ -55,7 +55,7 @@ type chainBlockRequest struct{} type chainBlockReply struct{} -func dispatchChain(ctx context.Context, instance chainEntity, method string, args any, reply any) error { +func dispatchChain(ctx context.Context, instance chainGrain, method string, args any, reply any) error { switch method { case "Chain": return instance.Chain(ctx, args.(*chainRequest).A0) @@ -77,49 +77,49 @@ func newChainCall(method string) (args any, reply any) { } } -type chainEntityProxy struct { +type chainGrainProxy struct { invoker Invoker id GrainId } -func (p *chainEntityProxy) Chain(ctx context.Context, ring []string) error { +func (p *chainGrainProxy) Chain(ctx context.Context, ring []string) error { var reply chainReply return p.invoker.Invoke(ctx, p.id, "Chain", &chainRequest{A0: ring}, &reply) } -func (p *chainEntityProxy) Block(ctx context.Context) error { +func (p *chainGrainProxy) Block(ctx context.Context) error { var reply chainBlockReply return p.invoker.Invoke(ctx, p.id, "Block", &chainBlockRequest{}, &reply) } -func installChainWithFactory(t *testing.T, rt *Runtime, factory func(*Binder) chainEntity) { +func installChainWithFactory(t *testing.T, rt *Runtime, factory func(*GrainContext) chainGrain) { t.Helper() - if err := InstallType[chainEntity](rt, dispatchChain, func(invoker Invoker, id GrainId) chainEntity { - return &chainEntityProxy{invoker: invoker, id: id} - }, newChainCall, nil); err != nil { + if err := InstallType[chainGrain](rt, GeneratedCodeVersion, "gor.chainGrain", dispatchChain, func(invoker Invoker, id GrainId) chainGrain { + return &chainGrainProxy{invoker: invoker, id: id} + }, newChainCall, noReminderCall); err != nil { t.Fatal(err) } - if err := Register[chainEntity](rt, factory); err != nil { + if err := Register[chainGrain](rt, factory); err != nil { t.Fatal(err) } } func installChain(t *testing.T, rt *Runtime) { t.Helper() - installChainWithFactory(t, rt, func(b *Binder) chainEntity { - return &chainEntityImpl{b: b} + installChainWithFactory(t, rt, func(b *GrainContext) chainGrain { + return &chainGrainImpl{b: b} }) } func installBlockingChain(t *testing.T, rt *Runtime, started, release chan struct{}) { t.Helper() - installChainWithFactory(t, rt, func(b *Binder) chainEntity { - return &chainEntityImpl{b: b, blockStarted: started, blockRelease: release} + installChainWithFactory(t, rt, func(b *GrainContext) chainGrain { + return &chainGrainImpl{b: b, blockStarted: started, blockRelease: release} }) } func chainID(key string) GrainId { - return GrainId{GrainType: TypeName[chainEntity](), GrainKey: key} + return GrainId{GrainType: GrainType("gor.chainGrain"), GrainKey: key} } func assertCallCycle(t *testing.T, err error, keys ...string) { @@ -141,11 +141,12 @@ func assertCallCycle(t *testing.T, err error, keys ...string) { } } -func TestCallCycle_TwoEntityCycleNamesBothEntities(t *testing.T) { +func TestCallCycle_TwoGrainCycleNamesBothGrains(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installChain(t, rt) + mustStart(t, rt) ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() @@ -154,11 +155,12 @@ func TestCallCycle_TwoEntityCycleNamesBothEntities(t *testing.T) { }) } -func TestCallCycle_ThreeEntityCycleNamesWholeCycle(t *testing.T) { +func TestCallCycle_ThreeGrainCycleNamesWholeCycle(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installChain(t, rt) + mustStart(t, rt) ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() @@ -170,8 +172,9 @@ func TestCallCycle_ThreeEntityCycleNamesWholeCycle(t *testing.T) { func TestCallCycle_SelfCallIsACycle(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installChain(t, rt) + mustStart(t, rt) ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() @@ -183,10 +186,11 @@ func TestCallCycle_SelfCallIsACycle(t *testing.T) { func TestCallCycle_SelfCallDuringActivationIsRejected(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() - installChainWithFactory(t, rt, func(b *Binder) chainEntity { - return &selfCallingEntity{b: b} + defer closeRuntime(rt) + installChainWithFactory(t, rt, func(b *GrainContext) chainGrain { + return &selfCallingGrain{b: b} }) + mustStart(t, rt) ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() @@ -195,32 +199,33 @@ func TestCallCycle_SelfCallDuringActivationIsRejected(t *testing.T) { }) } -// selfCallingEntity calls itself from OnActivate. The triggering call occupies -// the entity from admission on, so the activation-time call back into it is a +// selfCallingGrain calls itself from OnActivate. The triggering call occupies +// the grain from admission on, so the activation-time call back into it is a // cycle rather than a wait for an activation that cannot complete. -type selfCallingEntity struct { - b *Binder +type selfCallingGrain struct { + b *GrainContext } -func (e *selfCallingEntity) Chain(ctx context.Context, ring []string) error { - return Ref[chainEntity](e.b, ring[0]).Chain(ctx, ring[1:]) +func (e *selfCallingGrain) Chain(ctx context.Context, ring []string) error { + return Ref[chainGrain](e.b, ring[0]).Chain(ctx, ring[1:]) } -func (*selfCallingEntity) Block(context.Context) error { +func (*selfCallingGrain) Block(context.Context) error { return nil } -func (e *selfCallingEntity) OnActivate(ctx context.Context) error { - return Ref[chainEntity](e.b, Self(e.b).GrainKey).Chain(ctx, []string{Self(e.b).GrainKey}) +func (e *selfCallingGrain) OnActivate(ctx context.Context) error { + return Ref[chainGrain](e.b, Self(e.b).GrainKey).Chain(ctx, []string{Self(e.b).GrainKey}) } func TestCallCycle_SlowCallWithoutCycleTimesOutPlainly(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) started := make(chan struct{}) release := make(chan struct{}) installBlockingChain(t, rt, started, release) + mustStart(t, rt) ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() @@ -246,7 +251,7 @@ func TestCallCycle_SlowCallWithoutCycleTimesOutPlainly(t *testing.T) { }) } -func TestCallCycle_ForwardedCycleNamesBothEntities(t *testing.T) { +func TestCallCycle_ForwardedCycleNamesBothGrains(t *testing.T) { synctest.Test(t, func(t *testing.T) { start := time.Unix(1500, 0).UTC() fakeClock := clock.NewFake(start) @@ -261,10 +266,12 @@ func TestCallCycle_ForwardedCycleNamesBothEntities(t *testing.T) { secondOptions = append(secondOptions, WithTransport(secondTransport)) first := mustNew(t, firstOptions...) second := mustNew(t, secondOptions...) - defer first.Close() - defer second.Close() + defer closeRuntime(first) + defer closeRuntime(second) installChain(t, first) installChain(t, second) + mustStart(t, first) + mustStart(t, second) synctest.Wait() <-firstTransport.served <-secondTransport.served @@ -285,7 +292,7 @@ func findChainOwner(t *testing.T, rt *Runtime, owner string) GrainId { view := rt.clusterView.Load() for index := 0; index < 4096; index++ { candidate := chainID(fmt.Sprintf("chain-%d", index)) - if candidateOwner, ok := cluster.Owner(*view, store.GrainId(candidate)); ok && candidateOwner == owner { + if candidateOwner, ok := cluster.Owner(*view, toStoreGrainID(candidate)); ok && candidateOwner == owner { return candidate } } diff --git a/deactivation_reason_test.go b/deactivation_reason_test.go index 55829d1..b7f7647 100644 --- a/deactivation_reason_test.go +++ b/deactivation_reason_test.go @@ -35,12 +35,13 @@ func TestDeactivationReason_Idle(t *testing.T) { WithIdleTimeout(time.Second), WithEvictionInterval(time.Second), ) - defer rt.Close() - installLifecycleAccount(t, rt, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.deactivateReasons = reasons + defer closeRuntime(rt) + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateReasons = reasons }) + mustStart(t, rt) - id := GrainId{GrainType: TypeName[lifecycleAccount](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: "alice"} if err := rt.Invoke(context.Background(), id, "Value", &lifecycleAccountValueRequest{}, &lifecycleAccountValueReply{}); err != nil { t.Fatalf("initial Value: %v", err) } @@ -58,6 +59,34 @@ func TestDeactivationReason_Idle(t *testing.T) { }) } +func TestDeactivationReason_ApplicationRequested(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + reasons := make(chan DeactivationReason, 1) + factoryCalls := new(atomic.Int32) + rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + installLifecycleAccount(t, rt, factoryCalls, func(grain *lifecycleAccountGrain) { + grain.deactivateReasons = reasons + }) + mustStart(t, rt) + + ref := Ref[lifecycleAccount](rt, "alice") + if err := ref.Deactivate(context.Background()); err != nil { + t.Fatalf("Deactivate: %v", err) + } + synctest.Wait() + if reason := <-reasons; reason != ApplicationRequested { + t.Fatalf("deactivation reason = %v, want ApplicationRequested", reason) + } + if _, err := ref.Value(context.Background()); err != nil { + t.Fatalf("Value on new Activation: %v", err) + } + if got := factoryCalls.Load(); got != 2 { + t.Fatalf("factory calls = %d, want 2", got) + } + }) +} + // TestDeactivationReason_ContextIsBackground pins the lifecycle context // contract: the hook receives a fresh context with no deadline that is never // canceled, independent of any caller's context. @@ -70,12 +99,13 @@ func TestDeactivationReason_ContextIsBackground(t *testing.T) { WithIdleTimeout(time.Second), WithEvictionInterval(time.Second), ) - defer rt.Close() - installLifecycleAccount(t, rt, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.deactivateContexts = contexts + defer closeRuntime(rt) + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateContexts = contexts }) + mustStart(t, rt) - id := GrainId{GrainType: TypeName[lifecycleAccount](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: "alice"} if err := rt.Invoke(context.Background(), id, "Value", &lifecycleAccountValueRequest{}, &lifecycleAccountValueReply{}); err != nil { t.Fatalf("initial Value: %v", err) } @@ -107,19 +137,20 @@ func TestDeactivationReason_RuntimeClosed(t *testing.T) { reasons := make(chan DeactivationReason, 1) releaseHook := make(chan struct{}) rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - installLifecycleAccount(t, rt, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.deactivateReasons = reasons - entity.releaseDeactivate = releaseHook + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateReasons = reasons + grain.releaseDeactivate = releaseHook }) + mustStart(t, rt) - id := GrainId{GrainType: TypeName[lifecycleAccount](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: "alice"} if err := rt.Invoke(context.Background(), id, "Value", &lifecycleAccountValueRequest{}, &lifecycleAccountValueReply{}); err != nil { t.Fatalf("initial Value: %v", err) } closeDone := make(chan struct{}) go func() { - rt.Close() + closeRuntime(rt) close(closeDone) }() synctest.Wait() @@ -150,10 +181,11 @@ func TestDeactivationReason_FaultedPanic(t *testing.T) { synctest.Test(t, func(t *testing.T) { reasons := make(chan DeactivationReason, 1) rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() - installLifecycleAccount(t, rt, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.deactivateReasons = reasons + defer closeRuntime(rt) + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateReasons = reasons }) + mustStart(t, rt) if err := Ref[lifecycleAccount](rt, "alice").Panic(context.Background()); err == nil { t.Fatal("panicking method returned a nil error") @@ -175,7 +207,7 @@ func TestDeactivationReason_FaultedPanic(t *testing.T) { } // TestDeactivationReason_FaultedDiscard covers the discard mapping: a failed -// state write makes the entity discard the current instance and the hook sees +// state write makes the grain discard the current instance and the hook sees // Faulted. func TestDeactivationReason_FaultedDiscard(t *testing.T) { synctest.Test(t, func(t *testing.T) { @@ -185,10 +217,11 @@ func TestDeactivationReason_FaultedDiscard(t *testing.T) { WithIdleTimeout(0), WithEvictionInterval(0), ) - defer rt.Close() - installLifecycleAccount(t, rt, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.deactivateReasons = reasons + defer closeRuntime(rt) + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateReasons = reasons }) + mustStart(t, rt) err := Ref[lifecycleAccount](rt, "alice").SetValue(context.Background(), 5) if err == nil { @@ -224,12 +257,13 @@ func TestDeactivationReason_IdleSurvivesClose(t *testing.T) { WithIdleTimeout(time.Second), WithEvictionInterval(time.Second), ) - installLifecycleAccount(t, rt, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.deactivateReasons = reasons - entity.releaseDeactivate = releaseHook + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateReasons = reasons + grain.releaseDeactivate = releaseHook }) + mustStart(t, rt) - id := GrainId{GrainType: TypeName[lifecycleAccount](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: "alice"} if err := rt.Invoke(context.Background(), id, "Value", &lifecycleAccountValueRequest{}, &lifecycleAccountValueReply{}); err != nil { t.Fatalf("initial Value: %v", err) } @@ -238,7 +272,7 @@ func TestDeactivationReason_IdleSurvivesClose(t *testing.T) { closeDone := make(chan struct{}) go func() { - rt.Close() + closeRuntime(rt) close(closeDone) }() synctest.Wait() @@ -274,9 +308,10 @@ func TestDeactivationReason_OwnershipLost(t *testing.T) { network := newTestTransportNetwork() first := mustNew(t, clusterRuntimeOptions(backend, members, fakeClock, "node-a", "generation-a", network.add("node-a"))...) reasons := make(chan DeactivationReason, 1) - installLifecycleAccount(t, first, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.deactivateReasons = reasons + installLifecycleAccount(t, first, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateReasons = reasons }) + mustStart(t, first) before := cluster.NewView([]store.Member{{ NodeAddr: "node-a", @@ -297,9 +332,9 @@ func TestDeactivationReason_OwnershipLost(t *testing.T) { }) var target GrainId for index := 0; index < 4096; index++ { - candidate := GrainId{GrainType: TypeName[lifecycleAccount](), GrainKey: strconv.Itoa(index)} - beforeOwner, beforeOK := cluster.Owner(before, store.GrainId(candidate)) - afterOwner, afterOK := cluster.Owner(after, store.GrainId(candidate)) + candidate := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: strconv.Itoa(index)} + beforeOwner, beforeOK := cluster.Owner(before, toStoreGrainID(candidate)) + afterOwner, afterOK := cluster.Owner(after, toStoreGrainID(candidate)) if beforeOK && afterOK && beforeOwner == "node-a" && afterOwner == "node-b" { target = candidate break @@ -315,6 +350,7 @@ func TestDeactivationReason_OwnershipLost(t *testing.T) { second := mustNew(t, clusterRuntimeOptions(backend, members, fakeClock, "node-b", "generation-b", network.add("node-b"))...) registerAccount(t, second) + mustStart(t, second) synctest.Wait() fakeClock.Advance(time.Second) synctest.Wait() @@ -327,7 +363,7 @@ func TestDeactivationReason_OwnershipLost(t *testing.T) { default: t.Fatal("deactivation hook did not run after ownership change") } - first.Close() - second.Close() + closeRuntime(first) + closeRuntime(second) }) } diff --git a/design/README.md b/design/README.md index bdc56a1..4f9154e 100644 --- a/design/README.md +++ b/design/README.md @@ -18,10 +18,12 @@ When a document diverges significantly from the code, it lists a "Gap" section i - [persistence.md](persistence.md) — state storage, CAS, backend choice. - [timers.md](timers.md) — persisted Reminders: the table, the poller, delivery semantics. - [cluster.md](cluster.md) — membership, placement, directory consistency. -- [transport.md](transport.md) — byte transport between nodes: frames, connections, multiplexing, and close semantics; the substrate boundary for forwarding. -- [errors.md](errors.md) — stable error codes, the call error envelope, and the cross-node cancellation boundary. +- [transport.md](transport.md) - byte Transport between Silos. +- [errors.md](errors.md) - stable codes, the Call error envelope, and the + Cross-Silo cancellation boundary. - [request-context.md](request-context.md) — the per-Call Request Context API, encoding, lifetime, and failure rules. -- [codegen.md](codegen.md) — typed proxies generated from Go interfaces. +- [codegen.md](codegen.md) - typed Grain Reference implementations from Go + interfaces. - [testing.md](testing.md) — unit tests and deterministic simulation tests. - [simulation.md](simulation.md) — the simulation skeleton: seed, fault injection, crashes, event log. - [observability.md](observability.md) — minimal runtime observability facts and performance bounds. diff --git a/design/api-documentation.md b/design/api-documentation.md index 9abf596..84fd0f0 100644 --- a/design/api-documentation.md +++ b/design/api-documentation.md @@ -34,10 +34,16 @@ Distinguish packages first, then judge symbols; "all exported names" cannot be t | `gor`, called directly by users | The package doc and every independently usable entry point get a contract: startup and shutdown, registration and references, state, scheduling, lifecycle, observability, errors, options, and the seams generated artifacts call. | | Extension packages users implement or pass to `gor`: `clock`, `store`, `transport` | Package doc, interfaces and their methods, constructors, closable resources, errors, state values, and fields that affect implementer correctness all get contracts. | | `cmd/gorgen` | A package doc for the command stating inputs, artifacts, and failure exits; concrete flags defer to the command help and `design/codegen.md`. The generated `Install` that applications call at startup also gets an English comment. | -| Implementation packages in the architecture: `runtime`, `mail`, `timer`, `cluster` | Each package gets at least a doc stating its responsibility and the "applications must not depend on this directly" boundary. Exporting a name does not automatically make it a supported API; no useless per-name comments. | -| `internal/`, test fixtures, example applications, test facilities under the `sim` build tag only | This spec does not apply. They are not part of gor's public surface. | +| Importable implementation package: `cluster` | The package gets at least a doc stating its responsibility and the "applications must not depend on this directly" boundary. Exporting a name does not make it a supported Application API. | +| `internal/mail`, `internal/runtime`, `internal/timer`, other `internal/` packages, test fixtures, example applications, and test facilities under the `sim` build tag | This spec does not apply. They are not part of gor's public surface. | -`gor` is the architecture's public API and configuration assembly layer. `clock`, `store`, and `transport` appear in its public configuration or interfaces; users must be able to implement or pass them, so they are a supported extension surface. The other production packages, though importable in Go, are not promised for direct application use; the package doc must say so, not imply it by blank space. +`gor` is the architecture's public API and configuration assembly layer. +`clock`, `store`, and `transport` appear in its public configuration or +interfaces. Users must be able to implement or pass them, so they are a +supported extension surface. `cluster` remains importable for internal +assembly but is not a supported Application API. Mailbox, execution Runtime, +and Reminder poller code are under `internal/` and cannot be imported by an +Application. A declaration inside a supported package gets its own comment if and only if users need to decide or act on it directly: @@ -59,7 +65,8 @@ The comment starts from the symbol's name. First say at which step of the caller 2. After failure, cancellation, or close, what the caller may no longer assume; whether side effects that already happened may still exist. 3. Resources and retry responsibilities owned by the caller, the implementer, or the callback, respectively. 4. Concurrency, ordering, blocking, and lifecycle constraints: whether calls can be concurrent, what happens after close, whether a callback may block. -5. Boundaries that change the caller's choices: true only locally, or true cross-node only after encoding. +5. Boundaries that change the caller's choices: true only locally, or true + Cross-Silo only after encoding. Plain data names do not have to answer all five mechanically. Conversely, whenever an item changes correct usage, it must not be omitted just because the sentence gets longer. `State.Set`'s persistence failure, `Runtime.Done`'s termination meaning, the store's ETag conflicts, the transport's request-completion boundary — all are contracts of this kind that cannot be guessed from a signature. @@ -72,7 +79,7 @@ A doc comment is not a second user manual. The table below gives each kind of in | The full mental model of Grains, GrainIds, calls, state, and Reminders; the complete path of cross-Grain calls | `docs/programming-model.md` | | What may be relied on, version upgrades, and breaking changes | `docs/compatibility.md` and release notes | | Activation cache, mailbox, CAS tables, polling, membership table, frame format, algorithm trade-offs | The relevant `design/` document | -| Multi-node limitations, protocol details, the full matrix of configuration combinations | `docs/` or the relevant `design/` document | +| Cluster limits, protocol details, the full matrix of configuration combinations | `docs/` or the relevant `design/` document | | Full startup tutorial, code-generation flow, end-to-end examples | `docs/`, `examples/shadow/`, and command help | | Issues, task numbers, design-document links, historical rationale | Not written; history belongs to git log | @@ -82,7 +89,9 @@ A comment may state one local restriction; it must not restate a whole model jus gor adds no Go `Example` functions and sets none as a release gate. -A realistic root-package call must at least define a Grain interface, generate proxies, build a runtime, install the artifacts, register factories, and then obtain a reference and call. It is not a lightweight example demonstrating one declaration. Inside an `Example`, it would compile and run on every `make test`; any v0 change to artifact shape, startup order, or public signatures would mean maintaining a third call path, plus text results when `Output` is present. `docs/example.md` and `examples/shadow/` already carry the full path; duplicating it would not improve v0.1's contract clarity. +A complete root-package example must define and generate a Grain interface. It +must also start a Runtime, install code, register a factory, and make a Call. +The device shadow example already tests this complete path. Later, an `Example` is added only for a stable scenario that a local comment cannot explain and that genuinely deserves to run directly on pkg.go.dev. Before adding, all of the following must hold: no dependence on real time, network, or processes; no hidden generation step; runs within the default test constraints; its output expresses a stable observable contract. Otherwise keep it as a documentation snippet or an example application; do not create a test entry that rots. @@ -92,11 +101,18 @@ The release candidate's API documentation batch is accepted in this order: 1. Step 6c is done, and the candidate commit's public declarations leave no undecided API shape versus the product docs. 2. List the supported entry points and the aggregate fields intentionally omitted from independent comments per this document's package categories, and review the reasons by hand; no fake metric of "every exported name non-empty". -3. Review the signature-only pages with `go doc .`, `go doc ./store`, `go doc ./clock`, `go doc ./transport`, and `go doc -cmd ./cmd/gorgen`. For implementation packages, confirm the package doc states the no-direct-dependency boundary. +3. Review the signature-only pages with `go doc .`, `go doc ./store`, `go doc ./clock`, `go doc ./transport`, and `go doc -cmd ./cmd/gorgen`. Confirm that public signatures do not expose an `internal/` type. For `cluster`, confirm the package doc states the no-direct-dependency boundary. 4. Run `make ci`. Comment changes do not alter behavior, but the release candidate must still pass the full gate. From then on, the same change that adds or alters a supported declaration must update its comment in the same change. A comment left unchanged while public behavior changed is a stale contract; "the implementation already explains it" is not a reason to keep it. ## Gap -The candidate API documentation batch is in place: the supported packages (`gor`, `clock`, `store`, `transport`, and `cmd/gorgen`) carry package docs and per-symbol contracts on their independently usable entry points, and the implementation packages (`runtime`, `mail`, `timer`, `cluster`) carry package docs stating the no-direct-dependency boundary. The manual review this section's acceptance step 2 prescribes has been performed: the root `Activation` alias keeps no independent comment because `Activations()` documents the aggregate as a sorted snapshot of the runtime's active Grains, the `GrainId` field is a documented type, and `Queued` is a plain count of calls awaiting dispatch with no separate default, failure, concurrency, or lifecycle rule — the aggregate exception applies. No Go `Example` functions exist, as prescribed. +The candidate API documentation batch is in place. The supported packages +(`gor`, `clock`, `store`, `transport`, and `cmd/gorgen`) carry package docs and +per-symbol contracts on their independently usable entry points. `cluster` +carries the no-direct-dependency boundary. The execution packages are under +`internal/`. The manual review in acceptance step 2 is complete. The root +`Activation` type keeps no independent field comments because `Activations()` +defines the snapshot, `GrainId` is documented, and `Queued` is a plain count. +No Go `Example` functions exist, as required. diff --git a/design/architecture.md b/design/architecture.md index 2047655..2906905 100644 --- a/design/architecture.md +++ b/design/architecture.md @@ -4,54 +4,68 @@ The diagram shows only direct imports of production packages. Solid lines are edges that exist in the current code. -``` -gor ────────▶ runtime ────────▶ mail - │ └────────────▶ clock - ├───────────▶ store - ├───────────▶ timer ──────────▶ clock - │ └──────────▶ store - ├───────────▶ clock - └───────────▶ cluster ────────▶ clock - └──────────▶ store - -gor ──────────▶ transport +```text +gor ----------> internal/runtime ----------> internal/mail + | |------------------> clock + |------------> store + |------------> internal/timer ------------> clock + | |------------------> store + |------------> clock + |------------> cluster -------------------> clock + | |------------------> store + |------------> transport ``` `sim` exists only under the `sim` build tag and depends on production packages to set up test scenarios; it is not part of this production dependency diagram. `cluster` and `transport` are both optional capabilities, and `gor` uses both; `transport` remains its own package. `cluster` does not import `transport`. It only computes remote ownership; `gor` takes that address and forwards. Deciding "who owns it" and "how to get it there" are two things; making the former know the latter only drags a network stack into ring tests. -`timer` knows only interfaces: a table, a `Clock`, and something that can initiate calls. `gor` wires the `store` implementation and `runtime` onto it. +`internal/runtime` owns Activation-local Grain Timers. They use the Activation +mailbox and the injected `Clock`. They do not use a Store. `internal/timer` +owns only the persisted Reminder poller. It knows a table, a `Clock`, and +something that can start Calls. `gor` wires the Store and Runtime onto it. -Dependencies point only downward. `runtime` does not know `cluster` exists, and **does not need to leave any interface for it**. +Dependencies point only downward. `internal/runtime` does not know `cluster` +exists, and does not need to leave an interface for it. -[Step 6b](../ROADMAP.md#6b-forwarding)'s routing happens in the `gor` layer: every call first asks the ring who owns this GrainId; if it is self, hand to `runtime`; if someone else, forward (see [cluster.md](cluster.md)). `runtime`'s interface does not change one word — it is about "calls on the same key are serialized", unrelated to why this key lands on this node. +[Step 6b](../ROADMAP.md#6b-forwarding)'s routing happens in the `gor` layer: +every Call first asks the ring who owns this GrainId. A local Call goes to +`internal/runtime`; a remote Call is forwarded (see [cluster.md](cluster.md)). +The execution interface is only about serial Calls on one key. It does not +depend on placement. -`runtime` also does not import `store`: Grain State is read and written by -`gor` inside the factory closure; `runtime` only hands out a GrainId and gets -back an opaque instance (see [persistence.md](persistence.md)). +`internal/runtime` also does not import `store`: Grain State is read and +written by `gor` inside the factory closure. The execution runtime only hands +out a GrainId and gets back an opaque instance (see +[persistence.md](persistence.md)). -The only thing `runtime` gains because of the cluster is an entry to drop activations by GrainId: after a view change, `gor` uses it to drop Grains that no longer belong to this node. It shares the idle-eviction path and does not reveal the cluster's existence. +The only cluster-facing execution entry drops Activations by GrainId. After a +view change, `gor` uses it to drop Grains that no longer belong to this Silo. +It shares the idle-eviction path and does not reveal the cluster. -**Single-node mode injects nothing**; `gor` takes the local branch directly. No fake implementation that always returns this node is built to "leave an extension point" — that is living dead code; not one line of it is needed in steps 1 through 5. +**Single Silo mode injects nothing**. `gor` takes the local branch directly. +It does not add a fake implementation that always returns this Silo. Steps 1 +through 5 do not need this code. ## Package responsibilities | Package | Responsibility | Not its job | | --- | --- | --- | -| `gor` | Public API, configuration assembly | Any algorithm | -| `runtime` | Activation cache, lifecycle, local directory, request dispatch | Network, storage implementations | -| `mail` | The serial execution queue of a single Grain | Knowing what a Grain is | +| `gor` | Public API, public contract types, configuration assembly | Any algorithm | +| `internal/runtime` | Activation cache, lifecycle, local directory, request dispatch, Grain Timers | Network, storage implementations | +| `internal/mail` | The serial execution queue of a single Grain | Knowing what a Grain is | | `store` | State read/write plus the CAS table abstraction and its backends | Knowing Grain semantics | -| `timer` | Scan due, claim, deliver (see [timers.md](timers.md)) | Knowing Grain semantics | -| `cluster` | Membership table, node state machine, view polling, consistent-hash ring (see [cluster.md](cluster.md)) | Executing Grain methods, forwarding | -| `transport` | Byte transport between nodes | The semantics of the encoding format | +| `internal/timer` | Scan, claim, and deliver persisted Reminders (see [timers.md](timers.md)) | Activation-local Grain Timers | +| `cluster` | Membership table, Silo state machine, view polling, consistent-hash ring (see [cluster.md](cluster.md)) | Executing Grain methods, forwarding | +| `transport` | Byte transport between Silos | The semantics of the encoding format | | `sim` | Fake network, fake clock, fault injection, invariant assertions | Production code paths | | `cmd/gorgen` | The code generator | Runtime behavior | ## Three boundary rules -**Execution and decision are separated.** `mail` only makes "these calls run one after another"; it does not decide who should run or where. `runtime` decides. Mixed together, scheduling could not be exhaustively tested on its own. +**Execution and decision are separated.** `internal/mail` only makes Calls run +one after another. It does not decide who should run or where. +`internal/runtime` decides. This keeps mailbox behavior testable on its own. **All I/O sits behind interfaces.** `store`, `transport`, clocks — no exceptions. This is the hard precondition of DST in [testing.md](testing.md); one violation leaves an entire path impossible to simulate. @@ -59,13 +73,18 @@ The only thing `runtime` gains because of the cluster is an entry to drop activa ## Encoding -Inter-node transport needs encoding. **No self-invented serialization format.** +Cross-Silo transport needs encoding. **No self-invented serialization format.** -Orleans has 30k lines of serialization code, mostly for version tolerance (old and new nodes interoperating during rolling upgrades). The Go side does not pursue this capability: `gor` assumes every node in a cluster runs the same version of the binary, and incompatibilities during rolling upgrades are resolved at the application layer through downtime or dual-write schemes. +Orleans has 30k lines of serialization code. Much of it supports old and new +Silos during a rolling upgrade. `gor` does not provide this capability. It +requires every Silo in a Cluster to run the same binary version. The +Application must handle version changes. The cost is explicit: **rolling upgrades to incompatible method signatures without downtime are not supported.** The gain is an entire subsystem not built. -`encoding/json` is chosen, the same story as Grain state persistence. The reason is no second serialization story, plus humans can read it directly in production — worth more than performance when debugging cross-node problems. +`encoding/json` is the transport format and the State format. This keeps one +serialization model. A person can also read it when a Cross-Silo problem +occurs. **No `Codec` interface.** An interface with one implementation is ceremony. When encoding really needs to change, the change is in the few encode/decode lines, not in the shape of an interface. @@ -75,12 +94,16 @@ Encoding happens in the `gor` layer. `transport` moves opaque bytes and does not The Grain model follows Orleans. The Go implementation does not copy all Orleans source. The source of the size difference is recorded. Orleans' -`src/` measures 274k lines, of which only about 26k need rebuilding on the Go -side (directory, membership, Activation, placement, hash ring, and Reminder -scheduler), because: +`src/` measures 274k lines. About 26k lines cover runtime areas that are +relevant to `gor`. This is not a 0.1.0 implementation estimate. Catalog and +scheduler behavior inform Single Silo. Directory, membership, placement, and +the hash ring inform a future Cluster. - Serialization (31k lines) — not done, as said above. -- Cloud provider code (about 26k lines) — replaced by the embedded store and Postgres backends. +- Cloud provider code (about 26k lines) — not required for Single Silo. + Postgres remains an unprovided target for a future Cluster. - The `src/api/` baseline snapshot (35k lines) — not implementation code at all. -And the Go side can save even more: the scheduler needs 823 lines of custom `TaskScheduler` in .NET, versus a goroutine plus a channel in Go, about 100 lines. Details in [research/orleans-internals.md](../research/orleans-internals.md) (in Chinese). +The Go runtime does not need the custom .NET `TaskScheduler`. It uses a +goroutine and a channel. See +[research/orleans-internals.md](../research/orleans-internals.md). diff --git a/design/benchmarks.md b/design/benchmarks.md index 45602d0..963c58c 100644 --- a/design/benchmarks.md +++ b/design/benchmarks.md @@ -23,8 +23,10 @@ Each of the three gets its own benchmark; no composite score. A composite score ## What is not measured - **No comparison with Orleans.** The runtimes, GCs, and serialization all differ; the resulting numbers explain nothing and become a freely quotable marketing line. -- **No per-node QPS.** It depends on what the user's method bodies do; it has nothing to do with the library. -- **Cross-node forwarding is not merged into the three above.** [6b](../ROADMAP.md#6b-forwarding) is implemented; measure "how much more expensive forwarding is than local" separately, rather than compositing it with single-process absolute numbers. +- **No per-Silo QPS.** It depends on the Application methods. It is not a + property of the library. +- **Cross-Silo forwarding is separate.** [6b](../ROADMAP.md#6b-forwarding) is + implemented. Measure its cost against a local Call. ## Every number must carry its measurement conditions diff --git a/design/cluster.md b/design/cluster.md index a07171b..ab5c553 100644 --- a/design/cluster.md +++ b/design/cluster.md @@ -6,13 +6,18 @@ `gor` has no Raft, no Paxos, no quorum logic of any kind. Linearizability is outsourced to a **shared table with CAS**. -This is not a compromise; it is copying a proven approach. Grep `quorum|consensus|Raft|Paxos` in Orleans' `MembershipService`: **zero hits** — it relies on a shared table plus ETag/CAS, heartbeat probing, and death votes with expiry. Measured details in [research/orleans-internals.md](../research/orleans-internals.md) (in Chinese). +This follows the Orleans approach. A source search for +`quorum|consensus|Raft|Paxos` in Orleans `MembershipService` has zero matches. +It uses a shared table with ETag/CAS, probes, and death votes that expire. See +[research/orleans-internals.md](../research/orleans-internals.md). -The cost is explicit: **the shared table is a single point of failure.** While the table is unavailable, the cluster cannot change membership (existing nodes keep serving). This cost buys the removal of an entire consensus implementation; it is worth it. +The shared table is a single point of failure. While it is unavailable, the +Cluster cannot change membership. Existing Silos continue to serve Calls. +This limit removes the need for a consensus implementation. ## Membership -One table, one row per node: +One table, one row per Silo: ``` member(node_addr, generation, status, iam_alive_at, suspect_votes, etag) @@ -20,9 +25,13 @@ member(node_addr, generation, status, iam_alive_at, suspect_votes, etag) This is the shape after step 6c. `suspect_votes` is only written once probing and voting are enabled. -A cluster node must be configured with both the membership table and the transport; configuring only one of them is an invalid configuration. The membership table provides the shared membership view; the transport provides calls and direct probing to other members; a node missing either one does not join the cluster. +A Cluster Silo must have a membership table and a Transport. A configuration +with only one is invalid. The table gives the shared membership view. The +Transport gives Calls and direct probes. A Silo without both does not join. -The primary key is (node_addr, generation). **`generation` is a fresh value taken at every node start**; after a restart, the same address gets a new row. Without it, a restarted node would claim the row of its previous incarnation, while others may still be casting death votes on that row. +The primary key is (`node_addr`, `generation`). Each Silo start creates a new +`generation`. A restart at the same address creates a new row. Thus, a +restarted Silo cannot claim its old row while other Silos vote on that row. ### The table's interface @@ -30,37 +39,68 @@ Three operations: - **Write your own row** — CAS with the etag. Joining, heartbeat, and state changes all go through here. A zero etag means "this row must not exist yet", the same convention as the state table. - **Write someone else's vote** — the current probe neighbor can CAS-update the target row's `suspect_votes` with the etag. Declaring death also writes the target row. -- **Read the whole table** — the node computes its view from it. +- **Read the complete table** - the Silo computes its view from it. -No convenience methods like "read one row" or "read only the live ones". The view must be computed from one full-table snapshot; reading in fragments lets a node see mutually contradictory pieces. +The store does not have methods to read one row or only live rows. The Silo +computes its view from one complete snapshot. Fragment reads can give +inconsistent data. -No "delete a row" either. Dead rows stay; they are how a restarted node recognizes its previous incarnation. Cleanup is an operations concern, not the runtime's. +The store does not delete a row. Dead rows let a restarted Silo find its +previous generation. Cleanup is an operations task. -One full-table read returns a `MemberSnapshot`: a single snapshot of the rows plus `TableNow`. `TableNow` comes from the injected `Clock` held by the membership table and shared by all clients. It is the only time base for vote expiry. A node's own `Clock` cannot be used to judge other nodes' votes, because node clocks can be offset. +One complete table read returns a `MemberSnapshot`. It contains the rows and +`TableNow`. The membership table gets `TableNow` from its injected `Clock`. +All Silos use it for vote expiry. A Silo must not use its local `Clock` for +this decision because Silo clocks can have offsets. -Production code also takes time from this `Clock`. The membership table and the node code must not call `time.Now()` directly. +Production code also gets time from this `Clock`. The membership table and +the Silo component must not call `time.Now()` directly. -### The node state machine +### The Silo state machine ``` joining → active → dead ``` -**Joining**: write your own row as `joining` (zero etag), read the whole table, then CAS to `active`. The write-then-read order is deliberate: it guarantees that any node that can see itself is also seen by the other side. +**Joining**: write the Silo row as `joining` with a zero ETag. Read the complete +table. Then use CAS to set the row to `active`. This order makes the Silo +visible before it computes its first view. -**Heartbeat**: periodically CAS-update your own `iam_alive_at` against the table's `TableNow`. +The startup context limits all join steps. A membership write can succeed in +the table but return an error to the Silo. After a failed join, the Silo reads +the table with a separate context and the same table-latency limit as a +self-check. It finds its row and uses the latest ETag to write `dead`. For the +first write, the Silo also checks the join time before it changes the row. The +returned error includes a cleanup error when the cleanup does not succeed. -**Leaving**: CAS yourself to `dead` on `Close()`. A node that leaves cleanly does not need others to declare it dead. +If startup is canceled just after the row becomes `active`, Runtime uses a +bounded clean leave. It waits for at most the table-latency limit. It then uses +an abrupt stop if the leave write is still blocked. -**Declaring death**: decided only by the current neighbors' unexpired `suspect_votes`. A stale `iam_alive_at` is not evidence of death. +**Heartbeat**: periodically use CAS to update `iam_alive_at` with `TableNow`. -`dead` is terminal. A declared-dead node must not modify its own row even if it is still alive — its CAS fails on the etag mismatch, and then it must self-terminate; it must not keep serving under a GrainId the whole world considers dead. +**Leaving**: use CAS to set the Silo row to `dead` on `Close()`. Other Silos do +not need to declare a cleanly stopped Silo dead. -**But a CAS failure alone is not proof of your own death.** A heartbeat CAS collision has two causes: someone else changed the row to `dead`, or the previous heartbeat actually landed and only the reply was lost on the way — the latter advances the etag without the node knowing. Both causes give the same signal, and self-termination is irreversible, so on a collision the node must read the whole table again: if its row is `dead` it self-terminates; otherwise it takes the fresh etag and keeps heartbeating. +The leave write uses a separate context. `Kill()` cancels this context. Thus, +an abrupt stop can interrupt a blocked leave write and complete the stop. -Self-terminating without reading would let one dropped packet kill a healthy node. +**Declaring death**: only current and valid `suspect_votes` can declare death. +A stale `iam_alive_at` value is not death proof. -A declared-dead node does not crawl back on its own. Coming back means re-joining: a fresh generation, a new row. +`dead` is terminal. A declared-dead Silo must not modify its own row. Its CAS +fails on an ETag mismatch, and it must stop. It must not continue to serve +Grains after the Cluster declares it dead. + +**A CAS failure alone is not proof of Silo death.** Another Silo can set the +row to `dead`. A heartbeat can also commit while its reply is lost. Both cases +change the ETag. After a Conflict, the Silo reads the complete table. It stops +if its row is `dead`. Otherwise, it uses the new ETag and continues. + +Without this read, one lost reply can stop a healthy Silo. + +A declared-dead Silo does not become active again. It must rejoin with a new +generation and row. ### The view is computed only from rows read from the table @@ -74,47 +114,84 @@ Death must go through the table; only then does the matter have an answer everyo ### After self-termination, the runtime must also stop -"Must not modify your own row" is not enough. A declared-dead node still holds several activations whose ETags are stale, while new calls were already routed to other nodes. +The row rule is not sufficient. A declared-dead Silo can still hold +Activations with old ETags while new Calls go to other Silos. -So when a node sees its current generation as `dead` in a successfully read snapshot, it must report the cause "declared dead externally" to the root runtime. The root runtime first stops admitting Grain calls and closes the public stop signal, then follows the abrupt stop: cancel executing calls, reject the queue, and drop activations. Calls after that return an error that the node has stopped serving. In the view a dead node computes, it owns nothing — but rejecting local calls cannot wait for the view to change; the old view may still assign some GrainId to it for a while. +When a Silo reads its current generation as `dead`, it reports an external +death declaration to the root Runtime. The Runtime stops Call admission and +closes the stop signal. It then cancels active Calls, rejects queued Calls, and +drops Activations. Later Calls return the stopped-Silo error. This rejection +must not wait for a membership view change. -An active `Close()` also writes the node's membership row as `dead`. That is only a normal leave — the root runtime already began a graceful stop — and the cluster node's completion signal must not be mistaken for an external death declaration. The cluster node must hand its end reason to the root runtime; a bare `Done` channel that carries no reason is not enough. +An active `Close()` also writes the Silo row as `dead`. This is a normal leave +after the root Runtime starts graceful stop. The Cluster Silo component must +give its end reason to the root Runtime. A `Done` channel without the reason is +not sufficient. -**An embedding application must be able to know this.** The runtime provides a channel whose closing means "no longer serving"; `Close()`, `Kill()`, and being declared dead all close it. Without this signal, the application could only guess from every call erroring — and by then it is already serving a service that does not work. +**An embedding Application must know when serving stops.** The Runtime gives a +channel that closes when the Silo stops serving. `Close()`, `Kill()`, and an +external death declaration close it. ### Gap -The cluster node implements an explicit end reason: both an active close and an external death declaration close `Done()`, but only the external death declaration closes `DeclaredDead()`. The root runtime distinguishes the two by this channel, no longer inferring from "did I initiate the close myself". A declared-dead node no longer publishes the final empty view: in the view a dead node computes it owns nothing, but rejecting local calls takes effect immediately at the stop transition through the root admission gate, without waiting for the view to change, so this view must not be published (it would trigger graceful deactivation on view change, conflicting with the abrupt stop). This section is implemented. +The Cluster Silo component has an explicit end reason. Active close and +external death both close `Done()`. Only external death closes +`DeclaredDead()`. The root Runtime uses this channel to distinguish them. A +declared-dead Silo does not publish a final empty view because that view can +start graceful Deactivation during abrupt stop. This behavior is implemented. ## Probing and death votes -Direct probing is the second information source for death judgment. A slow membership table cannot masquerade as probe success; one table read failure can no longer declare every node dead. +Direct probes are the second source of death information. A slow membership +table cannot act as a successful probe. One table read failure cannot declare +every Silo dead. ### Who probes whom -Probing uses a single-point membership ring, not the placement ring's virtual points. Each `active` `(node_addr, generation)` places exactly one point: `hash(node_addr + generation)`. Equal hashes order by the full member ID. +Probing uses a single-point membership ring. It does not use the placement +ring virtual points. Each active (`node_addr`, `generation`) has one point: +`hash(node_addr + generation)`. Equal hashes use the complete member ID. -A node probes its clockwise and counter-clockwise neighbor members. When both directions point at the same member, it probes once. With one `active` member there is no target; with two, each probes the other once; with three or more, every node has two targets. +A Silo probes its clockwise and counter-clockwise neighbors. If both directions +select the same member, it probes once. One active member has no target. With +two members, each probes the other. With three or more, each Silo has two +targets. -The ring is rebuilt on every successful read of a fresh member snapshot. New neighbors start counting failures at zero. In-flight probes to old neighbors are canceled, their failure counts are deleted, and their votes on the old neighbor's row are retracted by CAS. Same address with a different generation is also a new neighbor; it inherits no old count and no old vote. +Each new member snapshot rebuilds the ring. New neighbors start with zero +failures. The Silo cancels probes to old neighbors and deletes their failure +counts. It uses CAS to remove its votes from old neighbor rows. A different +generation at the same address is a new neighbor. -This keeps the probe count per node constant at two. A bigger cluster does not increase any single node's probing load. +This keeps the probe count for each Silo at two. A larger Cluster does not +increase one Silo's probe load. ### The probe path -Probing reuses `Transport.Send` from [transport.md](transport.md). It goes through the same lazy dialing, framing, multiplexing, and fake-transport path. No separate UDP, HTTP, or side-channel sockets. +Probes use `Transport.Send`. They use the same connection, frame, multiplex, +and fake Transport paths. They do not use another network protocol. See +[transport.md](transport.md). -`cluster` does not import `transport`. It only depends on an async `Prober`: give it a target member ID, get back a reply channel. `gor`'s adapter sends the `probe` request defined in [Envelope](#envelope) via `Transport.Send`. The transport's server-side handler dispatches on `kind`; `probe` goes straight to `cluster`, not through a Grain call. +`cluster` does not import `transport`. It depends on an asynchronous `Prober`. +The Prober accepts a target member ID and returns a reply channel. The `gor` +adapter sends the `probe` request through `Transport.Send`. A probe does not +use a Grain Call. -A probe request carries only `kind`. The server's current member ID goes into the ordinary response's `reply`, and the initiator compares it with the target in its snapshot; only an exact match counts as success. A new process reusing the address must not erase votes for an old generation. +A probe request has only `kind`. The response has the current member ID in +`reply`. The source compares it with the target snapshot. Only an exact match +is a success. A new generation at the same address must not remove old votes. -The server replies with its current member ID while its local member is still `active` and not stopping; otherwise it returns an error response with no member ID. Probing itself does not read or write the membership table, does not refresh heartbeats, and is not forwarded a second time. +An active and running Silo replies with its current member ID. Other Silos +return an error without a member ID. A probe does not access the membership +table, refresh a heartbeat, or forward again. -The probe state machine waits on the reply channel, the close signal, and a timeout channel created from the `Clock`. A timeout cancels this `Send`. No `context.WithTimeout`, no wall clock. +The probe state machine waits on reply, close, and `Clock` timeout channels. A +timeout cancels `Send`. It does not use `context.WithTimeout` or wall time. ### Failure thresholds -Each target has its own consecutive-failure count. Success resets it to zero. Timeout, dial failure, connection drop, and error reply each add one. A vote is written only when consecutive failures reach `ProbeFailures`. +Each target has a consecutive-failure count. Success resets it. A timeout, +dial failure, connection loss, or error reply adds one. The Silo writes a vote +when the count reaches `ProbeFailures`. Default parameters: @@ -124,8 +201,8 @@ Default parameters: | `ProbeTimeout` | 500 ms | Shorter than one period, so unfinished probes do not pile up. | | `ProbeFailures` | 3 | One or two dropped packets do not vote; the judgment time stays around the old three-second window. | | `VoteTTL` | 6 s | Twice the three-probe window; two neighbors can converge, and old votes expire quickly. | -| `MaxTickGap` | 2 s | Beyond two probe periods, the node can no longer judge consecutive failures reliably. | -| `MaxTableLatency` | 500 ms | When the node's own membership-table access is slower than one probe, it must not declare others dead. | +| `MaxTickGap` | 2 s | After two probe periods, the Silo cannot reliably judge consecutive failures. | +| `MaxTableLatency` | 500 ms | If table access is slower than one probe, the Silo must not declare another Silo dead. | All of these live in `cluster.Config`. A zero value means "use the default"; only a negative value is an invalid configuration. `VoteTTL` is six times `ProbeInterval`, `MaxTickGap` twice; `ProbeTimeout` and `MaxTableLatency` are each half. @@ -159,43 +236,74 @@ With `n` active members, the required vote count is `min(2, n-1)`: - `n == 2`: one vote from the only neighbor suffices to declare death. - `n >= 3`: both current neighbors must vote. -Only an `active` node that is locally self-check healthy may CAS the target to `dead` on sufficient votes. It keeps the target row's valid votes and uses the current etag. On a CAS conflict it re-reads the table; if the target is already `dead` it is done, and only if it is still `active` does it recompute. +Only an active and locally healthy Silo can use CAS to set a target to `dead`. +It keeps valid votes and uses the current ETag. After a Conflict, it reads the +table again. It only computes again if the target is active. -The threshold is not a majority of all members. Only the two neighbors probe this target directly; demanding more votes would let nodes without probing duty decide death, and bring per-node work back to a shape that grows with the cluster. +The threshold is not a majority of all members. Only two neighbors probe a +target. More votes would let other Silos decide death. It would also increase +work for each Silo as the Cluster grows. ### Symmetric partitions -When a transport partition does not affect the membership table, the table's final state depends on how the two groups sit on the probe ring. With each group contiguous, every boundary node has only one cross-partition vote, short of the two-vote threshold. With the groups interleaved, every node's two neighbors are on the other side; four nodes split 2+2 can CAS all four rows to `dead`. +If a Transport partition does not affect the membership table, the result +depends on the probe ring. In contiguous groups, each boundary Silo gets +one vote. In interleaved groups, both neighbors are in the other group. Four +Silos split 2+2 can set all rows to `dead`. -This is an accepted cost. The membership table has no connectivity topology and no consensus; it cannot distinguish "the other side is dead" from "the other side is alive but unreachable". There is no floor rule of "the last `active` member must not die". Such a rule would only let CAS timing pick an arbitrary survivor; it cannot restore connectivity, nor prove the survivor more trustworthy than the node voted dead. +This is an accepted limit. The membership table has no network topology or +consensus. It cannot distinguish a dead Silo from an unreachable Silo. There +is no rule that keeps the last active member. CAS timing would select an +arbitrary survivor and would not restore connectivity. -At `n == 1` no new death votes are produced, but this is not a global floor: while two nodes are both still `active`, they can simultaneously CAS each other to death, and the table can still end up with zero `active` rows. +At `n == 1`, no new death votes are made. This is not a global minimum. Two +active Silos can set each other to `dead`, and the table can have no active +rows. -After total death, the membership table holds only `dead` rows, every node's `Done()` is closed, and the membership view is empty. A node still running that sees the empty view returns the no-owner error for calls and never falls back to local execution; a node already declared dead returns the stopped-serving error first. Nodes do not resurrect themselves. +After total death, the table has only `dead` rows. Each Silo closes `Done()`, +and the membership view is empty. A running Silo returns the no-owner error. +A declared-dead Silo returns the stopped-serving error. Silos do not restart +themselves. -Recovery requires operations to start at least one node that joins the same membership table with a fresh generation. It restores the membership view after writing a new `joining`/`active` row; old `dead` rows stay. Other nodes join the same way with fresh generations. There is no automatic recovery, and changing an old row back to `active` is not allowed. +Recovery requires an operator to start one Silo with a new generation. It +joins the same membership table and writes new `joining` and `active` statuses. +Old `dead` rows stay. Other Silos join in the same way. Recovery is not +automatic, and an old row must not become active again. ### Self-check -Every probe period, the node checks three things: +Every probe period, the Silo checks three things: - The local `Clock` interval from the previous tick to this one. Going backward or exceeding `MaxTickGap` means a GC pause, a scheduling stall, or a clock jump. - The completion time of one full-table read. - The completion time of one CAS on its own membership row. -If either of the last two exceeds `MaxTableLatency`, or does not finish before its `Clock` timeout, the node enters `unhealthy`. A static clock offset does not affect the first item; only two consecutive readings of the same node are compared. +If either of the last two checks exceeds `MaxTableLatency`, the Silo enters +`unhealthy`. It also enters this mode after a `Clock` timeout. A fixed clock +offset does not affect the first check. It compares consecutive values from +the same Silo. -While `unhealthy`, the node keeps reading the table, heartbeating, and probing, but clears failure counts, writes no new votes and renews none, and declares no deaths from votes. Existing votes expire naturally by TTL. A successful probe can still retract its own vote. +While `unhealthy`, the Silo reads the table, sends heartbeats, and probes. It +clears failure counts, writes no votes, renews no votes, and declares no death. +Existing votes expire by TTL. A successful probe can remove its own vote. -After three full periods with none of the above anomalies, the node returns to `healthy`. Three beats keep a brief recovery from immediately regaining voting rights. +After three healthy periods, the Silo returns to `healthy`. This prevents a +short recovery from restoring voting rights immediately. -A failed self-check is not a reason to self-terminate. A table read failure still keeps the old view; only seeing its own current generation as `dead` in a successfully read snapshot stops the node. A node whose membership-table access merely slowed down thus does not amplify the fault in turn. +A failed self-check does not stop the Silo. After a table read failure, it +keeps the old view. Only a successful read of its generation as `dead` stops +it. Slow table access does not amplify the failure. -### A node voted dead +### A Silo voted dead -Every successful table read, and every re-read after a heartbeat CAS collision, checks the node's own `(node_addr, generation)`. If the state is already `dead`, it immediately takes the existing stop path: drop all activations, reject later calls, and close `Done()`. After that it no longer answers probes. +Each successful table read checks the Silo's own (`node_addr`, `generation`). +The same check runs after a heartbeat CAS Conflict. If the row status is `dead`, +the Silo drops all Activations, rejects later Calls, and closes `Done()`. It no +longer answers probes. -So a node whose transport is cut off but whose process is healthy sees the `dead` its neighbors voted while it can still reach the membership table, and stops itself. It cannot resurrect with one later probe success; resurrection means re-joining with a fresh generation. +A healthy process with a Transport partition can still read the table. It +stops when it sees the `dead` vote from its neighbors. A later probe success +cannot restore it. It must rejoin with a new generation. ### The place of `iam_alive_at` @@ -207,27 +315,37 @@ Step 6c deletes all "CAS to `dead` when stale beyond `DeadAfter`" logic. Table r All of the following are verified in `make sim` with the fake transport and the fake membership table: -- Three nodes share the membership table; one node's transport is partitioned without stopping its process or its table access. The two neighbors each write a vote, the target row becomes `dead`, and the partitioned node closes `Done()`. +- Three Silos share the membership table. One Silo has a Transport partition, + but its process and table access continue. Its two neighbors vote. The row + becomes `dead`, and the partitioned Silo closes `Done()`. - One neighbor leaves a vote after flapping and stops renewing it. Table time passes `VoteTTL`; the other neighbor then also suffers three failures. A healthy target must not be declared dead on one expired vote plus one fresh vote. - Freeze the target's `iam_alive_at` while direct probes keep succeeding. The target stays `active`. -- Inject local clock jumps, GC pauses, and a slow membership table. The node writes no votes, renews none, and declares no deaths from existing votes; it only regains voting after three consecutive healthy beats. -- Four nodes interleave 2+2 on the probe ring while the membership table stays available. All four rows can become `dead` and every `Done()` closes; starting one node with a fresh generation brings `active` members back into the view. +- Inject local clock jumps, GC pauses, and a slow membership table. The Silo + writes and renews no votes. It regains voting after three healthy beats. +- Four Silos interleave 2+2 on the probe ring while the table stays available. + All rows can become `dead`. A new Silo generation restores an active member. ## Placement -A consistent-hash ring: nodes hash onto the ring by address, and Grains land on the first `active` node by hashing their GrainId. +A consistent-hash ring puts Silos on the ring by address. A Grain belongs to +the first active Silo after its GrainId hash. -A hash ring is chosen over "random placement plus directory lookup": **a hash ring makes locating mostly pure local computation, with no network round trip.** The cost is that node changes cause activation migration. +A hash ring makes Ownership a local computation without a network Call. A +Silo membership change can move Activations. No load-based placement. It needs a global load view, and a global view is extremely hard to verify in DST. The scale target is small clusters; a hash ring is enough. ### Hashing must be stable across processes -**`maphash` cannot be used.** It has a random seed per process; two nodes would compute two rings, and a converged view would not help. Use a fixed function written into the code: `hash/fnv` is enough. +**`maphash` cannot be used.** It has a random seed for each process. Two Silos +would compute different rings. Use the fixed `hash/fnv` function. -**Each node places several virtual points on the ring, not one.** With three to five nodes at one point each, the distribution skews badly and one node carries half the keys. The virtual-point count is a constant, not a configuration knob — it is not something users should tune. +**Each Silo has several virtual points on the ring.** One point for each Silo +can give one Silo half the GrainIds. The virtual-point count is a constant. It +is not an Application setting. -A virtual point's position comes from `hash(address + generation + index)`. Including the generation lets a node restarted at the same address land on new positions instead of inheriting its previous incarnation's load distribution. +A virtual point uses `hash(address + generation + index)`. The generation +gives a restarted Silo new positions at the same address. ### The ring is the directory; there is no second table @@ -235,19 +353,26 @@ Placement is computed from `hash(GrainId)` plus the current membership view — Orleans has a directory table because it does not place by hash: it puts activations on chosen silos and uses the ring only to partition the directory, so something must keep the books. `gor` places by ring; the ring itself is the ledger. -A directory table would narrow the double-activation window: two nodes with inconsistent views each register, and CAS makes one lose. But it cannot narrow the window to zero — the loser already activated — and the cost is one more round trip per activation, one more table, and one more degradation path when the table is unavailable. The window must be acknowledged in the docs anyway; narrowing it is not worth that price. +A directory table can make the double-Activation window smaller. Two Silos +with different views register, and CAS makes one fail. It cannot remove the +window because both already activated. It also adds a Call for each Activation +and another table failure mode. `gor` does not add this table. ### When the view changes, drop activations that are no longer yours -After a membership view change, some activations on the node no longer hash to it. **These activations are dropped on the spot**, not left to idle eviction. +After a membership view change, a Silo can lose Ownership of some Activations. +It drops these Activations immediately. -Keeping them has only downsides: they hold stale ETags, so the next write must hit a conflict; and new calls are already routed to the new node, so they will never be served. The drop takes the same path as idle eviction; no new path is added. +These Activations hold old ETags, and their next write can cause a Conflict. +New Calls already go to the new Silo. The drop uses the idle eviction path. ## Directory consistency — the part that must be honest **`gor` does not guarantee a single activation world-wide at any moment.** -While nodes join or leave, nodes' views of the member list briefly disagree, so the same key can be computed to different nodes, producing two activations. The window closes once the membership views converge. +While Silos join or leave, their membership views can differ. They can assign +one GrainId to different Silos and create two Activations. The window closes +when the views converge. Rejecting the directory table above is admitting this window cannot be closed: an arbitration layer only narrows it, and the code users write is the same whether the window is narrow or not. So: @@ -259,20 +384,28 @@ Rejecting the directory table above is admitting this window cannot be closed: a ## Routing -Every call first computes which node `hash(GrainId)` lands on: +Each Call first computes which Silo owns `hash(GrainId)`: -- **Self** — hand to `runtime` as usual, exactly as in single-process mode. -- **Someone else** — forward it (transport in the next section). +- **Self** - hand to `internal/runtime`, as in Single Silo mode. +- **Someone else** - forward it through Transport. -**This step happens in `gor`, not in `runtime`.** `runtime` is about "calls on the same key are serialized"; it must not know the cluster exists, just as it does not know `store` exists today. Ring computation and forwarding both happen in the `gor` layer; `runtime`'s interface does not change. +**This step happens in `gor`, not in `internal/runtime`.** The execution +Runtime serializes Calls on one key. It does not know that the Cluster or +Store exists. Ring computation and forwarding both happen in `gor`. -The ring and the membership view get their own package, shaped like `timer`: it takes a membership-table interface, a `Clock`, and its own address, and periodically reads the full table against the injected clock to compute the view. `gor` wires it up and, on view changes, hands the activations that no longer belong to this node to `runtime` for dropping. +The ring and membership view get their own package. Like `internal/timer`, it +takes a table interface and a `Clock`. It also takes its own address. `gor` +wires it and asks `internal/runtime` to drop Activations that moved after a +view change. -**The ring is a pure function.** Give it a membership view and a GrainId, and it computes a node. It reads no time, does no I/O, holds no state; unit tests feed it views directly. Fetching the view is the stateful half, kept separate from the ring. +**The ring is a pure function.** It computes a Silo from a membership view and +a GrainId. It reads no time, does no I/O, and keeps no data. Tests give views +to it directly. View retrieval stays separate. ## Transport -One long-lived connection between nodes, requests multiplexed, no gRPC. Interface, frame format, and connection lifecycle in [transport.md](transport.md). +Silos use long-lived connections and multiplex requests. They do not use +gRPC. See [transport.md](transport.md). `cluster` does not import `transport` — the ring computes an address, and forwarding is initiated by the `gor` layer. @@ -300,32 +433,44 @@ Step 6b's forwarding has only the `invoke` shape. Step 6c adds `kind` to the sam ### Not forwarded a second time -A node that receives a forwarded call executes it directly, **without computing ownership again**. Its view may differ from the caller's, and recomputing has only two outcomes: bouncing back (a loop waiting to happen) or an error the caller cannot handle. +A Silo executes an inbound forwarded Call without another Ownership check. +Its view can differ from the source view. Another check could forward the Call +back or return an error that the caller cannot resolve. -Not computing ownership is not asking nothing: **a node that is already `dead` or stopping rejects all forwarded calls.** This is a rule about itself; it needs no view, so it cannot fight with anyone else's view. +A Silo that is `dead` or stopping rejects forwarded Calls. This rule needs no +membership view. -When views disagree, two nodes each activate the same key — that is the double-activation window acknowledged above, held off by the ETag against concurrent writes, not by this rule. +When views differ, two Silos can activate the same GrainId. ETags protect +concurrent State writes during this double-Activation window. ### Cancellation does not cross the network -When the caller's ctx is canceled, the forwarding side drops the pending request and returns `ctx.Err()`; the method on the other side keeps running to completion. The server-side context carries no caller cancellation or deadline. Full rules in [errors.md](errors.md). +When the caller context ends, the source Silo drops the pending request and +returns `ctx.Err()`. The target method continues. The target context has no +caller cancellation or deadline. See [errors.md](errors.md). ### Forwarding does not retry -Cannot send, connection dropped, the other side rejected — the error goes straight to the caller. Only the user knows whether retrying is safe; this is the same stance as with `State.Set()` conflicts and Reminder delivery failures. +A send failure, connection loss, or target rejection goes to the caller. The +Application decides if a repeat is safe. This is also the rule for State +Conflicts and Reminder delivery failures. ## Migration -When a node leaves, its activations disappear; requests routed to the new node reactivate them from the store. +When a Silo leaves, its Activations end. Calls to the new Silo reactivate the +Grains from the Store. -**No hot state migration.** State is in the store anyway; rebuilding from the store is the same path, and an extra migration path is a pile of code that only runs on failure. +**No hot State migration.** State is in the Store. Rebuilding from the Store +uses the normal Activation path. `gor` does not add a second migration path. ## Rolling upgrades No-downtime upgrades with incompatible changes are not supported; the reason is in [architecture.md](architecture.md): no version-tolerant encoding. -The same cluster is assumed to run the same version of the binary. This is a capability the project explicitly gives up relative to Orleans. +All Silos in a Cluster must run the same binary version. ## Gap -The current implementation covers this section's membership snapshots, probing, votes with expiry, self-checks, and the node self-termination path. Still uncovered: operational cleanup not listed in step 6c, and rolling-upgrade capability. +The current implementation covers membership snapshots, probes, vote expiry, +self-checks, and the Silo stop path. It does not cover operations cleanup or +rolling upgrades. diff --git a/design/codegen.md b/design/codegen.md index 051aae8..7cb0e9e 100644 --- a/design/codegen.md +++ b/design/codegen.md @@ -8,25 +8,41 @@ Go generics cannot synthesize a type implementing `T` from `T` itself. So acct := gor.Ref[Account](rt, "alice") // returns Account ``` -cannot be done with generics alone. The common solution in similar Go projects is to drop the types: +cannot be done with generics alone. An untyped API can avoid generation: ```go resp, err := system.AskGrain(ctx, grainID, msg, timeout) // any in, any out ``` -goakt does exactly this (measured: `AskGrain(ctx, *GrainIdentity, message any, timeout) (any, error)`, `GrainContext.Message() any` / `Response(any)`). The cost is that all type errors are deferred to runtime. - -`gor` does not accept that cost and goes with code generation. +This shape moves argument errors to runtime. `gor` uses code generation to +keep the Grain interface types. ## The input contract -The generator reads user-written Go interfaces. A valid Grain interface method must: +The generator reads user-written Go interfaces. A valid Grain interface method +must: + +- be exported; +- take `context.Context` as its first parameter; +- return `error` last; +- have no variadic parameter; +- use contract types that the generated package can name. -- take `context.Context` as its first parameter -- return `error` last -- have encodable parameters and return values in between +The marked interface must be exported and must not have type parameters. A +contract type used by generated code must be accessible from the generated +subpackage. The generator rejects an unexported named type in a method +contract. This check is recursive for unnamed compound types and type +arguments. An exported named type is one accessible unit. The generator does +not inspect its private fields. -The number of return values is unrestricted. "Encodable" means `encoding/json` can encode it: local calls pass values through without serialization, but the same method goes through JSON when forwarded, and an unencodable type blows up at exactly that moment, when it is already too late. +The source package must be importable. The generator rejects a marked Grain +interface in `package main`. + +The number of return values is unrestricted. Contract values must work with +`encoding/json` when a Call is forwarded. The generator does not try to prove +this. A type can use custom marshal methods, so a static check would reject +valid code or accept code that later fails for one value. Runtime reports the +encode failure at the Call boundary. ```go type Account interface { @@ -36,24 +52,37 @@ type Account interface { A method that does not satisfy the contract makes the generator report an error with the line number — **no silent skipping**. Silent skipping lets users believe the method was generated and only discover otherwise at runtime. -This contract comes from `alecthomas/go-rpcgen`'s approach (interface + named return values + trailing error), a shape proven in Go. +This contract uses a common Go RPC shape: an interface and a final error +result. ## Which interfaces are generated -Only marked ones are generated: +Only marked interfaces are generated. This document is the authority for the +marker grammar: ```go //gor:grain type Account interface { ... } + +//gor:grain billing.account +type BillingAccount interface { ... } ``` +The bare marker sets GrainType to +`.`. This is the name used by the 0.0.x +Runtime. The second form sets an explicit GrainType. + +The marker must be on the interface declaration. It must occur once. An +explicit value has 1 to 255 ASCII bytes. Its first and last bytes are letters +or digits. Middle bytes can also be `.`, `/`, `_`, or `-`. + The generator ignores the package's other interfaces. The "all exported interfaces in the package" rule is rejected: then an ordinary helper interface would also be checked against the contract, and users would have to move it to another package to silence the generator. The marker is explicit and coexists with "report errors on contract violations": unmarked ones are not checked; marked ones must comply. ## Output -Each interface gets one generated proxy: +Each interface gets one generated Grain Reference implementation: ```go type accountProxy struct { @@ -72,7 +101,8 @@ Plus a server-side dispatch function that turns a method name plus arguments bac ## Arguments and return values each go into a struct -`Invoke` takes one value on each side, while a method can take several parameters and return several values. So **each method generates a pair of structs**, used by both the proxy and the dispatch function: +`Invoke` takes one request and one reply. Each method generates one struct for +each side. The Grain Reference implementation and dispatch function use both: ```go type accountDepositRequest struct { A0 int64 } @@ -81,7 +111,9 @@ type accountDepositReply struct { R0 int64 } No special case for "only one". Passing `&amount` directly for a single value does save one layer, but then the generator has two paths, and two paths each need their own tests and maintenance. Nobody reads generated code; one less layer of indentation is not worth that price. -**Methods with no parameters or only an `error` return still get the empty structs.** Across nodes, this pair of structs is the bytes on the wire: empty structs encode to `{}`, and there is exactly one encode/decode path; without them, an "is it nil" check would be needed before and after encoding. Without a network it is indeed dead code; with a network it is the shortest form of that path. +**Methods with no parameters or only an `error` return still get the empty +structs.** Across Silos, this pair of structs is the data on the wire. Empty +structs encode to `{}`. Thus, all Calls use one encode and decode path. ## Import names @@ -95,24 +127,82 @@ The source package's own import line participates in the same allocation. When i The artifacts land in a subpackage, and the user's interface package does not import it (reason: the type-checking deadlock below). So how does the runtime know where `Account`'s dispatch function lives? -The generator additionally emits an install function: +The generator emits one installer for each Grain and one package installer. +It also emits a stable GrainType: + +```go +const accountGrainType gor.GrainType = "domain.Account" +``` + +Install all Grain types in the package: ```go if err := gorgen.Install(rt); err != nil { return err } ``` -It registers each interface's dispatch function and proxy constructor on `rt`. After that: +As an alternative, install only `Account`: + +```go +if err := gorgen.InstallAccount(rt); err != nil { return err } +``` + +The Application uses one of these forms. It must not install the same Grain +type two times. + +The public generated-code seam is: + +```go +type GrainType string + +func InstallType[T any]( + rt *Runtime, + generatedVersion int, + grainType GrainType, + dispatch func(context.Context, T, string, any, any) error, + newProxy func(Invoker, GrainId) T, + newCall func(string) (any, any), + newReminderCall func(string, TickStatus) (any, any), +) error + +func GrainTypeOf[T any](scope Scope) GrainType +``` + +Generated code passes `generatedVersion`. Application code must use the +generated installer and must not call this seam directly. + +`GrainId.GrainType` uses `GrainType`. The generated constant is not exported. +Application code uses `GrainTypeOf` when it needs the installed value. +`GrainTypeOf` panics with `ErrTypeNotInstalled` when T is not installed in the +scope. This matches `Ref` for a missing generated installation. + +It registers each dispatch function and Grain Reference constructor on `rt`. +After that: ```go gor.Register[Account](rt, factory) // dispatch function taken from the registry -acct := gor.Ref[Account](rt, "alice") // proxy taken from the registry; return type is Account +acct := gor.Ref[Account](rt, "alice") // Grain Reference from the registry; type is Account ``` -`Register` therefore loses one parameter: the hand-written `dispatch` from [step 1](../ROADMAP.md#1-single-process-runtime) is taken over by the generator. This is a planned breaking change. +The generator replaces the manual dispatch from +[step 1](../ROADMAP.md#1-single-silo-runtime). `Register` therefore needs one +less parameter. + +**Automatic registration via `init()` is rejected.** It removes the +`Install(rt)` line, but requires a blank import. A missing import then fails +only at run time. It also makes the registry global to the process. Simulation +tests run several Silos in one process. A Runtime registry and an explicit +`Install` call prevent both problems. -**Automatic registration via `init()` is rejected.** It saves the `Install(rt)` line, at the cost of users having to remember a blank import — forget it and the failure only shows up at runtime — and the registry becoming a process-wide global, while step 4's simulation tests run several nodes in one process. With the registry on `rt` and `Install` called explicitly, both problems disappear together. +Generated Install passes the GrainType to `gor.InstallType`. The Runtime keeps +one registry by local Go type and one by GrainType. `Register`, `Ref`, and +`GrainTypeOf[T](scope)` use the local type registry. Stored records and encoded +Calls use the GrainType registry. -The generator does not hardcode type-name strings. It emits a generic call like `gor.InstallType[Account](rt, dispatchAccount, newAccountProxy, newAccountCall)`, and `gor` itself computes the names with the same rules as `Register` / `Ref`: one shared function for all three places, so they cannot disagree. +Install rejects a second registration for the same local Go type. It does not +reject a different local type only because its GrainType is already present. +Start validates every GrainType. It then builds the GrainType registry and +rejects two local Go types that declare the same GrainType. This check must +happen before Call admission and Reminder claims. ## How the server rebuilds types from bytes @@ -122,13 +212,18 @@ A forwarded call carries only a method name and a JSON blob (envelope in [cluste func newAccountCall(method string) (args any, reply any) ``` -It builds a pair of empty shells by method name: `"Deposit"` yields `&accountDepositRequest{}` and `&accountDepositReply{}`. An unrecognized method name yields nil for both — something that really happens between nodes on mismatched versions. +It builds a pair of empty shells by method name. `"Deposit"` gives +`&accountDepositRequest{}` and `&accountDepositReply{}`. An unknown method name +gives nil for both. This can occur when Silos run different versions. **The name carries the type, like `dispatchAccount` and `newAccountProxy`.** A package can hold several Grain interfaces; a `newCall` without the type name would not compile once there is a second one. Everything in the artifacts that is generated per type carries the type in its name; no exceptions. -From here on it is all existing machinery: `json.Unmarshal` fills the args, they go through **the same `Invoke`**, and the result comes back as `json.Marshal(reply)`. From this point, forwarded calls and calls initiated by local proxies share one path; serialization, activation, and dispatch are not duplicated. +`json.Unmarshal` fills the arguments. They use the same `Invoke` path. +Forwarded Calls and local Grain References share serialization and dispatch. -**Only `newCall` exists for the network.** Proxies, dispatch functions, and structs existed anyway. This is also why `Invoke`'s argument changed from `[]any` to `any`: `[]any` cannot hold the types back, a struct pointer can. +**Only `newCall` exists for the network.** Grain Reference implementations, +dispatch functions, and structs already exist. `Invoke` uses a struct pointer +to keep the generated type. `Register` returns an error for a type missing from the registry. `Ref` has no error return; it panics when the type is missing. This is not a runtime condition; it is wiring that was never connected: the same type's `Register` reports first at startup, and the `Ref` panic is only a fallback. @@ -150,6 +245,31 @@ This trick comes from `segmentio/glue`: a narrow `Call` interface that fully dec Load packages with `golang.org/x/tools/go/packages`, get type information from `go/types`, emit code with `text/template`. +Generated output starts with the standard generated-code header. It also +contains a literal generated-code version. Every generated `Install` +passes that version to `gor.InstallType`. Installation fails before it changes +the Runtime if the version does not match `gor.GeneratedCodeVersion`. The +check is in each typed installer because an Application can call it without +calling the package `Install` function. + +Generated request and reply names encode the interface and method name +boundaries. Two valid interfaces cannot produce the same generated name. +Generated Grain Reference parameters use stable names such as `ctx`, `arg0`, and +`arg1`. User parameter names do not enter generated identifiers. + +The command renders and formats the complete file in memory. It writes a +temporary file in the output directory, syncs it, and renames it over the old +file. The replacement uses the operating system's atomic replace operation. +A load, render, format, write, sync, close, or replace failure leaves the old +file unchanged. A failed command removes its temporary file when the process +can still run cleanup. + +The formatted artifact uses LF line endings on every platform. The repository +records Go source with LF so a Windows checkout does not change a committed +generated artifact before `-check` compares it. This keeps the generated byte +contract deterministic instead of making the checker ignore platform-created +differences. + **A known pitfall**: `go/types` requires the loaded package to pass type checking. If the artifacts lived in the same package as the user interface, then "the artifacts do not exist yet" → "user code references them → the package fails type checking" → "the generator cannot load the package" — a deadlock. The solution: **the artifacts land in their own package**. The Grain package does not import them; `gor.Register` / `gor.Ref` connect them at runtime through the registry. That package must also be importable from the startup code that calls `Install`, so it is not `internal` — by default it sits at `/gorgen` (see [Invocation](#invocation)). @@ -165,6 +285,10 @@ The rendering layer does not know `go/types`; tests hand-build the model directl The split is not just for testing. Contract violations are reported in the loading layer, and the model the rendering layer receives is compliant by construction — the two layers' responsibilities were always separate. +Command tests also cover the default output path, stale checks, and a failed +atomic write. A clean external module runs its `go:generate` command and +builds generated code without importing an internal gor package. + ## Invocation Add the generator as a tool when you add gor: @@ -182,6 +306,16 @@ Generate after creating or changing a marked interface: go tool gorgen -pkg ./domain ``` +The source package should keep one canonical command: + +```go +//go:generate go tool gorgen -pkg . +``` + +`go tool gorgen -pkg ./domain -check` renders without writing. It returns a +nonzero result when `generated.go` is missing or different. CI runs this check +for each generated package. + The output is a non-`internal` subpackage of the Grain package — `/gorgen` — so the startup code can import it and call `Install`. `-out` picks another directory; the package name is always `gorgen`. `//go:generate` works too. Why a separate `tool` line rather than plain `go run`: the generator depends on `golang.org/x/tools`, which the library never imports, so `go get` of the library alone leaves it out of `go.sum`. Splitting `cmd/gorgen` into its own module would fix the same thing, but it would force `internal/codegen` to leave `internal/`; the `tool` directive keeps the generator in-tree. @@ -200,9 +334,18 @@ No generation runs on `go build`: Go has no such hook, and forcing one would mak Grain methods. The Grain package cannot import its generated package because that would create an import cycle. See [timers.md](timers.md). +The generator accepts a bare marker or a marker with an explicit GrainType. +It generates a stable GrainType. Runtime does not derive this value from Go +type text. Generated output has a fixed header and version. The command uses +atomic replacement and supports `-check`. The Loader rejects unsupported +interface forms with source positions before render. + +There is no open gap in the 0.1.0 generator contract. + ## Rejected approaches -**Runtime reflect-synthesized proxies.** Go's `reflect.MakeFunc` can construct function values but not a type implementing an arbitrary interface. It cannot be done. +**Runtime-generated Grain Reference implementations.** Go reflection cannot +create a type that implements an arbitrary interface. **The other way around: users write a struct, the generator produces the interface.** One less hand-written piece, but users would not see their own API surface — and the API surface is the most important thing callers see. diff --git a/design/conformance-example.md b/design/conformance-example.md index 6ce9bc6..c44b1cc 100644 --- a/design/conformance-example.md +++ b/design/conformance-example.md @@ -76,6 +76,21 @@ business.db Application pending actions and applied records The example runs without membership, so the membership table is unused. The ApplicationStore owns `business.db`; the Runtime owns the other two files. +Both SQLite owners encode an absolute platform path as a file URI before they +add connection options. Integration tests use only file names legal on the +host platform; synthetic URI tests cover reserved characters when necessary. + +Before it opens a file, the command compares the coordination, State, and +Application database families. Each family includes the main file and its +`-wal` and `-shm` sidecars. The command resolves symbolic links and rejects two +members that have the same resolved path or the same existing file identity. +A symbolic link in the surrounding platform path namespace is valid when the +three families still resolve to separate identities. This is required on +systems such as macOS, where a temporary directory can be named through a +system symbolic link. Resolved member names are also compared without case. +This deliberately rejects a case-only distinction even on a case-sensitive +file system, so one configuration has the same isolation meaning on every +supported system. Parent traversal in a supplied path remains invalid. ## Application records @@ -85,7 +100,8 @@ The ApplicationStore has the smallest interface needed by the two Grains: type ApplicationStore interface { SavePending(context.Context, PendingAction) error ListPending(context.Context) ([]PendingAction, error) - ApplyPending(context.Context, string) error + ReadPending(context.Context, string) (PendingAction, bool, error) + CompletePending(context.Context, string) error ReadApplied(context.Context, string) (AppliedRecord, bool, error) Close() error } @@ -99,14 +115,14 @@ action. `SavePending` inserts a new action. Repeating the same ActionID with the same payload is a no-op. Reusing an ActionID with a different payload returns an -application error. `ListPending` returns only pending actions in a stable -ActionID order. +Application error. `ListPending` returns only pending actions in their stable +save order. A repeated save does not change this order. -`ApplyPending` is one application transaction. It reads the action, creates -one applied receipt, and marks the action applied. A unique ActionID makes the -transaction safe to repeat. If the action is already applied, the operation -returns success without creating another receipt or changing the business -result. +`ReadPending` reads one pending action by ActionID. `CompletePending` is one +Application transaction. It creates one applied receipt and removes the +pending action. A unique ActionID makes this transaction safe to repeat. If +the action is already applied, the operation returns success without a new +receipt or a changed business result. The Runtime does not know these records and does not make this transaction atomic with a State or Reminder write. The separate boundary is the reason @@ -116,12 +132,13 @@ the ActionID and Safe Repeat rule are required. ### Start recovery -The application opens both stores, creates a Single Silo, installs generated -bindings, and registers both Grain factories. The factories capture the -ApplicationStore. They still receive the public `*gor.Binder` and create all -Runtime State and Reminder handles from that Binder. +The Application opens both stores and creates a Single Silo. It installs +generated bindings, registers both Grain factories, and calls `Runtime.Start`. +The factories capture the ApplicationStore. They receive the public +`*gor.GrainContext` and create all State and Reminder handles from that Grain +Context. -The first call is: +The first Call is valid only after `Runtime.Start` succeeds: ```go gor.Ref[RecoveryCoordinator](rt, RecoveryCoordinatorKey).Start(ctx) @@ -158,12 +175,12 @@ err = gor.Ref[Device](rt, "device-1").ReportAction(ctx, "report-1", "temperature 3. Read the current shadow State. 4. Write the new shadow with `State.Set`. -The pending save comes before the State write so a deterministic -`ErrPendingActionConflict` leaves Device State unchanged. This is not a -distributed transaction: if a pending save succeeds but the later State write -has an uncertain result, retry the same ActionID. The copied trace ID is -application data. It is not Request Context after the write. Request Context -is not stored in Runtime State or Reminder records. +The pending save comes before the State write. A deterministic +`ErrPendingActionConflict` leaves Device State unchanged. The Application +record and Runtime State use separate writes. Either write can have an Unknown +Result. A Safe Repeat uses the same ActionID. The copied trace ID is +Application data after the save. Request Context is not stored in Runtime +State or Reminder records. The `Recover` method must observe an empty Request Context because the Runtime creates Reminder Calls with a fresh context. @@ -173,10 +190,10 @@ from a present zero value. ### Recover after a stop -The deterministic restart test calls `rt.Kill()` after `Report` returns and -before the next Reminder delivery. It then creates a new Runtime with the -same public stores, installs the same generated bindings, and advances the -injected clock. No private runtime call and no database repair is allowed. +The normal restart test calls `Runtime.Shutdown(ctx)` after `Report` returns and +before the next Reminder delivery. It then closes the stores. The test opens +the same files, creates and starts a new Runtime, and advances the injected +clock. No private Runtime call and no database repair is allowed. The persisted Reminder poller does this for each due row: @@ -185,18 +202,35 @@ The persisted Reminder poller does this for each due row: 3. Build `TickStatus` and invoke `RecoveryCoordinator.Recover` as an ordinary typed Grain Call. -`Recover` lists pending Application actions in ActionID order. For each one it -calls `Device.ApplyPending`. The Device delegates to the ApplicationStore -transaction. A successful recovery leaves one applied record and no pending -record for that ActionID. +`Recover` lists pending Application actions in save order. For each one it +calls `Device.ApplyPending`. The Device uses this order: -The test also wraps the public `ReminderStore` in a test-only adapter that -blocks after a successful `Claim`. The test driver calls public `rt.Kill` -while the claim is blocked, then releases the adapter. This proves the -claim-before-delivery boundary: the claimed due time can be missed, the -Device method does not run, and the pending application record remains. A -periodic Reminder gives the Application a later recovery attempt. The test -does not insert or edit a Reminder row by hand. +1. If the receipt exists, return success. +2. Read the pending action and check its Device GrainKey. +3. Confirm that the requested Grain State is current. Write it when needed. +4. Call `CompletePending` only after the State step succeeds. + +A successful recovery leaves the requested Confirmed State, one applied +record, and no pending record for that ActionID. + +For `ReportAction`, the requested State is the reported value with `Online` +set to true. Recovery keeps `Configuration` and `WorkshopID`. If recovery must +write State, it sets `ReportedAt` to the current injected time. If the +requested value and `Online` mark are already confirmed, it keeps the current +`ReportedAt` and does not write State again. + +Save order is the Business Action order. This rule matters when one Device has +more than one pending action. Recovery applies the actions in that order, so +the last saved action supplies the final reported value. + +The process-stop test runs the Application in a child process. A release-proof +`ReminderStore` adapter prints a marker after a successful `Claim`. It then +blocks before delivery. The parent uses `Process.Kill` at that marker. This +proves the claim-before-delivery boundary. The +claimed due time can be missed, the Device method does not run, and the +pending Application record remains. A periodic Reminder gives the Application +a later recovery attempt. The test does not insert or edit a Reminder row by +hand. ### Safe Repeat @@ -208,7 +242,23 @@ result can also leave the caller unable to know whether a Business Action ran. The recovery method therefore uses this Safe Repeat rule: > For one ActionID, `ApplyPending` may run any number of times, but the -> Application transaction may create one applied receipt only. +> Application transaction may create one applied receipt only. The receipt +> can exist only after the requested State is confirmed. + +If a State write returns an Unknown Result, the pending action remains. The +Runtime discards the Activation. A later Call reads Confirmed State from the +Store. If the requested State is already current, the Call does not write it +again. Otherwise, the Call writes the requested State before it completes the +pending action. + +If `CompletePending` returns an Unknown Result, the State is already +confirmed. A later Call reads the receipt first. It returns success when the +receipt exists. If the action is still pending, it safely repeats the final +Application transaction. + +An existing receipt must not make a later Safe Repeat restore old State. A +later Business Action can change the State after this receipt. The receipt is +the proof that the State step for its action already succeeded. The repeat test invokes `ApplyPending` twice through the typed Device Reference. A second call returns success and does not create a second receipt. @@ -226,11 +276,13 @@ The example always installs both `gor.OnError` and `gor.OnCall`. - A foreground Device or coordinator Call returns its error to the caller. `OnCall` records the method, duration, and error for the test. - A failed `Recover` method has no waiting caller. `OnError` receives the - original error with `ReminderInvocation{Method: "Recover"}`. + original error. The source has name `recovery`, method `Recover`, and the + `TickStatus` of that delivery. - A failed `OnDeactivate` hook is reported with `gor.Deactivation`. -- Reminder scan and claim failures are scheduler failures. They do not enter - `OnError`; the test observes that no Device Call ran and that a failed claim - leaves the row available. The next poll retries the scan. +- Reminder scan, decode, and claim Store errors enter `OnError` with distinct + sources. A lost claim CAS is normal contention and does not enter `OnError`. + The Runtime does not promise that an unknown claim result left the row + available. - A failed Reminder method is not retried for the same due time. The application may recover the pending ActionID on a later periodic tick. @@ -240,7 +292,7 @@ The example documents these intentional Unknown Results: | --- | --- | --- | | Device `State.Set` or `State.Clear` | A non-context store error can mean that the write committed or did not commit. The activation is discarded. | Call again to load confirmed State. Retry a Business Action only with the same ActionID and a Safe Repeat rule. | | `ApplicationStore.SavePending` | The pending row can exist even when `ReportAction` returns an error. | Read by ActionID before creating another action. Retry the same ActionID. | -| `ApplicationStore.ApplyPending` | The application transaction can commit before its result reaches the Grain. | Retry the same ActionID. The unique receipt makes the retry a no-op. | +| `ApplicationStore.CompletePending` | The Application transaction can commit before its result reaches the Grain. | Retry the same ActionID. Read the receipt first. The unique receipt makes the final transaction a no-op. | | `Reminder.Set` or `Reminder.Cancel` | The unconditional write or delete can be complete when the caller sees an error. | Repeat Set by the same name or repeat Cancel. Do not edit Runtime tables. | | Call timeout or cancellation | The caller stopped waiting; the Grain method may have started and may have saved State or an action. | Treat the result as unknown. Query the application record and use the same ActionID before retry. | | Process stop after Reminder claim | The due occurrence may be missed because claim happens before delivery. | Leave the pending action in ApplicationStore and wait for the next periodic Reminder. | @@ -250,10 +302,10 @@ transport failure. The public call contract still treats a transport failure as unknown when a clustered caller uses the API outside this example. The example must not turn any row count, error, or timeout into a false -success. Every error from store open, Runtime creation, registration, a Call, -Reminder setup, ApplicationStore, or close is returned or reported. The only -errors not sent to `OnError` are the scheduler failures and shutdown -cancellations defined by the public Reminder contract. +success. Every error from store open, Runtime creation, registration, Start, a +Call, Reminder setup, ApplicationStore, Shutdown, or close is returned or +reported. A lost Reminder claim CAS and shutdown cancellation do not enter +`OnError`. ## Tests and evidence @@ -263,26 +315,33 @@ polling, private Runtime fields, or manual SQL repair. The minimum test cases are: -1. Install generated bindings, register both Grain types, obtain typed - References, and call the Device and fixed-key coordinator. +1. Install generated bindings, register both Grain types, start the Runtime, + obtain typed References, and call the Device and fixed-key coordinator. 2. Set shadow State, assert `Exists`, clear it, and assert absence after reactivation. Also test a present zero value separately from absence. 3. Add Request Context, copy its value deliberately into Application data, and prove it is absent from Runtime State, Reminder data, and a Reminder Call. 4. Save an action, stop the Runtime before delivery, restart with the same - stores, and recover the action without repair. + stores, and recover the State and receipt without repair. 5. Run two concurrent public `Claim` attempts and assert one winner. A successful claim must precede the Device Call. 6. Stop after a successful claim and before delivery. Assert that the action remains pending and that the next periodic tick recovers it. -7. Fail before an application commit and assert `OnError`, one later retry, - and one applied receipt. +7. Fail before an Application commit and assert `OnError`, one later retry, + Confirmed State, and one applied receipt. 8. Commit then return an injected unknown result. Retry the same ActionID and - assert one applied receipt. + assert Confirmed State and one applied receipt. 9. Invoke the same ApplyPending action twice and assert Safe Repeat behavior. -10. Cancel the recovery Reminder, clear coordinator State, and assert that no +10. Fail a State write before commit. Assert that the receipt does not exist + and the action stays pending. Restart and recover both State and receipt. +11. Commit a State write and return an Unknown Result. Restart and assert that + recovery keeps the confirmed value and completes the pending action. +12. Cancel the recovery Reminder, clear coordinator State, and assert that no later tick runs. +13. Register a Grain Timer through Grain Context. Prove that its callback is + serialized with a Call. Request Deactivate on Idle and prove that the timer + does not move to the next Activation. The existing State, ReminderStore, Request Context, error-sink, and Runtime restart tests remain lower-level evidence. The conformance tests prove their @@ -291,25 +350,36 @@ assert the observed Call, stored record, error source, and nonzero test count. ## Clean module and release gates -The generated file is committed. A clean consumer check must build the -committed example without access to the repository's build cache, then run -the two process phases against a temporary pair of database paths: +The generated file is committed. Run the external release proof with: + +```bash +make external +``` + +The test creates a consumer module outside the repository. It uses empty +`GOMODCACHE` and `GOCACHE` directories and sets `GOWORK=off`. Before the tag, +the module replaces `v0.0.0` with the candidate repository. + +The test builds the committed conformance command once. It runs `prepare` and +`recover` in separate processes with the same SQLite paths. The `recover` +process must verify the requested Grain State and the full receipt. The test +also stops one process after a successful Claim. A later process must recover +the action. Recovery from new databases must fail with a nonzero exit code. + +After the tag exists, the final proof must use exact version `v0.1.0`. It must +not use a local replacement. Run it with: ```bash -consumer_dir="$(mktemp -d)" -mod_cache="$consumer_dir/modcache" -(cd "$consumer_dir" && go mod init conformance-check) -GOMODCACHE="$mod_cache" go run github.com/suraciii/gor/examples/shadow/cmd/conformance@v0.1.0 \ - -phase prepare -db "$consumer_dir/runtime.db" -business-db "$consumer_dir/business.db" -GOMODCACHE="$mod_cache" go run github.com/suraciii/gor/examples/shadow/cmd/conformance@v0.1.0 \ - -phase recover -db "$consumer_dir/runtime.db" -business-db "$consumer_dir/business.db" +make external-tagged ``` -The prepare phase saves the action and exits with Reminder polling disabled. -The recover phase starts a new process with polling enabled and waits for the -observed recovery Call. It then reads the application receipt and exits with a -nonzero status if the receipt is missing or duplicated. The release check -uses the candidate version in place of `v0.1.0` before the tag exists. +This command is fixed to `v0.1.0`. It resolves that version in an empty module +cache. It ignores local module proxy and private-module settings. It uses +public Go module proxies or direct source with the public Go checksum +database. The command fails if the resolved version is different, if Go +reports a replacement, or if the module source is outside that cache. Before +the tag exists, a failed exact-version request is a diagnostic result only. It +is not release proof. The release candidate must pass every existing gate: @@ -325,20 +395,23 @@ make ci `make ci` is the required aggregate gate. It includes format checking, lint, the default tests, race tests, simulation, generated-code tests, and network -tests. The clean consumer build and the two-phase run are additional release -evidence; they do not replace `make ci`. - -## API decision and Gap - -No new framework feature is required. Public `State`, `Reminder`, typed Grain -References, Request Context, `OnError`, `OnCall`, `Kill`, `New`, `Store`, and -`ReminderStore` provide all Runtime seams used here. The ApplicationStore is -application code because the Runtime must not own or interpret business -records. - -The coordinator, ApplicationStore, process driver, generated bindings, and -conformance tests are implemented in `examples/shadow`. The existing shadow -`Device.Report` keeps its workshopID contract, so the conformance path uses the -separate typed `ReportAction` method for an ActionID. This keeps workshopID and -ActionID separate while preserving the Single Silo boundary. The example does -not configure cluster membership, a Transport, or private Runtime state. +tests. `make external` and `make external-tagged` are additional release +evidence. They do not replace `make ci`. + +## API decision and status + +The complete 0.1.0 flow uses public `State`, `Reminder`, Grain Timer, typed +Grain References, Grain Context, Request Context, Deactivate on Idle, +`OnError`, `OnCall`, `New`, `Start`, `Shutdown`, `Store`, and `ReminderStore`. +The process driver stops the process from outside for crash proof. The +ApplicationStore is Application code because the Runtime must not own or +interpret business records. + +The coordinator, ApplicationStore, process driver, and generated bindings are +implemented in `examples/shadow`. The local external proof passes on Linux. +The exact tagged module and hosted systems still need final proof. The example +does not configure Cluster membership, a Transport, or private Runtime State. + +`Device.ApplyPending` confirms the requested State before it completes the +pending action. The Memory and SQLite Application stores keep pending actions +in save order. The process proof reads the recovered State and the receipt. diff --git a/design/constraint-checks.md b/design/constraint-checks.md index d152a3c..7c0b3f5 100644 --- a/design/constraint-checks.md +++ b/design/constraint-checks.md @@ -29,6 +29,10 @@ The checker handles default import names, aliased imports, and dot imports; it c ## The sim fixture boundary -`time.Sleep` in default library source has no exception; even `time.Sleep(0)` is a violation. Only simulation fixtures under `sim/`, whose build constraints explicitly require `sim` and whose files are not `_test.go`, may use `time.Sleep`. Here `Sleep` represents the injected storage response delay, scheduled uniformly by the `testing/synctest` bubble: the shared fake store belongs to no single node, and tying its delays to a node's `Clock` has no clear semantics, especially once node clocks have offsets. +`time.Sleep` in default library source has no exception. Even `time.Sleep(0)` +is a violation. Only simulation fixtures under `sim/` can use `time.Sleep`. +Their build constraint must require `sim`, and the file must not be a test +file. Here, `Sleep` is an injected storage response delay. The shared fake +store belongs to no Silo. Thus, it cannot use one Silo's `Clock`. This boundary is drawn by build target and responsibility, not as a special case for the existing four call sites, and not as a time-API whitelist for the `sim` package: `time.Now`, `time.After`, and the like in sim fixtures are still violations, and `_test.go` under the sim tag still forbids `time.Sleep`, so it cannot become a test synchronization device. diff --git a/design/errors.md b/design/errors.md index c076f90..5952ddd 100644 --- a/design/errors.md +++ b/design/errors.md @@ -2,7 +2,9 @@ ## Goal -Call outcomes can leave the process. Arbitrary Go `error` objects cannot. Cross-node outcomes are therefore projected onto a stable code and text; no attempt to pack the object back. +Call outcomes can leave the process. Arbitrary Go `error` objects cannot. +Cross-Silo outcomes use a stable code and text. The protocol does not encode +the original object. The projection rule looks only at whether the error carries a `Code`. It does not whitelist concrete error types. Framework errors carry their own codes where they are produced; application errors are coded by the application; everything else is an opaque error. @@ -32,7 +34,11 @@ A coded error rebuilt on the remote side also implements `Coded`. Its `Is` compa errors.Is(err, ErrWorkshopIDRequired) ``` -When the error tree has no code, or more than one distinct code, `CodeOf` returns empty, the envelope carries no code, and the source side rebuilds a text-only error; callers cannot match such an error against a code across nodes. This is the full parity scope of `errors.Is`. The remote error is not equal to the original: no arbitrary sentinel is restored, `errors.As` is not guaranteed to yield the original type, and wrap depth, fields, and merged members are not preserved. +When the error tree has no code, or has different codes, `CodeOf` returns an +empty value. The envelope then has no code. The source Silo rebuilds a +text-only error. A caller cannot match this error by code across Silos. The +remote error is not equal to the original error. It does not restore an +arbitrary sentinel, type, wrap depth, field, or merged member. ## The code space @@ -43,15 +49,18 @@ This version's framework code set is sealed as follows: | Code | Outcome | | --- | --- | | `gor.no_owner` | The current view has no routable owner. | -| `gor.node_dead` | The target node has stopped serving. | +| `gor.node_dead` | The target Silo has stopped serving. | +| `gor.runtime_not_started` | A Call was attempted before Start succeeded. | | `gor.runtime_closed` | The runtime or the Grain's mailbox is closed. | -| `gor.overloaded` | The call was rejected for a full queue before the method started. | -| `gor.type_not_installed` | The target node does not have this Grain type. | +| `gor.setup_frozen` | Setup was changed or started after the Runtime left the constructed state. | +| `gor.invalid_setup` | Runtime setup or Store readiness was not valid. | +| `gor.overloaded` | The Call was rejected for a full mailbox or a full Silo Activation limit before the method started. | +| `gor.type_not_installed` | The target Silo does not have this GrainType. | | `gor.unknown_method` | The target type has no such method. | | `gor.invalid_request` | The request's shape, arguments, or Request Context cannot be decoded under the current contract. | | `gor.persistence_conflict` | The state write hit a version conflict. | | `gor.persistence_failed` | The state write failed, and it was not a version conflict. | -| `gor.panic` | The factory or the Grain method panicked. | +| `gor.panic` | The factory, `OnActivate`, Grain method, or reported lifecycle hook panicked. | | `gor.request_encode_failed` | The source could not validate Request Context or encode the invoke request, including its arguments. | | `gor.reply_encode_failed` | The return values of a successful call could not be encoded. | | `gor.transport_failed` | The request, response, or connection failed to transfer; the execution outcome is unknown. | diff --git a/design/observability.md b/design/observability.md index 38bb35b..3729e73 100644 --- a/design/observability.md +++ b/design/observability.md @@ -4,7 +4,8 @@ `gor` is responsible for providing a minimal set of runtime observability facts. -Application-side proxies can count calls but cannot see the activation directory or mailboxes. They cannot reliably answer how many activations exist, nor find Grains with a backlog. Only the runtime holds these facts. +Application-side Grain References cannot see the Activation directory or +mailboxes. Only the Grain Runtime holds these facts. `gor` is not responsible for aggregation, storage, export, or alerting. Those depend on the monitoring system the application already has. Building them into the library would pull in dependencies and decide labels, retention, and sampling policy for the user. @@ -23,15 +24,22 @@ type Activation struct { func (rt *Runtime) Activations() []Activation ``` -`Activations` returns the activations on this node in the `active` state. Instances being created, deactivating, or already stopped are not in the result. +`Activations` returns the active Activations on this Silo. It does not return +an Activation that is starting, deactivating, or stopped. The result is sorted by `(GrainId.Type, GrainId.Key)`. One call returns a copy of one point in time. It is not retained and does not refresh itself. -`len(rt.Activations())` answers how many activations exist right now. `Queued` answers whose mailbox is backing up. Comparing it with the runtime's configured capacity tells how far from overload rejection you are. +`len(rt.Activations())` answers how many Activations are active in this +snapshot. It does not include Activations that are starting or deactivating, so +it does not report the Silo's reserved Activation slots. `Queued` answers whose +mailbox is backing up. Comparing it with the runtime's configured mailbox +capacity tells how far from mailbox overload rejection you are. `Queued` does not include the method currently executing. It only counts calls that entered the mailbox and have not started. -The snapshot observes only this node. In cluster mode, each node collects on its own; cross-node aggregation belongs to the application's monitoring system. +The snapshot observes only this Silo. In Cluster mode, each Silo collects on +its own. Cross-Silo aggregation belongs to the application's monitoring +system. Activation time, last-used time, the executing method, and per-Grain cumulative counts are not exposed. These values would let the runtime make no new decision, yet they grow state, lock contention, and label cardinality. @@ -50,7 +58,9 @@ func OnCall(func(CallObservation)) Option `OnCall` follows the configuration shape of `OnError`. It is a callback, not an exporter interface. There is exactly one action here; inventing a single-method interface for it has no value. -A call made through a Grain proxy or `Runtime.Invoke` fires the callback once, after the outcome is settled and before returning to the caller. `Duration` spans from entering the runtime to the settled outcome, including routing, activation, queuing, and method execution, not the callback itself. `Err` is the same error this call returns to the caller. +A Call made through a Grain Reference or `Runtime.Invoke` fires the callback +once. `Duration` includes routing, Activation, queuing, and method execution. +`Err` is the error returned to the caller. Applications choose metric dimensions with `GrainType` and `Method`, record latency distributions with `Duration`, and compute error rates with @@ -64,9 +74,17 @@ A Grain method delivered by a Reminder is an ordinary Call and produces this event. `OnDeactivate` is not a Call and produces none; its failures still go only through `OnError`. -When a forwarded call completes, the originating node records one end-to-end call. The receiving node must not record the same logical call again. Inbound forwarded calls still go through the same local execution path; no second dispatch semantics are set up. +When a forwarded Call completes, the source Silo records one end-to-end Call. +The target Silo must not record the same logical Call again. An inbound Call +uses the local execution path. -The callback runs synchronously in the call's goroutine, after the outcome is settled. The runtime sets up no queue, goroutine, or clock for it. A callback must not block or do I/O; otherwise it is the caller itself that gets delayed. Callback panics are handled like `OnError` panics: as ordinary user callbacks; the runtime does not recover. +The callback runs synchronously in the Call goroutine after the outcome is +settled. The Runtime sets up no queue, goroutine, or clock for it. A callback +must not block or do I/O because it delays the caller. + +The Runtime catches an `OnCall` panic. The Call keeps its settled result. The +Runtime does not invoke `OnError` for this panic because that could create a +callback cycle. No built-in counters, histograms, traces, or metrics exporters. The callback already hands the application the facts needed to compute error rates and latency quantiles; keeping an aggregate state in addition would only add hot-path work and make reset, label, and export semantics the library's responsibility. @@ -90,9 +108,17 @@ This is not zero cost in the machine-instruction sense. A runtime-configurable f Enabled, each call adds two `Clock.Now` reads, one stack-allocated `CallObservation` value, and one direct callback. The callback's own cost is the application's to bear. -The existing invocation round-trip baseline is 0.89 us/op. The implementation must record both the disabled and enabled-empty-callback results under the same conditions: disabled stays at zero allocations, and sustained regression against the baseline must stay within 5%; the in-library overhead of an enabled empty callback must stay within 20%. These numbers are not CI gates, but any exceedance must prompt a re-examination of the design, not silent acceptance. +The implementation must measure the disabled path and an enabled empty +callback under the same conditions. Both paths must report the same +allocations per Call. The review uses the median `ns/op` from at least ten +samples for each path. It must also record the complete sample range and +`allocs/op`. The median for the empty callback must add no more than 20% to the +disabled median. This limit measures `OnCall`, not other Runtime work. Changes +to the total Call cost belong to [benchmarks.md](../benchmarks.md). These +numbers are not CI gates. An exceedance requires a design review. -`Activations` is an operations query, off the call hot path. Its time and allocation scale with the number of activations on this node. +`Activations` is an operations query outside the Call hot path. Its time and +allocation increase with the number of Activations on this Silo. ## Testing @@ -104,7 +130,20 @@ Unit tests verify the following facts: - An injected fake Clock yields exact `Duration` values. The observation implementation adds no wall-clock reads. - With the callback disabled, no events, Clock reads, allocations, or goroutines are added. - The callback runs synchronously after the call outcome is settled; tests collect callbacks through a buffered channel, with no sleep or polling. +- A callback panic does not change the Call result or stop the Runtime. + +Simulation tests must verify that the source Silo records one logical Call +once. Observation events must not change the decision sequence for a fixed +seed. + +The performance test reuses the invocation round-trip benchmark. One variant +has an empty `OnCall` callback. Both variants report `ns/op` and `allocs/op`. +The result is compared with this document's `OnCall` cost limit. + +## Status -Once forwarding is implemented, simulation tests must also verify that one logical call is recorded once, at the originating node, and that observation events introduce no new decision sequence under a fixed seed. +The Runtime catches an `OnCall` panic. The panic does not change the settled +Call result and does not stop later observations. -The performance test reuses the invocation round-trip benchmark with an added variant carrying an empty `OnCall` callback. Both report `ns/op` and `allocs/op` and are compared with the baseline against this document's cost bounds. +The 2026-08-11 candidate A/B check reports 8.0% overhead for an empty +callback and no additional allocation. See [benchmarks.md](../benchmarks.md). diff --git a/design/persistence.md b/design/persistence.md index 367ef07..7ee53e3 100644 --- a/design/persistence.md +++ b/design/persistence.md @@ -13,6 +13,7 @@ Item 2 is the foundation of `cluster` correctness (see [cluster.md](cluster.md)) ```go type Store interface { + Check(ctx context.Context) error Read(ctx context.Context, id GrainId) (Record, error) Write(ctx context.Context, id GrainId, data []byte, expect ETag) (ETag, error) } @@ -25,6 +26,19 @@ type Record struct { `Write` returns `ErrConflict` when `expect` does not match the current ETag in storage. No retries, no merging. A conflict is a business-semantics problem; the runtime has no right to decide for the user. +`Check` verifies that the Store can serve the configured data before the +Runtime starts. It must not change Application data. `Runtime.Start` calls it +before it starts Call admission or Reminder claims. + +The memory Store returns `ctx.Err()` when the context is done and nil +otherwise. The SQLite Store checks its connections, required columns, and +compound primary keys for State and Reminders. +The separate `ReminderStore.Check` contract is in [timers.md](timers.md). + +The Application owns an injected Store. It closes the Store after Runtime +Shutdown. This order prevents a Call or Reminder worker from using a closed +Store. + An empty record (first access) is represented by the zero ETag; passing the zero ETag to `Write` means "require that this record currently does not exist". **A missing record is not an error.** `Read` on a record that does not exist @@ -34,7 +48,10 @@ present with the type's zero value, or the value is present with other data. ## ETag is not optional -In cluster mode, the directory is eventually consistent: during node failures and network partitions, the same key may have one activation on each of two nodes (see [cluster.md](cluster.md)). The only thing preventing the states from overwriting each other is the ETag. +In Cluster mode, the directory is eventually consistent. During Silo failures +and network partitions, two Silos can each have an Activation for the same +GrainId. The ETag prevents one State write from overwriting the other. See +[cluster.md](cluster.md). So the ETag is not an "advanced feature"; it is a necessary condition for correctness under this architecture. The API offers no write path that bypasses it. @@ -44,10 +61,24 @@ This matches Orleans' stance: when the official docs discuss the eventual consis Two goals: -**Embedded** — the single-node default: SQLite (`modernc.org/sqlite`, pure Go, no CGO). SQLite is the selected embedded backend. bbolt and pebble remain unprovided backend goals. +**Embedded**: SQLite (`modernc.org/sqlite`, pure Go, no CGO) is the selected +Single Silo backend. The Application must select it explicitly. The Runtime +does not select an in-memory Store by default. + +The memory Store remains an explicit test and development backend. It does not +provide Durability. SQLite satisfies both state storage and coordination tables (coordination needs transactions plus CAS; bbolt's single-writer model can do it too, but SQL expresses multi-row conditional updates like membership more directly), and it has the best operational observability — when something goes wrong, you can look directly with `sqlite3`. The cost is being slower than bbolt/pebble, and the pure-Go SQLite is slower still. +SQLite file names are encoded as file URIs before connection options are +added. URI data characters such as `?` and `#` must remain part of the file +name. The URI also keeps the host platform's absolute-path meaning. In +particular, a Windows drive path uses a URI path such as +`file:///C:/data/gor.db`; the drive must not become a URI authority. Derived +State file names use the host platform's path separators. Filesystem tests use +URI data characters that the host permits in file names. A synthetic URI test +covers reserved characters that a platform such as Windows cannot create. + What to measure is how these two `Store` methods behave under real access patterns, not generic read/write throughput: - Point-read one record by key. @@ -57,7 +88,9 @@ What to measure is how these two `Store` methods behave under real access patter Record the conditions with the numbers: record size, concurrency, machine, WAL on or off. Numbers without conditions are not numbers. -**Postgres** — the cluster-mode target. It is the only choice that satisfies all three: shared across multiple nodes, conditional updates for CAS, and the ops team already has it. It remains an unprovided backend goal. +**Postgres** is the target backend for Cluster mode. It supports shared access +from multiple Silos and conditional updates for CAS. This backend is not +provided. No Redis backend: the atomic conditional updates the membership table needs would rely on Lua scripts on Redis, and Redis' persistence semantics make failures like "lost death votes" hard to reason about. @@ -74,15 +107,21 @@ The current code provides only the in-memory and SQLite backends. bbolt, pebble, The solution is for the factory to take one more parameter: ```go -gor.Register[Account](rt, func(b *gor.Binder) Account { - return &account{balance: gor.NewState[int64](b, "balance")} +gor.Register[Account](rt, func(g *gor.GrainContext) Account { + return &account{balance: gor.NewState[int64](g, "balance")} }) ``` -`NewState` registers the State value on the Binder. When the Runtime +`NewState` registers the State value on the Grain Context. When the Runtime activates a Grain, it reads the store once and distributes named values to the State cells. +State declarations are open only during the factory call. The Runtime freezes +the declaration set before it reads the record. A later `NewState` call is a +programming error and panics with `gor: NewState called after Grain factory`. +The panic faults that Activation. This rule prevents `OnActivate` from adding +an unloaded State value. + Each State value has four operations: ```go @@ -102,46 +141,55 @@ conflict or unknown write result follows the same deactivation rule as GrainId comes from here too: ```go -func Self(b *Binder) GrainId +func Self(g *GrainContext) GrainId ``` -The binder already holds the GrainId — `State` needs it to locate the storage row, and `Reminder` needs it to write the table. `Self` just hands it to the user; it adds nothing. +The Grain Context already holds the GrainId. `State` needs it to locate the +storage row, and `Reminder` needs it to write the table. `Self` returns that +value. -**It must be a value on the Binder, not Grain state.** Storing its own key in state rolls back together with write conflicts, and in a double-activation window an activation would read a stale value — the Grain would mistake its own GrainId. +**It must be a value on Grain Context, not State.** Storing its own key in State +can give an Activation a stale GrainId. **Reflection-based struct field scanning and backfilling is rejected.** It would keep the factory as `func() Account` and save one line. It would also need `unsafe` for unexported fields and would hide how State becomes live. -### Two more things on the Binder +### Two more things on Grain Context Time: ```go -func Now(b *Binder) time.Time +func Now(g *GrainContext) time.Time ``` -The binder already holds the injected `Clock` — `Reminder` needs it to compute due times. `Now` just hands it out. If it were not handed out, users would write `time.Now()` in their Grains — and that is the very first thing this project forbids. +The Grain Context holds the injected `Clock`. `Now` returns its current value. +Grain code must not call `time.Now()`. Calling others: ```go -type Scope interface{ /* sealed: only *Runtime and *Binder implement it */ } +type Scope interface{ /* sealed: only *Runtime and *GrainContext implement it */ } func Ref[T any](scope Scope, key string) T ``` -`Ref` originally took only a `*Runtime`, so a Grain wanting to call another Grain would need the factory closure to capture the runtime object as well, changing the factory signature from `func(b *Binder) T` to `func(rt *Runtime, b *Binder) T`. Cross-Grain calls are the most common thing virtual Grains do; it should not be the heaviest parameter on the signature. +`Ref` accepts Runtime or Grain Context scope. A Grain does not need to capture +the Runtime to call another Grain. -So the binder holds the runtime, `Ref` takes a sealed interface, and both sides share the name. Sealing — an unexported method in the interface — stops users from implementing it: it is not an extension point, just two shapes of "a place that can resolve Grains". +The Grain Context holds the Runtime. `Ref` takes a sealed interface. The seal +prevents Application implementations because Scope is not an extension point. -**No third thing on the Binder.** It is the seam between Grain and runtime; anything stuffed into the seam must first answer "what happens if it is not stuffed". +Grain Context also owns Grain Timers, Reminders, and lifecycle controls. These +capabilities all belong to the current Activation. ## runtime does not import store -`runtime` and `store` are siblings in the architecture diagram; neither imports the other. But the `Binder` must reach both sides: only `runtime` knows the GrainId at activation time, and `Store` is injected when `gor` assembles the configuration. +The internal execution runtime and `store` are siblings in the architecture. +Neither imports the other. Grain Context connects both sides in the root +package. -The solution: the factory is called by `runtime` and provided by `gor`: +The solution: the factory is called by `internal/runtime` and provided by `gor`: ```go type Registration struct { @@ -150,26 +198,30 @@ type Registration struct { } ``` -`runtime` hands out the GrainId and gets back an opaque instance. It does not know the instance carries a Binder, nor that construction read storage once. `gor` is the only package that sees both `runtime` and `store`; the conversion between the two GrainId types happens in exactly this one place. +The execution runtime gives the GrainId to the root package and gets an opaque +Activation value. Only the root package sees both execution and Store types. The factory can now return an error — reading storage during activation can fail. This merges with factory panic into the same path: the activation is not established, and the error returns to the caller. -## How conflicts get back to the runtime +## How State write failures get back to the Runtime -When `Set()` or `Clear()` hits `ErrConflict`, the Activation must be -deactivated. `runtime` does not know `ErrConflict` and should not. +Each failed Store write ends the Activation. This rule includes a Conflict, +cancellation, timeout, and an Unknown Result. The `internal/runtime` package does not +know these Store results and must not depend on them. -The error cannot be relied on to propagate up. User methods can perfectly well swallow it and return nil, by which time the cached ETag is stale. Deactivation must be independent of how user code handles the error. +The Application can ignore the error and return `nil`. The cached ETag is then +not safe to use. Deactivation must not depend on how the Application handles +the error. -So `Set()` and `Clear()` immediately set a flag on the Binder. `gor`'s -dispatch wrapper checks it after every Call and wraps the result into a shape -`runtime` recognizes: +Thus, `Set()` and `Clear()` record the failure on Grain Context. The `gor` +dispatch wrapper checks it after every Call. It then returns a result that +`internal/runtime` recognizes: ```go type Discard struct{ Err error } ``` -When `runtime` sees `Discard`, it deactivates the Activation and returns +When `internal/runtime` sees `Discard`, it deactivates the Activation and returns `Err` unchanged to the caller. It only knows that the Grain is no longer safe to use. This shares the path with post-panic deactivation. @@ -194,7 +246,8 @@ from multiple values. The outer container and the values must use one encoding. A second codec would add a second storage model without a current product need. -Encoding for transport between nodes is a separate matter; see [architecture.md](architecture.md). +Encoding for transport between Silos is separate. See +[architecture.md](architecture.md). No version-tolerant encoding (automatic compatibility when fields are added or removed). Orleans paid 30k lines for it. gor's stance: state structure evolution is handled by users at the application layer — read the old format, write the new one; the runtime does not intervene. @@ -202,14 +255,56 @@ No version-tolerant encoding (automatic compatibility when fields are added or r When a `Set()` or `Clear()` write returns an error, two things happen: -1. **The in-memory value stays unchanged.** Without confirmation of the write, a cell must not pretend the write succeeded — otherwise memory and storage diverge from then on, and the user reads a value that may not exist in storage at all. -2. **Deactivate the activation.** The next call reads from the store again, gets a fresh ETag, and continues. - -This is the same line of thinking as panic handling ([runtime.md](runtime.md)): once the instance's in-memory state is untrustworthy, don't keep using it — rebuilding is cheaper than repairing. - -**Conflicts and other write errors are not distinguished.** Hitting `ErrConflict` means the ETag is definitely stale; other errors mean the outcome of this write is unknown — storage may have changed, only the reply did not arrive. In both cases the ETag held by this activation is untrustworthy, so the handling should be the same. Separate handling adds a rule and buys nothing but "sometimes it can still muddle through a while longer". - -The error is returned to the caller as usual; the runtime does not retry for him — only he knows whether retrying is safe. +1. **The Runtime does not commit the candidate value in memory.** The State + value, presence mark, and ETag stay at their last confirmed values. This + rule cannot undo a change that the Application already made through a map, + slice, pointer, or interface returned by `Get`. The Runtime discards the + Activation so that a later Call cannot use that unconfirmed alias. +2. **End the Activation.** The next Call reads the Store again and gets a new + ETag. + +This is also the panic rule in [runtime.md](runtime.md). The Runtime must not +use an Activation when it cannot trust its in-memory State. + +**Conflicts and other write errors use the same recovery rule.** A Conflict +means that the ETag is old. Another error can have an Unknown Result. In both +cases, the Runtime cannot trust the Activation ETag. + +Cancellation and deadline errors follow the same rule. The Runtime cannot know +whether the Store observed the write before it observed cancellation. Every +attempted Write error marks the Activation for discard. + +Encoding errors happen before Store I/O. They leave memory unchanged but do +not discard the Activation. + +The rule applies in every Activation turn. A failed write in the factory or +`OnActivate` prevents that Activation from becoming active. A failed write in +a Call, Reminder, or Grain Timer faults the Activation after the turn. A failed +write in `OnDeactivate` is reported through the deactivation error event; the +Activation is already ending. + +This rule is stricter than the Orleans rule. Orleans automatically ends an +Activation only after an optimistic concurrency exception leaves it. gor also +ends it after a Store error, cancellation, timeout, or lost reply. These +results do not prove which record and ETag the Store holds. + +Read, decode, Write, and Clear failures keep their stable persistence code. +Every diagnostic names the operation and GrainId. A named State decode, Write, +or Clear failure also names the State. A whole-record Read or document decode +has no single State name and says `State record`. Error text is for diagnosis; +callers branch on the stable code. + +The Runtime returns the error to the caller. The Runtime does not retry the +Business Action. The Application decides if a Safe Repeat is valid. + +The release lifecycle proof must use injected Store failures and no wall +clock. It must cover a mutable value returned by `Get`. The test must show that +the Runtime cannot roll back the alias after a failed `Set`. A new Activation +must read the Confirmed State from the Store. + +The proof must also fail a State write during a Reminder turn. The failed turn +must end only its Activation. A later Call must create a new Activation and +read the Confirmed State. An Activation for another GrainId must stay active. ## When to write @@ -259,7 +354,13 @@ db, err := store.OpenSQLite("data/gor.db", ) ``` -`Durability` and its two values (`DurabilityFull`, `DurabilityRelaxed`) live in the `store` package: every backend that implements `Store` shares one type, and the dependency direction (`gor` imports `store`, never the reverse) is what forces it there. Both SQLite constructors take the option — `OpenSQLite` and `OpenSQLiteWithClock` — so a cluster node, which opens with a clock for membership snapshots, sets the state tier the same way a single-node program does. The option does not add a second path to the API: the user names one database, and the store derives the location of any additional database file from it (see "What this means for the SQLite backend"). The in-memory store has no durability tier — it holds nothing across a crash by design, so the option does not apply to it; the tiers are a property of the on-disk backends only. The runtime, the Grain, and the write path are unaware of the tier. `Store.Write`'s contract — write the bytes, return the new ETag — is identical at both tiers; only how hard the backend pushes the bytes to storage differs. +`Durability` and its values are in the `store` package. Each `Store` backend +uses the same type. Both SQLite constructors accept the option. Thus, a Silo +can use the same State tier with or without Cluster mode. The Application gives +one database path. The store derives other file paths from it. The in-memory +store has no Durability tier because it keeps no data after a crash. The Grain +Runtime and the Grain do not know the tier. `Store.Write` has the same contract +for both tiers. Only the storage flush behavior is different. ### Scope: the state table only @@ -282,11 +383,33 @@ Putting the tables in separate files does not break atomicity that existed befor On disk the store may hold more than one database file, each carrying its own `-wal`/`-shm` sidecars. Backups and direct `sqlite3` inspection must cover every file the store creates, not just the path the user named — copying only the main file leaves the write-ahead log behind, and the recovered state is stale or torn. +SQLite read pools must set both maximum open and maximum idle connections. The +0.1.0 default is 16 for each read pool. Write pools keep one connection. These +limits bound file descriptors and preserve the single-writer rule. + +### Cold backup and restore + +The documented backup path is cold: + +1. Stop Call admission with `Shutdown(ctx)`. +2. Close the SQLite Store. +3. Copy the coordination and State database files as one backup set. +4. Keep any WAL file that still exists with its database file. + +Restore replaces the complete closed backup set. The Application then opens +the Store and passes it to the Runtime. Runtime Start calls `Check`, which runs +`PRAGMA integrity_check` for the coordination and State databases. A backup +test must write State and a Reminder, close, copy both database files and each +existing WAL file, restore the set, run `Check`, and read both values. + ### Old databases An earlier 0.0.x database keeps the state, reminder, and membership tables in one database file. When the state tier calls for it, the state rows move to their own database, and such a database is read into the new layout on first open. The constraint is fixed, not optional: every confirmed state row must come through the move — nothing lost, and the store stays readable. This is not a new promise; it is the 0.0.x promise that a later 0.0.x reads state an earlier 0.0.x wrote, applied to the layout change ([compatibility.md](../docs/compatibility.md)). -The migration is the store's job, done once on first open of an old database, not the user's; the user does not hand-move rows or convert formats. An interrupted migration must be redoable or resumable on the next open: the old database is not destroyed until the new one is complete, so a crash mid-move leaves the store recoverable. Whether the implementation keeps the single-file layout when the tier is Full and splits only at Relaxed, or splits uniformly regardless of tier, is the implementer's choice — but whichever it is, the constraint above holds: confirmed state survives the upgrade, and the user passes one path either way. +The Store does the migration on the first open. The Application does not move +rows or convert files. The Store checks the copied State database before it +deletes the old State table. An interrupted migration must be safe to run +again. Confirmed State must survive the migration. ### Relationship to ETag and optimistic concurrency @@ -327,11 +450,17 @@ details are in [timers.md](timers.md). ### SQLite Reminder migration -When SQLite opens an existing coordination database, it must add the -`first_tick_time INTEGER` column to the existing coordination table during -open. The migration must also handle old rows that have no value in this -column. For each such row, it must use the row's current `due_at` as -`first_tick_time`. +When SQLite opens an existing coordination database, it must add a missing +`first_tick_time INTEGER` column to the coordination table. The migration must +also handle old rows that have no first tick value. For each such row, it must +use the row's current `due_at` as `first_tick_time`. + +SQLite also stores one global Reminder version. The migration sets this value +to at least the largest ETag in the Reminder table. This prevents ETag reuse +after Delete and restart. + +The migration replaces an invalid or partial due index with one full index +that starts with `due_at`. This fallback keeps old rows readable, but it cannot recover the original first time. The original first time is not reconstructible from an old row. The @@ -339,3 +468,9 @@ migration must document and preserve that limit. The migration must be covered by a storage test that opens an old database, checks the fallback value, and checks that a new open keeps the value. + +## 0.1.0 Status + +Batch 5 implements the 0.1.0 State and SQLite requirements in this document. +It includes State failure recovery, stable diagnostics, bounded connection +pools, integrity checks, Store contract tests, and cold restore proof. diff --git a/design/release-0.1.0.md b/design/release-0.1.0.md index d8935d0..f3a71ad 100644 --- a/design/release-0.1.0.md +++ b/design/release-0.1.0.md @@ -1,194 +1,578 @@ # 0.1.0 Delivery Design -This document lists the work for the [0.1.0 product -contract](../docs/release-0.1.0.md). The other design documents define local -APIs and algorithms. This document defines the work order and the proof that -the parts work together. +This document defines the work order for the [0.1.0 product +contract](../docs/release-0.1.0.md). Detailed design documents own each local +API and algorithm. [ROADMAP.md](../ROADMAP.md) owns progress status. ## Design rules -The main product is one process with one local store. +The main product is one process with one local Store. -All input and output uses an interface. All time comes from the injected -`Clock`. A channel, not a mutex, waits for another call. Each component is an -explicit state machine. These rules make failure tests repeatable. +All input and output must use an interface. All time must come from the +injected `Clock`. Components with concurrent behavior must use explicit state +machines. Cross-call waits must use channels. -The application owns its local business records. The Runtime owns Grain State -and Reminder records. The Runtime does not combine application records with -these runtime records. +The Application owns its business records. The Runtime owns Grain State and +Reminder records. The Runtime must not combine these records. Future cluster work needs clear GrainId, Grain Reference, Call, State, and -encoding boundaries. Cluster health and cluster operations are not 0.1.0 -gates. +encoding boundaries. Cluster operation is not a 0.1.0 gate. -## Work packages +## Acceptance matrix + +This table maps each release promise to one detailed design and one proof. + +| Release promise | Detailed design | Required proof | +| --- | --- | --- | +| Complete setup before service | [runtime.md](runtime.md), [persistence.md](persistence.md) | Invalid setup starts no goroutine and accepts no Call. | +| Stable Grain identity | [codegen.md](codegen.md) | Generated GrainType survives restart and duplicate names fail at Start. | +| Typed serialized Calls | [runtime.md](runtime.md), [scheduling.md](scheduling.md) | Same-Grain Calls serialize and canceled queued Calls do not enter methods. | +| Request Context | [request-context.md](request-context.md) | Local and encoded Calls use one path and activation gets the first context. | +| Activation lifecycle | [runtime.md](runtime.md) | Start, panic, Deactivate on Idle, eviction, and stop have deterministic tests. | +| Confirmed State | [persistence.md](persistence.md) | Restart, zero value, Clear, Conflict, cancel, and Unknown Result are tested. | +| Grain Timers | [timers.md](timers.md) | Timer turns serialize, never overlap, can change or stop, and end on deactivation. | +| Reminders | [timers.md](timers.md) | Restart, claim, backlog, missed delivery, Terminal Result for an Invalid Reminder, dynamic name, and actual tick time are tested. | +| Background errors | [timers.md](timers.md), [runtime.md](runtime.md) | Every source is visible and observer panic cannot stop cleanup. | +| Observability | [observability.md](observability.md) | Snapshots and Call completion events match real Runtime state. | +| Public API usability | [conformance-example.md](conformance-example.md) | A clean module builds and runs the complete single-Silo flow. | +| Release reliability | [testing.md](testing.md), [release.md](release.md) | Supported systems pass the bounded release gate with nonzero tests. | -### 1. Freeze the contract +Behavior outside this matrix is not a 0.1.0 release promise. -Write one table for these rules: +## Key decisions -- GrainId, GrainType, and GrainKey names; -- Call order, overload, timeout, cancel, panic, and the non-reentrant rule; -- start and deactivation transitions; -- State read, write, Exists, Clear, conflict, unknown store result, and restart; -- Reminder setting, claim, cancel, tick status, and method failure; -- background error sources and call observations; -- application data and safe repeat rules. +### Runtime setup has a Start gate -The table must link to the detailed documents. It must not copy API text. -Behavior not in the table is not a release promise. +`New` assembles a Runtime. It does not start a poller, eviction loop, +transport, or cluster component. The Application then installs generated code +and registers Grain factories. `Start(ctx)` validates and freezes the setup. +Only Start can start background work. -### 2. Harden the Grain runtime +Automatic start in `New` was considered. It is rejected because Reminders can +be claimed before their Grain type is installed. It also makes complete setup +validation impossible. -Use one call path for local Calls and future remote Calls. The root Runtime -must decide if it still accepts a Call before the Call reaches a Grain. +A separate builder API was considered. It is rejected because `New`, +generated `Install`, `Register`, and `Start` already form one small setup path. -For each GrainId, start, queue, method run, and leave must follow one state -machine. +### Generated code owns GrainType -Tests must cover: +Generated code passes one stable GrainType to `InstallType`. Runtime maps use +the Go type only as a local lookup key. Stored records and Call envelopes use +the generated GrainType. -- two callers that start one key at the same time; -- a method panic with calls in the queue; -- cancel while a call waits and while a method runs; -- start failure and state write failure; -- normal close and forced stop; -- leave reason and background error reporting; -- a type name that does not depend on an incidental Go type string. +The marker has one grammar: -The runtime does not retry a business method. The caller must decide if a -retry is safe. +```go +//gor:grain +type Account interface { ... } -### 3. Harden state and Reminders +//gor:grain billing.account +type BillingAccount interface { ... } +``` -Keep State and Reminders as separate interfaces. Keep the version check on -State writes. A Reminder claim must select one delivery attempt. `Exists` and -`Clear` must distinguish absent State from a present zero value. +The first form uses `.`. This is the current +0.0.x stored name, so old State and Reminder rows remain readable. The second +form gives the Grain an explicit Application name. -Tests and simulation must cover: +The generator must reject an empty, malformed, or repeated marker. Start must +reject two installed Go types with the same GrainType. `GrainTypeOf[T](scope)` +returns the installed value. The old reflection-based `TypeName[T]` path is +removed. -- due work found after a process restart; -- two pollers that claim one record at the same time; -- a failed claim; -- process failure after claim and before method entry; -- a failed Reminder method; -- cancel and reset; -- a periodic Reminder after downtime without replaying every missed time. -- State that is absent, present with a zero value, and cleared; -- first tick time, period, and current tick time for a periodic Reminder. +Using the import path in the default name was considered. It prevents package +name collisions, but it changes existing stored names and changes again when +an Application moves a package. It is rejected. -The result is at-most-once delivery. Recovery is an application pattern: -save a pending action, wake a fixed Grain, and make the handler safe to run -more than once. The runtime gives the Reminder and call paths. It does not own -the business record. +Keeping `fmt.Sprintf("%T")` was considered. It can give two types the same +name and does not make the persisted contract explicit. It is rejected. -### 4. Complete call data and encoding +### Grain Context replaces Binder -Request Context is data that travels with a Call, such as a trace ID. Add the -smallest call path that supports it: +The public factory parameter is `*gor.GrainContext`. The old `Binder` name is +removed. A compatibility alias is not added because 0.1.0 is the first public +contract and one term must name one concept. -- the caller can add Request Context; -- the method can read incoming Request Context; -- local Calls and future remote Calls use the same call path; -- no shared Call Filter pipeline is part of 0.1.0; -- the call path cannot bypass call admission or change call order; -- stable error codes cross the call boundary; -- cancel rules stay clear after a method was sent. +The Grain Context is complete before the factory runs. State declarations are +open only during the factory call. The Runtime freezes them before it reads +State or runs `OnActivate`. A later `NewState` call panics with the stable +programming error defined in [persistence.md](persistence.md). -Keep JSON as the current encoding. Do not add an encoding plug-in or a -zero-downtime upgrade format without a real user need. Typed interfaces, -stable type names, and written compatibility rules define the application -contract. +### Grain Timer is an Activation turn -The Request Context API, lifetime, encoding, and failure rules are in -[request-context.md](request-context.md). +`RegisterGrainTimer` creates a timer for the current Activation. A tick enters +the same mailbox as a Call. It is local to that Activation and cannot move to +a new Activation. -### 5. Build the integration sample +The public timer handle supports `Change` and `Stop`. Stop is safe to repeat. +A zero period selects a one-shot timer. A positive period selects a repeating +timer. A negative due time or period is invalid. -Add a small example and tests that use only public APIs. It must contain: +The options contain `DueTime`, `Period`, and `KeepAlive`. Interleaving is not +an option in 0.1.0. A closure carries callback state, so the API does not need +a separate generic state parameter. -- a Grain with current State; -- a fixed-key coordinator or dispatcher; -- a saved pending action; -- application records in its own local rows; -- a periodic recovery Reminder; -- a handler that is safe to run more than once; -- a process stop between save and delivery; -- a repeated delivery that does not apply the business change twice. +A separate timer executor was considered. It is rejected because it would +bypass Grain serialization and lifecycle cleanup. -The example is not a second framework. It proves that the public Runtime -boundaries support a real durable application pattern with local application -data and safe repeat handling. The required Single Silo flow and its failure -evidence are specified in [conformance-example.md](conformance-example.md). +### One sink reports background failures -### 6. Run release checks +`OnError` receives failures that have no waiting caller. Its source is a +closed set. It covers Grain Timer callbacks, Reminder scan and delivery work, +and deactivation hooks. A lost Reminder claim Conflict is normal contention +and is not an error event. -The release candidate must pass: +The Runtime must catch a panic from `OnError`. It must preserve the first +error and finish cleanup. It must not call the same failing observer again for +the observer panic. -- `make test`; -- `make sim`; -- `make gen`; -- `make net`; -- `make lint`; -- `make ci`; -- race tests; -- a clean-module example build and run. +## Work packages -Storage tests that measure disk durability must use a real-disk path. The -simulation must report nonzero test cases and repeat the full seed set with -the same result. +Each package is one reviewed batch. A batch includes its tests and document +updates. Stop after each batch. Do not start the next batch before review. + +### Batch 1: Contract and delivery plan + +Update the product language, product contract, detailed design, acceptance +matrix, and ROADMAP status. -## Failure table +Proof: -| Failure | Runtime result | What the application must do | -| --- | --- | --- | -| Queue full | The call is rejected. The method does not start. | Apply back pressure or retry when safe. | -| Caller timeout or cancel | The caller stops waiting. The method may continue. | Use a safe repeat rule or a repair action before retry. | -| Method panic | The call fails. The Grain instance is removed. | Fix the method and decide if a retry is safe. | -| State conflict | The write fails. The old instance is not trusted. | Read again and use a clear retry rule. | -| Store result is unknown | The call fails. The write may or may not exist. | Check the stored result before a non-repeatable action. | -| Claim succeeds, then process stops | The Reminder may be missed. | Save a pending action and recover it safely. | -| Reminder method fails | The error goes to the background sink. No hidden retry starts. | Save retry state in the application when needed. | -| Normal close | New calls are rejected. Accepted calls follow close rules. | Stop new outside work and drain as needed. | -| Forced stop | Queued calls are rejected. A running method may finish later. | Recover work whose result is unknown. | - -The table must state unknown results. The runtime must not turn an unknown -result into a false success or an unsafe retry. - -## API usability gate - -The first example is the main usability test. A user must be able to: - -1. declare a typed Grain interface; -2. create named State in a factory; -3. get a Grain Reference by type and key; -4. call another Grain with the Binder; -5. add Request Context and read it in the called Grain; -6. set and clear a Reminder; -7. set an error sink and Call observation; -8. close and reopen the Runtime. - -Each step must have one clear public path. The example must not need cache -details, private store layout, cluster membership, or a second hidden retry -loop. The clean consumer build and two-process run are additional evidence -for the conformance example. - -## Work order - -Work lands in reviewable batches. Stop after each batch: - -1. contract and acceptance table; -2. Grain runtime edge rules; -3. state and Reminder failure rules; -4. call data and encoding boundaries; -5. integration sample and failure tests; -6. release docs and clean-install check. - -The first batch is documentation only. Code work starts after review of the -contract. +- all changed prose is simple English. +- links and terms are consistent. +- each implementation gap has one later batch. +- `make ci` passes on the documentation commit. + +### Batch 2: Runtime setup, GrainType, and Grain Context + +Implement the setup state machine and the public naming changes first. This +batch owns shared Runtime and generator contracts. + +Changes: + +- make `New` start no background work and add `Start(ctx)`. +- add `Check(ctx)` to the State Store and Reminder Store contracts. +- validate Store readiness, durations, mailbox capacity, and complete Grain + registration at Start. +- freeze install and registration after Start. +- make a durable Store explicit and keep the memory Store opt-in. +- generate the stable GrainType constant. Keep installation keyed by local + Go type and reject duplicate GrainType values at Start. +- replace Binder with Grain Context without an alias. +- add `Shutdown(ctx)`, `Stopping()`, and `Done()` with the stop meanings in + [runtime.md](runtime.md). +- update generated files, examples, comments, and public docs. + +Proof: + +- no background work runs before Start. +- a failed Start claims no Reminder and accepts no Call. +- a canceled Cluster start leaves its new membership row `dead`. +- an applied Cluster join write cannot leave a `joining` or `active` row after + the caller receives an error. +- abrupt shutdown interrupts a blocked Cluster leave write. +- old default GrainType rows remain readable. +- two same-name types from different packages fail with a clear error unless + one has an explicit GrainType. +- `NewState` after the factory returns fails with the specified programming + error. +- shutdown cleans a constructed, running, or start-failed Runtime. +- `Stopping` and `Done` close at their specified state transitions. +- focused startup and generator tests pass. +- `make ci` passes. + +### Batch 3: Activation, Call, and deactivation rules + +Implement one Activation state machine for start, active work, requested +deactivation, fault, and stop. + +Changes: + +- give activation work a Runtime-owned context. +- copy the starting Call's Request Context into activation. +- stop a canceled queued Call before method entry. +- add Deactivate on Idle and the `ApplicationRequested` reason. +- send not-started Calls to the next Activation after requested deactivation. +- fail queued Calls after panic or untrusted State. +- catch panics from factories, lifecycle hooks, and observers. + +Proof: + +- deterministic race tests cover cancel against dequeue and method entry. +- one caller can cancel while another still waits for shared activation. +- requested deactivation creates a new Activation for the next Call. +- lifecycle callback panic cannot stop the process or skip cleanup. +- [targeted mutations](../research/release-0.1.0-batch-3-mutations.md) that + remove each state transition fail their behavior tests. +- `make ci` passes. + +### Batch 4: Grain Timers + +Add the public Grain Timer API and Activation-local scheduler. + +Changes: + +- add register, change, stop, one-shot, repeat, and keep-alive behavior. +- deliver ticks through the Grain mailbox. +- stop all timers before deactivation completes. +- report callback errors through `OnError`. +- fault the Activation after a callback panic. + +Proof: + +- a callback never overlaps itself or a Call. +- period starts after callback completion. +- Change during a callback controls the next due time. +- Stop and deactivation prevent later ticks. +- a non-keep-alive timer does not prevent idle deactivation. +- fake-clock tests contain no sleeps. +- [targeted mutations](../research/release-0.1.0-batch-4-mutations.md) that + remove timer state and cleanup rules fail their behavior tests. +- `make ci` passes. + +### Batch 5: State failure hardening + +Make all State outcomes follow one rule. + +Changes: + +- freeze State declarations before load. +- do not commit a candidate value, presence mark, or ETag after a failed Write + or Clear. +- discard the Activation after Conflict, cancel, timeout, or Unknown Result. +- reject an Activation when its factory or activation hook ignores a failed + State write. +- report an ignored State failure from a deactivation hook. +- add GrainId and State name to stable persistence errors. +- set SQLite connection limits for each access mode. +- run SQLite integrity checks before Runtime Start. +- check a migrated State database before deleting the old State table. +- document State format changes and cold backup and restore. + +Proof: + +- canceled writes and writes with lost replies discard the Activation. +- a failed write cannot roll back a mutable value that the Application changed + through `Get`; the next Activation reads the Confirmed State. +- a State failure during a Reminder turn discards only the affected + Activation; a later Call creates a new Activation and reloads State. +- an ignored State failure in an activation hook prevents method entry. +- absent, zero, cleared, conflicted, and unknown values survive restart as + specified. +- a cold restore covers State, Reminders, and SQLite WAL files. +- integrity failures in either SQLite database fail the Store readiness check. +- a failed migration integrity check leaves the old State table intact. +- Store contract tests run against memory and SQLite. +- `make ci` passes. + +### Batch 6: Reminder hardening + +Make Reminder delivery bounded and observable. + +Changes: + +- validate name, method handle, due time, and period before Store I/O. +- acquire worker capacity before a claim. +- list due rows in bounded cursor pages. +- report the actual delivery start in `CurrentTickTime`. +- add Reminder name and TickStatus to background errors. +- report scan, decode, and claim failures through `OnError`. +- add the SQLite due-time index. + +Proof: + +- an unknown Grain type or method row is reported without a claim. +- a large due set keeps goroutine and memory use within fixed limits. +- two pollers deliver one claimed due time at most once. +- process failure after claim gives the documented missed-delivery result. +- scan failure and callback failure have different stable sources. +- [targeted mutations](../research/release-0.1.0-batch-6-mutations.md) that + remove Reminder limits and failure rules fail their behavior tests. +- `make ci` passes. + +### Batch 7: Generator and package boundary + +Make generated code and the public package surface release-ready. + +Changes: + +- reject generic Grain interfaces, variadic methods, unexported methods, and + unexported contract types. +- add a generated header and a generated-code version check. +- write generated files through a temporary file and atomic rename. +- add `gorgen -check` and a canonical `go:generate` command. +- move implementation-only mail, execution runtime, and Reminder poller code + under `internal/`. +- keep `Config` and `DeactivationReason` owned by the public `gor` package. +- make generated request, reply, and parameter names collision-safe. +- remove stale non-Orleans terms from public comments and examples. + +Proof: + +- every invalid input reports its source line. +- an interrupted generation leaves the prior file unchanged. +- generated diff checks fail when source and output differ. +- a clean external module generates, imports only public packages, and builds. +- [targeted mutations](../research/release-0.1.0-batch-7-mutations.md) that + remove generator and public-package guards fail their behavior tests. +- `make ci` passes. + +### Batch 8: Limits, portability, and release proof + +Complete the release evidence after all behavior batches pass. + +#### Batch 8a: Activation admission + +Implement and verify the Silo Activation limit before the other Batch 8 work. +The limit must protect the execution Runtime before a new Grain lane and +mailbox are allocated. The reservation must cover start, active use, and the +full deactivation lifecycle. + +Proof: + +- a full limit rejects a new Grain before its factory and before a new lane; +- an active Grain still accepts Calls when the limit is full; +- a failed start releases one reservation exactly once; +- a new Grain remains rejected until the old deactivation finishes; +- the public error uses `gor.overloaded` for both mailbox and Activation + admission; +- [targeted mutations](../research/release-0.1.0-batch-8a-mutations.md) that + remove these guards fail their behavior tests; +- `make ci` passes. + +#### Batch 8b: Bounded resource proof + +Prove the existing resource limits without a new product API or a general +soak-test framework. Use the completion signals and state that the Runtime and +Reminder poller already own. Do not use a process-wide goroutine count as leak +proof because unrelated test work can change that count. + +The Activation churn test must use seed `1945`, 16 Grain keys, and eight cycles. +Each Activation must own one far-future Grain Timer. Each cycle must fill the +configured Activation limit and then use a fake clock to evict all idle +Activations. The test must prove that all Activation mailboxes and Grain Timers +stop. The Runtime must have no Activation, lane, reservation, or Grain Timer +after each cycle. Runtime close must also complete. + +The Reminder backlog test must use seed `1945`, 257 due Reminders, a page limit +of 17, and a worker limit of five. It must prove that every Reminder is +delivered. It must also prove that the poller does not exceed its page limit or +worker limit. Poller close must stop all workers and release all worker slots. + +Test-only file-system probes must compile on all supported systems. A common +test file must call a build-tagged helper and must not import a Linux-only +`syscall` type. + +Proof: + +- the focused resource tests use no sleep, retry, or timing allowance; +- [targeted mutations](../research/release-0.1.0-batch-8b-mutations.md) fail + when the worker, page, or cleanup checks are removed; +- root and Store benchmark packages compile for each supported system; +- `make ci` passes. + +#### Batch 8c: Fail-closed release gates + +Add release checks without a new product API. One internal test command must +run `go test -json`, keep the test output and exit result, count started tests, +and fail when the count is zero. All Make test targets must use this command. +The `ci`, `external`, and `external-tagged` entry points must not accept a +caller-provided Make variable that replaces this command. A release-tagged +contract test must compare the complete dry-run plan for each entry point with +and without a caller override. Both plans must be equal. Each plan must contain +all required commands in the specified order. A parallel Make option must not +change this order. + +Add these fuzz targets: + +- `FuzzReadFrame` for Transport frame decode; +- `FuzzDecodeRequestContext` for Request Context decode; +- `FuzzParseGrainMarker` for the `gor:grain` marker parser. + +Each fuzz target must contain valid and invalid seed inputs. A successful decode +must satisfy the same size and round-trip rules as normal tests. `make fuzz` +must run each target separately for 10 seconds by default. Go must save a +failure in the normal fuzz corpus. + +Do not wrap a fuzz package in a process-wide goroutine count. The Go fuzz +runner owns background goroutines, and their lifetime is not a package +resource contract. A component must prove cleanup with its own completion +channels and state. + +Add `make generated-check`, `make tidy-check`, `make resource`, and `make fuzz`. +The generated check must use the generator's non-writing `-check` mode for each +generated package. The tidy check must use `go mod tidy -diff` and must not +change files. `make ci` must run all these checks. It remains the only automated +release gate. + +The Linux matrix must run `make ci` with Go 1.25 and stable Go on amd64. The +macOS job must run on `macos-15` and must verify arm64. The Windows job must run +on `windows-2025` and must verify amd64. Both platform jobs must run the +default, simulation, generator, and network test sets. The final `ci` job must +fail if any required job fails. + +Add Staticcheck `v0.7.0` and govulncheck `v1.6.0` as Go tools in `go.mod`. Use +`go tool` for both commands. A local cross-compile does not prove that a hosted +platform job passed. + +Proof: + +- the internal test command passes a selected test and rejects a zero-test + selection; +- a caller-provided Make variable cannot replace the internal test command in + `ci`, `external`, or `external-tagged`; +- a mutation that removes the zero-test rejection fails its behavior test; +- each fuzz target runs its seed corpus and one focused fuzz session; +- `make ci` passes on Linux amd64; +- all hosted platform jobs pass on the same commit. + +#### Batch 8d: External restart and public documents + +**Status: Complete.** + +Add `make external` as a separate release proof. It must not run as part of +`make ci`. The proof can need network access and empty Go caches. + +The proof must create a temporary consumer module outside this repository. It +must set `GOWORK=off`. It must also use empty `GOMODCACHE` and `GOCACHE` +directories. Before a tag exists, it must require `v0.0.0` and replace that +module with the candidate repository. + +The consumer module must build the committed conformance command. The proof +must run that binary in separate operating system processes. Each process must +use the same SQLite Runtime and Application database paths. + +The proof must include these cases: + +- `prepare` exits, then `recover` applies the pending action; +- a successful Reminder Claim blocks at a test barrier; +- the parent stops that process with `Process.Kill`; +- a new `recover` process applies the pending action once; +- a wrong receipt expectation makes `recover` fail; +- `recover` against new databases fails with a nonzero exit code. + +The successful cases must verify the requested Grain State and the full +Application receipt. They must also verify that no pending action remains. +The process driver must not use a shell, a fixed sleep, or a system-specific +signal. + +Clean the public documents in the same batch. Add a tested Quick Start. Its +release test must build the command, start a separate process, and verify the +documented HTTP flow. Remove stale findings and product comparisons. Mark +Cluster material as a preview. Use the terms in +[CONTEXT.md](../CONTEXT.md). Changed prose must follow +[writing-style.md](../docs/writing-style.md). + +Proof: + +- `make external` reports a nonzero test count and passes; +- [targeted mutations](../research/release-0.1.0-batch-8d-mutations.md) + fail when they skip the second process, `Process.Kill`, or receipt comparison; +- the Quick Start uses a committed command that the repository tests; +- `make ci` passes after the document and proof changes. + +#### Batch 8e: Final candidate proof + +**Status: Pending.** + +Before the final proof, complete the release audit for the upgrade from +`v0.0.5`. That version stores Reminder identity in the `entity_type` and +`entity_key` columns. The 0.1.0 Store must rename those columns to `grain_type` +and `grain_key` in one transaction. Coordination schema bootstrap must use the +same transaction. It must reject an incomplete or mixed pair of old and new +columns without a partial bootstrap. It must then add `first_tick_time`, create +the current due index, and initialize the schedule version from the largest +current ETag. + +The focused migration fixture must match the exact `v0.0.5` schema. The +external proof must also build a writer from exact `v0.0.5`, without a +replacement. The writer must use the public Shadow Application and its old +generated package. It must write State and a far-future Reminder to SQLite, +then close normally. A candidate process must open the same database, read the +old State, update it, and restart successfully. Tagged mode must repeat this +path with exact `v0.1.0` as the new process. The candidate must bind an +OS-selected port. The process driver must read the actual address from the +readiness event. It must not use a fixed sleep. + +The generated call artifacts are intentionally incompatible. The user must +update source for the 0.1.0 public API and run the generator again. The release +note must state this action. Old generated files are not upgrade input for the +new Runtime. + +Add `make external-tagged` before the release. This command is fixed to +`v0.1.0`. It must not accept a different version or add a local replacement. +It must use empty Go caches and `GOWORK=off`. It must resolve the conformance +package from `v0.1.0`, then read the resolved module data with `go list -m +-json`. It must fail unless the version is exactly `v0.1.0`, `Replace` is +absent, and the module directory is in the empty module cache. It must ignore +local module proxy and private-module settings. It must use public Go module +proxies or direct source with the public Go checksum database. + +The configuration test must prove that tagged mode writes no replacement. +Before the tag exists, a failed `make external-tagged` run is diagnostic +only. It must show an attempt to resolve exact version `v0.1.0`. A network or +proxy error is not release proof. The final tagged-module proof is a successful +run after the tag exists. + +The candidate is the final commit on `master`. A pull request merge preview is +not candidate proof. Run every candidate release command and the hosted Linux, +macOS, and Windows jobs on the final `master` commit. If a squash, rebase, or +merge changes the commit ID, run the commands again. Create `v0.1.0` on this +exact commit. After the tag exists, run `make external-tagged` from the tagged +commit. Publish the GitHub Release only after this proof passes. + +Proof: + +- all candidate commands in [release.md](release.md) pass on the candidate + commit; +- all required hosted jobs pass on the candidate commit; +- `make external-tagged` reports a nonzero test count and the exact `v0.1.0` + module passes the external restart and `v0.0.5` upgrade proofs; +- the worktree is clean and the release evidence names the full commit ID. + +## Batch dependency order + +```text +1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7 -> 8 +``` + +This order is deliberate. Batch 2 changes shared public contracts. Later +batches build only on reviewed contracts. Batch 8 measures the final code, +not an earlier commit. + +## Handoff evidence + +Each batch handoff must include: + +- the worktree path, branch, and full commit ID. +- all changed paths. +- focused test commands and results. +- `make ci` with its exit code. +- mutation results when the batch changes a critical invariant. +- the clean or dirty worktree status. +- all known gaps and environment limits. + +A plan, a build, or a test with zero cases is not implementation proof. ## Gap -The current design documents cover most single-node parts. This document adds -one failure table, one integration sample, and proof that the parts work -together after restart, failure, and a safe repeat. +Batch 1 defines the release contract. Batch 2 implements explicit Runtime +startup, stable GrainType, Grain Context, and context-aware shutdown. Batch 3 +implements Activation lifecycle, queued cancellation, Deactivate on Idle, and +callback containment. Batch 4 implements the public Grain Timer API and the +Activation-local scheduler. Batch 5 implements State failure recovery and +SQLite recovery checks. Batch 6 bounds Reminder work and reports its failures. +Batch 7 hardens generation and moves implementation code under `internal/`. +Batch 8a adds Activation admission. Batch 8b adds bounded resource proof and +portable benchmark probes. Batch 8c implements fail-closed local release +checks, fixed tools, fuzz checks, and the hosted platform workflow. The hosted +workflow has not run on this change. Batch 8d implements the external restart +proof and public document cleanup. A release audit follow-up passes dynamic +Reminder names to typed methods. A second follow-up prevents a caller-provided +Make variable from replacing the internal test command. A third follow-up +restores Grain State before it commits an Application receipt. An Invalid +Reminder now gets one Terminal Result. A fifth release audit follow-up proves +the mutable State alias and Reminder-turn State failure lifecycle. Batch 8e +owns hosted proof, exact tagged module proof, and the final same-commit release +proof. A sixth release audit follow-up migrates the exact `v0.0.5` Reminder +schema and adds the external upgrade proof. diff --git a/design/release.md b/design/release.md index cc9abbd..b525722 100644 --- a/design/release.md +++ b/design/release.md @@ -87,9 +87,13 @@ gor is in the 0.0.x band. A 0.0.x tag is publicly visible — the repository is The 0.0.x release-note question has a zero-maintenance answer: none is written. The `release-note` blocks in merged PRs still accumulate as raw material; their only consumer is the maintainer writing the first announced release's note. Nothing is assembled, published, or kept in sync per 0.0.x tag. -The pre-announcement readiness work is complete. Every [ROADMAP.md](../ROADMAP.md) checklist item is done — English documentation, public API doc comments, the error and cancellation contract, the root runtime shutdown contract, deactivation reasons and the background error sink, the example application, observability, and the performance baseline — and `make ci` passes. The 0.1.0 contract adds a release-specific composition gate: the subsystem promises must be verified together under restart, failure, and duplicate delivery, and a public-API conformance application must demonstrate the integration boundary. Those requirements are not retroactively claimed by the old checklist. +The earlier functional readiness work is complete. It includes public API +comments, Runtime shutdown, errors, lifecycle, the example, observability, and +benchmarks. The 0.1.0 contract adds composition and release evidence gates. +Public document cleanup is complete. Hosted jobs and the final tagged-module +proof remain open. See [ROADMAP.md](../ROADMAP.md). -Multi-node is still a preview capability and partitions can misjudge healthy nodes, so no release — 0.0.x or announced — should be treated as settled. +Cluster support remains a preview. A partition can misjudge healthy Silos. ## The bar for v1.0.0 @@ -109,12 +113,35 @@ For a 0.0.x tag, only the verification core runs: `make ci` (step 3), the instal 1. Hand-read the merged PRs' `release-note` blocks, list what this release includes, and choose the version number per the previous section. Breaking changes go only into v0 minors or new major versions. 2. Update the affected product promises, designs, and the roadmap; keep or add a "Gap" section where a limitation has not gone away. -3. Run `make ci` on the candidate commit. For the first announced release, a major version, artifact changes, or persistence-related changes, also run an install-and-upgrade check in a clean user project. -4. Inspect the to-be-released commit, version tag, and working tree. Only verified content enters the release. -5. Hand-write the GitHub Release note from these blocks as raw material, then create the version tag and the Release. The library produces no standalone service binary, so no fake download packages. -6. After the release, install the exact version in a clean user project and run one minimal call. On failure, retract or mark that Release first; do not ship a patch to cover up an uninstallable version. - -Steps 1, 2, 4, and 5 need maintainer judgment and stay manual. Step 3 already has `make ci`, the only release gate worth automating permanently. Step 6 is a low-frequency external availability check; scripts, workflows, or auto-created Releases cost more maintenance than they earn. +3. Integrate the release content into `master`. The candidate commit is the + final `master` commit, not a pull request head or merge preview. Run `make + ci`, `make external`, and all required hosted jobs on this exact commit. + A squash, rebase, or merge that changes the commit ID requires a new run. + Artifact and persistence changes also need a clean install and upgrade + check. The upgrade check must start from the exact latest public tag. A + process built from that tag must write the input data. A schema fixture is + useful focused proof, but it does not replace this process proof. +4. Inspect the full candidate commit ID, the planned version tag, and the clean + working tree. Only verified content enters the tag. +5. Hand-write the GitHub Release note from the raw material. Create and push the + annotated version tag on the exact candidate commit. Do not publish the + GitHub Release yet. +6. Run `make external-tagged` from the tagged commit. This command is fixed to + the exact release version and has no local replacement. Publish the GitHub + Release only after this proof passes. If it fails, do not move or reuse the + tag. Fix the problem in a new version and retract the failed version. The + library produces no standalone service binary, so do not add download + packages. + +Steps 1, 2, 4, and 5 need maintainer judgment and stay manual. `make ci` is the +permanent automated gate. `make external` proves the candidate source with +empty caches and process restarts. `make external-tagged` repeats the restart +proof with the exact published module and no local replacement. The version tag +makes the Go module public. The GitHub Release is the announcement and follows +the tagged-module proof. + +The three release entry points must use the repository `testcheck` command. +A caller-provided Make variable must not replace this command. ## Gap diff --git a/design/request-context.md b/design/request-context.md index ada2341..4b3f0ff 100644 --- a/design/request-context.md +++ b/design/request-context.md @@ -65,9 +65,10 @@ The design considered these shapes: | --- | --- | --- | --- | | Helper functions on `context.Context` | Fits the existing typed method and `Invoker` signatures. Parent contexts stay isolated. Nested Calls can pass the same context. | Callers must pass the context they want to use. | Chosen. | | A mutable package Request Context bag with `Set`, `Get`, and `Clear` | Short calls after setup. | Go has no safe task-local mutable bag. Concurrent Calls could share or overwrite values. Clear and restore rules would be easy to get wrong. | Rejected. | -| Call options added to every generated method | Makes Call data visible in each method signature. | Changes every Grain interface and generated proxy. It creates a second path beside the existing `context.Context` path. | Rejected. | +| Call options added to every generated method | Makes Call data visible in each method signature. | Changes every Grain interface and Grain Reference implementation. It creates a second path beside `context.Context`. | Rejected. | -The chosen shape keeps one call input, one proxy path, and one isolation rule. +The chosen shape keeps one Call input, one Grain Reference path, and one +isolation rule. It also leaves method signatures stable when a future transport is added. ## Context lifetime and isolation @@ -190,7 +191,8 @@ call serialization, or cycle detection. ## Local Calls -For a local Call, the generated proxy passes the same `context.Context` to +For a local Call, the generated Grain Reference passes the same +`context.Context` to `Runtime.Invoke`. The root Runtime passes it to the local execution Runtime, which passes it through activation, the mailbox, and dispatch. @@ -241,21 +243,24 @@ existing cancellation and unknown-result rules. ## Activation and Deactivation -An Activation is created for the first Call that needs it. That triggering -Call's context is used by the existing activation factory path. Therefore -`OnActivate` may read the triggering Call's Request Context. +An Activation is created for the first Call that needs it. The Runtime copies +that Call's Request Context into a Runtime-owned activation context. +`OnActivate` can read the copied Request Context. Only the triggering Call supplies that context. If another caller waits for the same Activation, its Request Context is not merged into the activation and does not run `OnActivate` again. -The Runtime stores no Request Context on the Activation. Each method receives -the context for its own Call. After idle eviction, a later Call can create a -new Activation and its context can be visible to that new `OnActivate`. +The Runtime keeps the copied Request Context only until activation completes. +It does not keep it on an active Activation. Each method receives the context +for its own Call. A later Activation can receive a different starting context. -`OnDeactivate` receives the existing fresh background context. It never sees -the Request Context of the last Call, including during normal close, idle -eviction, ownership loss, or fault handling. +The triggering caller can stop waiting without canceling shared activation +work. Caller deadlines and cancellation are not copied into the activation +context. + +`OnDeactivate` receives a Runtime-owned context with no Request Context. It +never sees the Request Context of the last Call. ## Reminders and persistence @@ -283,10 +288,9 @@ sender cancels after the request can be delivered, the target method can still run and read Request Context. A late response is discarded by the existing transport path. -Close, Kill, and cluster death keep their current admission and method rules. -Request Context does not keep a Call alive, cancel a target method, or add a -retry. A clean shutdown test must show that a Call cannot use Request Context -to enter after the root admission gate closes. +Shutdown keeps the normal admission and method rules. Request Context does not +keep a Call alive, cancel a target method, or add a retry. A shutdown test must +show that it cannot bypass the root admission gate. ## Acceptance test matrix @@ -315,7 +319,7 @@ The focused tests prove the feature contract. The implementation batch must also pass `make test`, `make sim`, `make net`, and the repository `make ci` gate before the feature can be called complete. -## Gap +## Status The two public helpers, immutable snapshots, scalar normalization, validation, and finite-float checks are implemented. String values are checked for valid @@ -327,3 +331,8 @@ values. Focused tests cover these Request Context behaviors and the existing runtime and transport tests cover the shared admission, mailbox, cycle, shutdown, and cancellation rules. The matrix is not claimed to have a separate Request Context test for every shared-runtime row. + +Activation now uses a Runtime-owned context. It copies only the starting +Call's Request Context. It does not copy the caller deadline, cancellation +signal, or other context values. A caller can stop waiting while another Call +uses the shared Activation. A canceled queued Call does not enter a method. diff --git a/design/runtime.md b/design/runtime.md index c6e96eb..0b95feb 100644 --- a/design/runtime.md +++ b/design/runtime.md @@ -1,5 +1,50 @@ # Runtime +## Runtime setup + +`New` constructs a Runtime but does not start it. A constructed Runtime accepts +generated type installation and Grain registration only. It starts no +background goroutine and does no Store claim work. + +`Start(ctx)` validates the complete setup. It then freezes the type registry +and factory registry before it starts background components. The validation +includes the Store, Clock, durations, mailbox capacity, GrainType uniqueness, +and one factory for each installed Grain type. + +Start calls `Check(ctx)` on the State Store and Reminder Store. If cluster +configuration is present, Start also validates it and joins with the same +startup context. The eviction loop, Reminder poller, transport use, and cluster +work can start only after all checks and registry validation pass. Work that +outlives a successful Start uses a Runtime-owned context, not the Start +context. If a later startup step fails, Start stops completed steps in reverse +order before it returns. + +The setup states are: + +```text +constructed -> starting -> running -> stopping -> stopped + | + +-------------> start-failed +``` + +A failed Start cleans up all work that it started. A failed Runtime accepts no +Call. Install, Register, and a second Start return stable setup errors after +the Runtime leaves `constructed`. + +The Start context bounds startup only. A successful Runtime does not retain +its deadline or cancellation signal. Shutdown on a constructed or failed +Runtime moves it to `stopped`. + +A successful `New` takes ownership of configured Runtime components, such as a +Transport. It does not take ownership of the Application Stores. `Shutdown` +closes owned components in a constructed or start-failed Runtime. The +Application closes its Stores after Shutdown. A failed Start closes all +components that the Runtime owns. If `New` returns an error, the caller keeps +ownership of every supplied component. + +Starting background work in `New` is not valid. It lets the Reminder poller +claim a row before generated code and factories are available. + ## Activation A Grain's Activation is its in-memory instance on some Silo. Lifecycle: @@ -30,6 +75,7 @@ type DeactivationReason uint8 const ( Idle DeactivationReason = iota + 1 + ApplicationRequested OwnershipLost RuntimeClosed Faulted @@ -38,18 +84,29 @@ const ( **Use optional interfaces, not required methods.** Most Grains need neither; making them write two empty methods would be pure ceremony. Passing functions at registration time is out too — it would put "what this Grain does on activation" a mile away from the Grain itself. -`OnActivate` runs after state is read back from the store and before the first call enters the mailbox. If it returns an error, activation failed: the call that triggered this activation gets the error, the activation is not established, the placeholder closes with the error, and the next call starts over. There is no "half-activated" intermediate state — a Grain that failed `OnActivate` yet still serves is worse than having no hook at all. +`OnActivate` runs after the Runtime reads State and before the first Grain +method. If it returns an error, the starting Call gets that error. The Runtime +does not establish the Activation. The next Call starts a new Activation. A +Grain that fails `OnActivate` must not serve a method. -`OnDeactivate` runs right before the instance disappears, after the mailbox has been drained. It receives the `DeactivationReason` that first started the deactivation. There are only four reasons: +`OnDeactivate` runs before the Activation ends. For terminal deactivation, +the old mailbox drains first. For Deactivate on Idle, queued Calls wait while +the hook runs. No queued Call enters a method until the hook ends. The hook +receives the `DeactivationReason` that first started deactivation. There are +only five reasons: | Reason | What first triggers deactivation | What the app can do with it | | --- | --- | --- | -| `Idle` | The instance idles past the timeout. | Don't treat a local reclamation as the business object going offline. | -| `OwnershipLost` | The current node no longer owns the GrainId, or the view has no active owner. | Release node-local leases or connections; don't announce that the business object is gone. | +| `Idle` | The Activation idles past the timeout. | Do not treat local cleanup as removal of the Grain. | +| `ApplicationRequested` | The Grain calls Deactivate on Idle. | Reload a fresh Activation on the next Call. | +| `OwnershipLost` | The current Silo no longer owns the GrainId, or the view has no active owner. | Release Silo-local leases or connections. Do not report that the Grain is gone. | | `RuntimeClosed` | The root runtime begins a graceful stop. | Do teardown before the process exits. | -| `Faulted` | A method panicked, or the Grain asked to discard the current instance. | Don't treat an untrusted instance as a normal farewell; raise the alert level. | +| `Faulted` | A method panicked, or State made the Activation untrusted. | Treat the Activation as untrusted. | -This is the complete public set. A value may join the set only if it forces the app to make a different decision; a reason must not be added just because the implementation gained a branch. Panic and discard both mean the current instance is no longer trustworthy; migration and no-owner both mean the current node loses ownership — hence one value each. +This is the complete public set. A value can join the set only if it forces the +Application to make a different decision. Panic and failed State both mean the +current Activation is not trusted. Deactivate on Idle is a clean Application +choice, so it has a different reason. The reason is written in the same atomic transition where `beginDeactivation(reason)` moves the activation from `active` to `deactivating`. Later events must not overwrite it once the activation is already deactivating. If the root runtime has entered `closing`, one activation may have already started deactivating for `Idle`; its reason stays `Idle`. A deactivation reason describes why an activation first leaves; the root state machine describes whether the whole runtime admits calls, how it waits, and with which stop error it rejects calls. These are two concepts and must not share one enum. @@ -58,42 +115,89 @@ State is in the store anyway. The error has no caller. Like Reminder delivery failures, it goes to the Runtime error sink (see [timers.md](timers.md)), with no retry. The sink source carries `Deactivation{Reason: reason}`. -Each normal deactivation hook gets a fresh `context.Background()`. This context has no deadline and is never canceled; it inherits nothing from any caller of the Grain. A graceful stop waits for hooks that already started, so a hook must finish promptly. +Each normal deactivation hook gets a Runtime-owned context. It inherits no +caller deadline or Request Context. A `Shutdown` budget limits how long the +Runtime waits. It does not cancel a hook that already started. -**Neither `Kill()` nor this node being declared dead starts `OnDeactivate`.** An abrupt stop gives no teardown chance to hooks that have not started. Hooks that already started are not canceled, and an abrupt stop does not wait for them; handing them a canceled context would only create a third semantics of partial teardown. +**Neither `Kill()` nor this Silo being declared dead starts `OnDeactivate`.** +An abrupt stop does not start hooks. It does not cancel hooks that already +started, and it does not wait for them. This rule also covers deactivations already in flight. Idle eviction fires first and `Kill()` arrives later; an instance whose hook has not run yet must not run it either — the criterion is whether the instance was killed before its hook ran, not who initiated this deactivation. So "skip" is a fact recorded on the instance, not a parameter of some deactivation path: there can be many paths, but only one instance. ### Migration -This is a planned v0 breaking change. Existing `Deactivatable` implementations change their deactivation method to take a second `DeactivationReason` parameter. Implementations must not treat unknown reasons as normal deactivation in a default branch; the runtime only passes the four values in the table above. +This is a planned v0 breaking change. Existing `Deactivatable` implementations change their deactivation method to take a second `DeactivationReason` parameter. Implementations must not treat unknown reasons as normal deactivation in a default branch; the runtime only passes the five values in the table above. ## Local directory -Each node keeps a table: +Each Silo keeps one Call lane for each active GrainId: ```go -type activation struct { - id GrainId - instance any - mailbox *mail.Box - lastUsed time.Time +type callLane struct { + id GrainId + mailbox *mail.Box + act *activation } ``` -Arriving requests are looked up: on a hit, deliver to the mailbox; on a miss, activate first. +The Call lane owns the FIFO mailbox. The current Activation does not own it. +The mailbox starts an Activation in its first selected turn. Other Calls wait +in the same FIFO queue. Thus, two Calls cannot start two Activations for one +GrainId. -**Activation must be idempotent and mutually exclusive** — two concurrent requests for the same key must not activate two instances. A per-key "activating" placeholder handles this: the second request sees the placeholder and waits instead of activating again. +The lane stays open during Deactivate on Idle. The current method ends. The +old Activation runs `OnDeactivate`. The first waiting Call then starts a new +Activation. All waiting Calls keep their original order. -```go -type entry struct { - ready chan struct{} // closed once activation completes - act *activation // readable only after ready closes - err error -} +A fault closes the lane. The mailbox fails each Call that did not enter a +method. A later Call gets a new lane and can start a new Activation. The +Runtime makes this decision. The `internal/mail` package only provides FIFO +order, capacity, and cancellation before method entry. + +Ownership loss also closes the lane. A Call that did not enter a method +returns to the root Runtime. The root Runtime reads the latest Cluster View +and routes the Call to the current owner. The old Silo must not start a new +local Activation from an old ownership decision. This reroute result is +separate from Grain method errors. A method error cannot request a reroute. + +Activation uses a Runtime-owned lifecycle context. The Call that creates the +Activation can stop waiting without canceling shared activation work. The +Runtime copies that Call's Request Context into `OnActivate`. It does not copy +the caller deadline, cancellation signal, or other context values. + +The factory receives a complete `*gor.GrainContext`. State declarations are +open only while the factory runs. The Runtime freezes them before State load. +`OnActivate` cannot add State after the load point. + +## Activation admission limit + +The Silo reserves one slot before it creates a new Grain lane. Starting and +active Activations both count. The default limit is 10,000. +`WithMaxActivations` selects another positive limit. + +A Call for an active Grain does not need a new slot. A Call for a new Grain +reserves a slot before it allocates the Grain lane or mailbox. If no slot is +free, the Call returns `ErrOverloaded` without a lane, mailbox, factory, State +load, or `OnActivate` work. + +One lane owns one reservation while its Activation exists. The reservation +follows this state path: + +```text +free -> reserved -> active -> deactivating -> free + \-> free (failed start) ``` -This pattern is standard Go practice (equivalent to `singleflight`), but `golang.org/x/sync/singleflight` is not used: it uses a mutex internally, and mutex blocking does not count as durably blocking in `synctest`, so the bubble could not determine quiescence. We implement it ourselves, with channels. This is a recurring trade-off: being observable to `synctest` outranks reusing an existing library. +Factory, State load, and `OnActivate` are one start operation. A failed start, +including a panic or a Runtime stop during start, cleans its timers and +releases the reservation exactly once. A normal deactivation releases the +reservation only after its timers and `OnDeactivate` work finish. Calls to a +current Activation can continue while the Silo has no free reservation. + +The limit bounds Runtime-owned Activation overhead. It cannot bound the State +that Application code keeps inside one Activation. The Application must still +set a value that fits its memory budget. ## Idle eviction @@ -110,10 +214,14 @@ call ──▶ ring: who owns this id? ── self ──▶ runtime ──▶ l (in gor) │ other ▼ - transport.Send ──▶ remote node + transport.Send ──▶ remote Silo ``` -**The fork is in `gor`, not in `runtime`.** `runtime` only sees the left branch: give it a GrainId, it finds or builds the activation and delivers the call into the mailbox. It does not know the right branch exists — and so single-node mode has no extra code to route around (see [cluster.md](cluster.md)). +**The fork is in `gor`, not in `internal/runtime`.** The execution Runtime only +sees the local branch. It gets a GrainId, finds or builds the Activation, and +delivers the Call into the mailbox. It does not know that the remote branch +exists. Single-Silo mode has no extra routing code (see +[cluster.md](cluster.md)). ## Reentrancy @@ -125,12 +233,19 @@ gor's stance: no reentrancy for now. A deadlock in gor shows up as a call timeou If practice proves it necessary, it will be added — at method granularity, not type granularity. -Call cycle detection requires carrying a set of already-occupied Grains along the call chain. Go has no `AsyncLocal`; the only option is to carry it explicitly in `context.Context` — a genuine disadvantage of Go relative to .NET, see [research/go-capabilities.md](../research/go-capabilities.md) (in Chinese). +Call cycle detection carries the occupied Grains along the Call chain. Go has +no `AsyncLocal`. The runtime must carry the set in `context.Context`. See +[research/go-capabilities.md](../research/go-capabilities.md). ## Errors and timeouts Every call carries a timeout (from `ctx`). The semantics of the timeout must be stated clearly: a timeout means the caller is no longer waiting, not that the Grain stops executing. The method body may already have changed state. +Cancellation before method entry is different. The mailbox checks the Call +context in the same turn that selects queued work. If the context is done, the +mailbox settles the Call without dispatch. A canceled queued Call must never +enter the method later. + No automatic retry is provided. The runtime does not know whether a method is idempotent; retrying on the user's behalf causes problems like duplicate charges. Retrying is the caller's decision. ## Panic handling @@ -149,9 +264,46 @@ Migration is rejected because it would either replay and hit the same panic agai To the caller, this is the same kind of event as a timeout: the call did not execute, and state did not change. +## Deactivate on Idle + +`gor.DeactivateOnIdle(grainContext)` marks the current Activation. It does not +end the current method. Deactivation starts after that method returns. + +Calls that have not entered a method are not replayed because they have not +run. They wait for the requested deactivation and enter the new Activation in +their original order. Calls queued behind a panic or an untrusted State result +still fail, as specified above. + +Deactivate on Idle is safe to call more than once in one turn. It does not +delete State, Reminders, or the GrainId. It stops all Grain Timers for the old +Activation. A queued timer tick is discarded. A callback that already runs can +finish. The Runtime waits for that timer turn before it completes deactivation +and runs `OnDeactivate`. The old timer cannot enter the next Activation. + +A panic or untrusted State result in the same turn wins over the request. The +reason is `Faulted`, and queued Calls fail. The request starts deactivation only +after a turn returns without such a fault. + +## Callback containment + +Application callbacks run behind a panic boundary. This rule covers the Grain +factory, `OnActivate`, `OnDeactivate`, Grain Timer callbacks, `OnError`, and +`OnCall`. + +A panic in the factory or `OnActivate` fails activation. A panic in a Grain +Timer turn faults the Activation. A panic in `OnDeactivate`, `OnError`, or +`OnCall` cannot stop a Runtime goroutine or skip Runtime cleanup. + +`OnError` is the last observer. If it panics, the Runtime catches that panic. +It does not call `OnError` again for the observer panic. `OnCall` observes a +settled Call and cannot replace its result. + ## Root runtime stop state machine -The root runtime owns call admission and the stop reason. The inner execution runtime only owns activations and mailboxes; the cluster node only owns membership state; the poller and the transport are not another set of stop switches. The root state machine: +The root Runtime owns Call admission and the stop reason. The inner execution +Runtime owns Activations and mailboxes. The Cluster Silo component owns +membership data. The poller and Transport do not own stop modes. The root +state machine is: ``` running ── Close ──▶ closing ── graceful stop done ──▶ stopped @@ -160,7 +312,7 @@ running ── Close ──▶ closing ── graceful stop done ──▶ stopp │ ├── Kill ──▶ killing │ - └── node declared dead ──▶ dead ── abrupt stop done ──▶ stopped + └── Silo declared dead ──▶ dead ── abrupt stop done ──▶ stopped ``` There are only four transition functions: `beginClose` moves `running` to @@ -185,34 +337,106 @@ The following entry points all use the same `admit`; none has its own closing ch Probes are not Grain calls and do not count toward the call count; but they read the same root state and refuse to reply when it is not `running`. When an inbound request is rejected for stopping, it is not first checked against a separate `Done` check, and the result does not differ by local or forwarded origin. -`closing`, `killing`, and the `stopped` reached from either of them all return the root package's `ErrRuntimeClosed`, whose stable code is `gor.runtime_closed`. `dead` and the `stopped` reached from it return `gor.node_dead`. The cross-node reconstruction rules for these two errors are in `errors.md`; direct and forwarded calls judge by the same stable code. Internal mailbox, execution runtime, or transport errors must not supersede this root-level admission result. +`closing`, `killing`, and their `stopped` states return `ErrRuntimeClosed`. +Its stable code is `gor.runtime_closed`. `dead` and its `stopped` state return +`gor.node_dead`. [errors.md](errors.md) defines the Cross-Silo reconstruction +rules. Local and forwarded Calls use the same stable code. Internal errors +must not replace this root admission result. ### Graceful stop and abrupt stop -After entering `closing`, the poller starts no new calls, the cluster node leaves gracefully, and the mailbox closes. Admitted methods already executing may finish; calls in the mailbox that have not started are rejected per existing mailbox rules — no migration, no replay. The root runtime waits until all admitted calls have released, executing methods have finished, deactivations have completed, and the infrastructure goroutines it started have exited, then calls `finishStop`. The transport must stay open until admitted forwarded requests and inbound replies are done. +After entering `closing`, the poller starts no new Calls. The Cluster Silo +component leaves, and the mailbox closes. Active methods can finish. Queued +Calls are rejected. The root Runtime waits for admitted Calls, methods, +Deactivations, and infrastructure goroutines. The Transport stays open until +forwarded requests and replies finish. After entering `killing`, the same order applies: reject new calls first, then cancel executing methods, reject the mailbox queue, and skip deactivation hooks that have not started. It does not wait for user methods — Go cannot forcibly abort code that ignores cancellation. `stopped` is reached once the abruptly stopped runtime infrastructure has exited; it does not mean such user code has necessarily returned. -A `Kill` during `Close` must immediately escalate to the latter semantics: cancel running calls, reject the queue, and skip deactivation hooks that have not started. All calls waiting for `Close` to finish conclude under the abrupt stop as part of this escalation; the runtime must not keep waiting for user methods in the name of graceful stop. The closing interfaces of the inner execution runtime, the cluster node, and the transport must all support this escalation. A subcomponent that already received the graceful stop command must not treat a later `Kill` as a no-op. +A `Kill` during `Close` must start abrupt stop. It cancels active Calls, +rejects the queue, and skips Deactivation hooks that have not started. The +inner execution Runtime, Cluster Silo component, and Transport must support +this change. A later `Kill` must not be a no-op. -The root runtime's waiting is expressed only with channels: admitted calls reaching zero, the execution runtime ending, the cluster node ending, the poller ending, and the transport ending each close their own completion channel; the stop coordinator only receives these channels. Short critical sections may guard a state transition or a count, but a mutex, condition variable, or `WaitGroup` must not be used to wait for another call to complete. +The root Runtime waits only on channels. The admitted Call count, execution +Runtime, Cluster Silo component, poller, and Transport each close a completion +channel. The stop coordinator receives these channels. It must not use a +mutex, condition variable, or `WaitGroup` to wait for a Call. + +The public stop API is: + +```go +func (rt *Runtime) Shutdown(ctx context.Context) error +func (rt *Runtime) Stopping() <-chan struct{} +func (rt *Runtime) Done() <-chan struct{} +``` + +`Shutdown` starts graceful shutdown and waits for `Done`. If `ctx` ends first, +Shutdown escalates to the abrupt stop path and returns `ctx.Err()`. `Stopping` +closes when Call admission ends. `Done` closes only after Runtime +infrastructure has ended. The old `Done` meaning, which reports the start of +shutdown, is removed. + +If `ctx` is already done when Shutdown starts, the Runtime enters the abrupt +stop path without starting graceful component shutdown. It does not race a +clean Cluster leave against an abrupt stop. ### Death declared by the cluster -The cluster node must report why it ended; it must not just close a bare `Done` channel that carries no reason: an active `Close` also writes the node's membership row as `dead`, which is not the same as the root runtime being declared dead by others. When the root runtime, still in `running`, receives an external declaration of death, it calls `becomeDead`: first closes admission and the public stop signal, then concludes under abrupt stop. A root runtime already in `closing` or `killing` keeps its state. +The Cluster Silo component must report why it ended. An active `Close` writes +the Silo membership row as `dead`. This is different from an external death +declaration. A root Runtime in `running` calls `becomeDead` for an external +declaration. It first closes admission and the public stop signal. It then +uses abrupt stop. A Runtime in `closing` or `killing` keeps its stop mode. Thus the inner execution runtime can still drain while `closing`, but it is no longer an externally reachable "stop signal closed, yet still admitting calls" window. That window is now a state with a name, admission rules, and completion conditions — not a gap left behind by goroutine execution order. -### Gap +### Current implementation The root runtime's stop state machine is implemented: the four transition functions `beginClose`, `beginKill`, `becomeDead`, `finishStop`, with the atomic `admit`/release as the only admission gate; the public `Runtime.Invoke`, the inbound `invoke` handler, and Reminder deliveries share the same entry and admit before ownership and forwarding. `closing`/`killing` and the `stopped` reached from them return `gor.runtime_closed`; `dead` and the `stopped` reached from it return `gor.node_dead`. -Stop coordination is implemented as pure channel waiting: the execution runtime exposes `BeginClose`/`BeginKill` plus a `Done()` channel, the cluster node exposes a `DeclaredDead()` channel, and the root coordinator in `closeGracefully`/`closeImmediately` receives only `clusterDone`, `engine.Done()`, `drained`, and `transportDone`. The inner execution runtime supports the `closing → killing` escalation (`BeginKill` from `closing` is not a no-op: it closes the `killing` channel, marks deactivation hooks that have not started to be skipped, and cancels execution). The cluster node explicitly reports "declared dead externally" via `DeclaredDead()` rather than "exited on its own", and the root layer no longer infers the reason from whether it initiated the stop itself. A declared-dead node no longer publishes the final empty view, so graceful migration and abrupt stop do not race to start. +Stop coordination uses only channels. The execution Runtime gives +`BeginClose`, `BeginKill`, and `Done()`. The Cluster Silo component gives +`DeclaredDead()`. The root coordinator receives `clusterDone`, `engine.Done()`, +`drained`, and `transportDone`. `BeginKill` changes `closing` to `killing`. +It skips hooks that have not started and cancels execution. `Kill` also +cancels a Cluster leave write. `DeclaredDead()` reports an external death. +A declared-dead Silo does not publish a final empty view. + +Transport teardown meets this invariant. A graceful stop calls +`Transport.Close`. It stops new requests and flushes active replies before it +closes sockets. Abrupt stop calls `Transport.Kill`. It cancels handlers and +drops replies that are not written. A later `Kill` changes a graceful stop to +an abrupt stop. Channel waits keep Transport on the stop path. The two-Silo +fake-network test is deterministic. See [transport.md](transport.md). + +The implementation has five deactivation entry points. Idle eviction passes +`Idle`. Deactivate on Idle passes `ApplicationRequested`. View eviction passes +`OwnershipLost`. Runtime stop passes `RuntimeClosed`. Panic and State discard +pass `Faulted`. + +The Runtime owns Activation lifecycle contexts. Only Request Context from the +starting Call enters `OnActivate`. Caller cancellation cannot cancel shared +Activation work. Each `OnDeactivate` gets a new context with no Request +Context. + +Factory, lifecycle, `OnError`, and `OnCall` callbacks have panic boundaries. +A callback panic cannot skip Runtime cleanup. An `OnCall` panic cannot change +the settled Call result. + +Call cycle detection is implemented: each call carries the chain of Grains it already occupies in its context; forwarded requests carry the chain on the wire; and a call whose target is already on its chain is rejected at delivery with an error naming the cycle, projected onto the stable code `gor.call_cycle`. The chain is per call, so a slow call that is not a cycle still times out as a plain timeout. -Transport teardown meets the invariant above. `closeTransport` routes by stop mode: a graceful stop calls `Transport.Close`, which stops accepting new requests and flushes in-flight replies to the wire before closing sockets; an abrupt stop and a declared-death collapse call `Transport.Kill`, which cancels handlers and drops replies not yet written, and a Kill arriving while the graceful close is in progress escalates it. The owner's root inflight still releases at method completion, and the runtime still cannot learn when the peer received the reply — the flush is the transport's job, and the transport-level join is channel-based so it stays on the stop-coordination critical path without violating the `synctest` rules ([testing.md](testing.md) rule 4; [transport.md](transport.md) restates it for the in-flight join). The two-node fake-network scenario test is deterministic: the fake transport's graceful close waits for in-flight deliveries, so the invariant is enforced by the transport itself rather than by scheduling luck, and the blocking-handler flush test specified in [transport.md](transport.md) is in place. +The only reason the abrupt stop primitive exists is simulation tests and +Shutdown escalation. It is not the normal Application stop API. -Deactivation reasons are implemented: `activation` saves the reason in the same atomic transition of `beginDeactivation(reason)`; `waitForDeactivation` and `skipOnDeactivate` read it in the same critical section and hand it to the hook. The reason is written only in that transition; later events (including the root runtime having entered `closing`) do not overwrite it. The four entry points map one-to-one onto the table above: idle eviction passes `Idle`, `Deactivate` (view eviction or no active owner) passes `OwnershipLost`, `beginStopDeactivationsLocked` passes `RuntimeClosed`, and `stopActivation` for panic and discard passes `Faulted`. Each hook gets a fresh `context.Background()` (no deadline, never canceled, inheriting no caller context); `Kill()` and being declared dead still skip hooks that have not started, and hooks that already started are neither canceled nor waited for. Hook errors are reported through the structured sink, with source `Deactivation{Reason: reason}`; see [timers.md](timers.md). +## 0.1.0 Status -Call cycle detection is implemented: each call carries the chain of Grains it already occupies in its context; forwarded requests carry the chain on the wire; and a call whose target is already on its chain is rejected at delivery with an error naming the cycle, projected onto the stable code `gor.call_cycle`. The chain is per call, so a slow call that is not a cycle still times out as a plain timeout. +Runtime setup and context-aware shutdown are implemented. `New` starts no +background component. `Start` validates and freezes setup before it starts +Runtime work. -The only reason `Kill()` exists is simulation tests — a real process crash does not politely call a function first. It is not a shutdown API for users; users shut down with `Close()`. +Activation lifecycle, queued cancellation, Deactivate on Idle, deactivation +reasons, callback panic containment, and Grain Timers are implemented. Normal +deactivation waits on Timer completion channels before `OnDeactivate`. Abrupt +shutdown stops Timer owners before Runtime `Done` but does not wait for user +callback code that ignores cancellation. diff --git a/design/scheduling.md b/design/scheduling.md index d4b2eb5..c9c5c02 100644 --- a/design/scheduling.md +++ b/design/scheduling.md @@ -22,6 +22,10 @@ type call struct { func (b *Box) run() { for c := range b.in { + if err := c.ctx.Err(); err != nil { + c.reply <- result{err: err} + continue + } v, err := c.fn(c.ctx) c.reply <- result{v, err} } @@ -35,7 +39,9 @@ Serialization falls out directly from the fact that only one goroutine runs the Not replaced by `select` with `ctx.Done()`: that would add a branch to every reply, and the thing it guards against is prevented by a single buffer slot. -The whole `mail` package is estimated at around 100 lines. For comparison: Orleans' `Scheduler/` measures 823 lines, of which `WorkItemGroup` is 336 — because .NET needs a custom `TaskScheduler` to guarantee returning to the same logical execution context after `await`. In Go, a goroutine is naturally an execution context; the problem does not exist. +The whole `internal/mail` package is small. For comparison, Orleans needs a +custom `TaskScheduler` to return to the same logical execution context after +`await`. In Go, a goroutine is already an execution context. ## Channel capacity @@ -46,11 +52,27 @@ Buffered or unbuffered directly changes the backpressure semantics: The choice: bounded buffer, reject when full (returning a clear overload error). An unbounded queue disguises a memory problem as a latency problem, and blocking lets one hot Grain drag down the whole process. Capacity is configurable. +Capacity must be greater than zero. A zero-capacity channel makes admission +depend on whether the mailbox goroutine is receiving at the same instant. That +result is not a useful or repeatable overload rule. + +## Cancellation before method entry + +The mailbox checks `ctx.Err()` after it selects a queued Call and before it +dispatches the method. This point decides method entry. A canceled queued Call +returns its context error and does not run. + +Cancellation can race with this check. If the check sees cancellation first, +the method does not run. If dispatch starts first, normal Call rules apply: the +caller can stop waiting, but the method can continue. + ## Request order Consecutive calls from the same caller to the same Grain execute in the order they were initiated — the channel is FIFO, which holds naturally in the local case. -**No order guarantee across nodes.** Network reordering plus reconnection breaks it, and the complexity of sequence numbers and reorder buffers is not worth it. The docs must state this explicitly; users must not be led to assume an order guarantee. +**No order guarantee across Silos.** Network reordering and reconnection can +change the order. `gor` does not add sequence numbers or reorder buffers. The +product documents must state this limit. ## Relationship with synctest @@ -59,9 +81,22 @@ That this design is fully observable by `testing/synctest` is not a coincidence - A goroutine blocked on a channel counts as "durably blocked"; `synctest.Wait()` can determine that the system is quiescent. - If serialization were done with mutexes plus condition variables, `synctest` could not determine quiescence (mutex blocking is not durably blocking), and the whole testing strategy collapses. -So `sync.Mutex` must not appear in the `mail` package for cross-call waiting. A short critical section that merely protects a map is fine; using it to "wait" is not. +So `sync.Mutex` must not appear in `internal/mail` for cross-Call waiting. A +short critical section that only protects a map is fine. It must not be used +to wait. -## Relationship with Reminders +## Relationship with Grain Timers and Reminders + +A Grain Timer tick is a local mailbox turn for one Activation. It follows the +same serialization rules as a Call. A timer waits for mailbox capacity instead +of failing with Call overload. `Stop` or deactivation cancels a tick that has +not entered its callback. After enqueue, the Timer owner waits for the mailbox +to report whether the callback ran or was discarded. Abrupt shutdown can end +that wait after callback entry because it does not wait for user code. A tick +cannot be forwarded to a new Activation. Activation setup failure also ends the +wait because the failed Factory still owns the current mailbox turn. Deactivate +on Idle ends the wait after the requesting turn. A queued Call can then create +the next Activation even when an old Timer turn is behind it. Persisted Reminders (`Reminder`) do not run on the mailbox's clock. They are "a table plus a poller": the poller finds due items and constructs an ordinary call delivered to the target Grain's mailbox. @@ -70,3 +105,10 @@ So to the Grain, a Reminder is indistinguishable from an ordinary method call an **Explicitly not a repeat of Orleans Reminders v1** — an in-memory cache plus ring partitioning plus complex ownership transfer, which Orleans itself replaced with `Orleans.DurableJobs` (v2, measured at 5278 lines, still preview). Table plus polling is dumber but easier to verify, and accurate enough — persisted Reminders should never promise millisecond precision. Details in [timers.md](timers.md). + +## Status + +The Runtime requires a positive mailbox capacity. The mailbox checks the Call +context after dequeue and before method entry. Deterministic tests cover both +sides of the cancellation and method-entry boundary. Grain Timer turns wait +for capacity and get an exact mailbox result after enqueue. diff --git a/design/simulation.md b/design/simulation.md index 25a2767..d6cc9eb 100644 --- a/design/simulation.md +++ b/design/simulation.md @@ -6,7 +6,8 @@ Skeleton = seed + fault injection + crash and restart + event log + invariant assertions. -**No real network.** Cross-node faults are injected through the fake `Transport` under the `sim` build tag; it hangs a new fault source onto the existing skeleton, not making tests depend on a real network. +**No real network.** The fake `Transport` injects Cross-Silo faults under the +`sim` build tag. Tests do not require a real network. This is what "DST before the cluster" actually means: build the driver and the assertions first; when the cluster arrives, it plugs straight in. @@ -49,15 +50,24 @@ This is not lowering the bar. What reproduction must guarantee is "re-running wi Interleaving-dependent correctness does not rely on reproduction; it relies on two things: invariant assertions (must hold for every interleaving) and porcupine (checks whether a history linearizes, which is interleaving-independent by nature). -Go offers nothing stronger ([testing.md](testing.md) cites Resonate's same conclusion). Writing the impossible as possible only makes nobody know what to trust when the first flake appears. +Go gives no stronger scheduling control. The design must not promise +reproduction for scheduling decisions that it cannot control. ## A fault names its target *The log splits in two* makes one promise about reproducibility: re-running with the seed yields byte-identical **decision** lines. For that to hold, the decision half must be a pure function of the seed — nothing the goroutine scheduler can change may flow into it. Tracing exactly where that line was crossed is the whole of this question. -The decision encoding reads node liveness — `liveNodeIDs()`, `stoppedNodeIDs()` — to pick valid targets: which node to crash, to call, to restart. That read is **necessary**; the driver cannot pick a live node without knowing which are live, and every action case depends on it. It is **safe** on one condition: the liveness it reads must itself be seed-determined. Crash and leave are decisions, so they move liveness deterministically. Restart is a decision to *attempt*; whether the attempt succeeds is an observation, pinned to the seed by the remedy below. Those three are the movers this section owns. A fourth mover — the cluster declaring a node dead by vote — is an observation that is *not* seed-determined; it has its own section, *Liveness has two sources*, because its remedy is at the read rather than at a fault. +The decision encoding reads Silo liveness through `liveNodeIDs()` and +`stoppedNodeIDs()`. It uses this data to select a Silo for a crash, Call, or +restart. This read is required. Its value must be a function of the seed. +Driver crash and leave actions meet this rule. Restart success is an +observation, so the fault target must also come from the seed. Cluster death +votes are observations and must not feed this decision. -A one-shot fault with no target breaks the condition. The member fault is a single field consumed by whichever `WriteMember` or `ListMembers` takes the lock first. During a restart the new runtime's join write and a survivor's heartbeat write both reach the member store while such a fault is pending; they write **different rows**, both can succeed, and whichever the scheduler runs first consumes the fault. The fault is **drawn** deterministically and **applied** nondeterministically. Its application fixes the join write's fate, which fixes restart-success — an observation — which moves liveness, which the decision encoding reads next step. A divergence the split permits on the observation side (restart outcome) walks through a necessary read into the decision side (which node the next step targets), and the decision lines split from the next step on. +A one-time fault without a target breaks this rule. A restart join write and a +survivor heartbeat write can reach the member store together. The scheduler +can then select which write consumes the fault. That result changes restart +success and Silo liveness. The next decision can select a different Silo. That is the defect, and it is **one root, not two**. "May a fault be consumed nondeterministically" and "may the decision encoding read runtime state" are the same leak seen from two ends: the fault's first-arrival consumption is what makes the runtime state scheduler-dependent, and reading that state is what carries the dependence into the decision half. Fixing the read is not an option — the read is necessary, and the state is scheduler-dependent only because of the fault. The fix is at the fault. @@ -65,7 +75,11 @@ That is the defect, and it is **one root, not two**. "May a fault be consumed no A fault is two facts — a kind and a target. The kind has always been a decision. The target must be a decision too. The store fault already does this: keyed by GrainId, drawn in the driver, read fresh on every call to that Grain, not consumed by first arrival. The member fault must meet the same bar. This is not new machinery; it is removing the inconsistency that left the member fault the odd one out. -The target of a member fault is a member row — the `(node address, generation)` the member store keys on — for the write and delay kinds, and a node for the list-error kind. The driver draws the target with the seed, the same way it draws node indices for calls and crashes, resolving a node to its current generation. The fault fires only on an operation addressing that target; if none does this step it does not fire — dropped, deterministically, the way a store fault on a Grain nobody calls does not manifest. The delay kind is already shape-bound to an active-refresh write; it takes the target row as well, so two survivors heartbeating no longer race for one delay token. With the target fixed by the seed, restart-success is fixed by the seed (a write or list fault fails restart exactly when it targets the restarting node; a delay never does), liveness is fixed by the seed, and the decision half is pure again. +The driver uses the seed to select a member fault target. Write and delay +faults target one (`node_addr`, `generation`) row. List faults target one Silo. +The fault only applies to that target. A delay also targets one active-refresh +write. Thus, concurrent heartbeats cannot race for one fault. Restart success +and Silo liveness are then functions of the seed. ### Per seam @@ -79,7 +93,9 @@ Whether a seam carries this defect turns on one test: does its first-arrival con - **Reminder list fault** — a single unkeyed field with first-arrival behavior. A list error is read-only and the poller retries on the next tick. It does not change the decision sequence today. -- **Network fault** — not this shape. A partition is a deterministic group map applied per node pair (a whole pair goes silent); a per-message drop is drawn by the seed in the driver; delay is drawn unconditionally in the driver and released by the clock. None latches onto a target by first arrival. +- **Network fault** - not this shape. A partition is a deterministic map of + Silo pairs. The seed selects a request drop and a delay. The `Clock` releases + the delay. First arrival does not select a target. ### What does not change @@ -91,45 +107,74 @@ The reproduction test does not change in shape. It still compares decision lines ## Liveness has two sources; only one may feed the decision -The liveness the decision encoding reads must be a pure function of the seed. A node stops being live in two unrelated ways, and only one of them qualifies. +The Silo liveness used by a decision must be a pure function of the seed. -- **Driver liveness.** The driver crashed the node, left it, or its restart failed. Crash and leave are decisions; restart-success is an observation pinned to the seed by *A fault names its target*. This liveness is a pure function of the seed. +- **Driver liveness.** The driver crashed or left the Silo, or its restart + failed. Crash and leave are decisions. A seeded fault target determines + restart success. This liveness is a function of the seed. - **Cluster liveness.** Probes time out, neighbours vote, the vote clears the threshold, and the runtime collapses itself to dead (`becomeDead`, stop code `ErrNodeDead`). Which probe `select` wins when several are ready is scheduling. This is an **observation** by design — the probe loop is supposed to be scheduling-dependent; making it deterministic would defeat the test. -Both close the same channel (`rt.Done()`), so one `runtimeStopped` check cannot tell them apart. The decision encoding must not read that one check through `liveNodeIDs()`, `stoppedNodeIDs()`, `targetPool()`, or the action choice — it would read both. An observation that moves the decision half breaks reproduction. This is the member-fault leak from the read end: same defect, a different door. +Both close `rt.Stopping()`, so one `runtimeStopped` check cannot tell them +apart. The decision encoding must not read that check through `liveNodeIDs()`, +`stoppedNodeIDs()`, `targetPool()`, or the action choice. It would read both +sources. An observation that moves the decision half breaks reproduction. +This is the member-fault leak from the read end. ### The read, not the source -*A fault names its target* remedies the member-fault leak at the source: the driver owns the fault, and the seed binds its target. That remedy does not exist here — the source of cluster liveness is the production probe/vote mechanism, and it is *meant* to be scheduling-dependent. The source stays; the decision encoding reads a liveness set the driver owns — the nodes it has crashed, left, or failed to restart — and nothing else. The cluster's `becomeDead` keeps closing `rt.Done()`; observations and invariants keep reading cluster liveness. The two notions do not have to agree, and after a vote they routinely will not. +*A fault names its target* fixes the member-fault leak at the source. The driver +owns the fault, and the seed binds its target. That fix does not apply here. +The production probe and vote code owns cluster liveness. Its result can depend +on scheduling. The decision encoding reads only liveness that the driver owns. +The set contains Silos that the driver crashed, left, or failed to restart. +The cluster's `becomeDead` closes `rt.Stopping()`. Observations and invariants +can still read cluster liveness. The two liveness sets can disagree after a +vote. ### What disagreement costs, and buys -When the driver thinks a node is live and the cluster has declared it dead, the driver still targets it. A call returns `ErrNodeDead` at admission; a crash is a no-op on an already-dead root; a restart closes and rebuilds it. None of this breaks an invariant — `ErrNodeDead` is a clean rejection and the store is untouched — and it is exactly the state the cluster exists to test: two views of who is alive, disagreeing. Reading driver liveness presses on that window; a decision fed by cluster liveness would fold to the cluster's view the instant a vote clears and never reach it. +The driver can target a Silo that the Cluster declared dead. A Call returns +`ErrNodeDead` before it changes the Store. A crash is a no-op for an already +dead root. A restart closes and rebuilds the Silo. These actions test the +window where the driver and Cluster views differ. -The cost is narrow. The fault target pool grows by the cluster-dead-but-driver-live nodes, and a fault drawn against such a node may not fire — the node rejects at admission before the fault injection point. This lowers the trigger rate, but only inside the disagreement window. Outside it the two liveness notions coincide and the rate is unchanged. If the rate regresses past what *A fault names its target* fought to reach, the answer is more seeds, not re-conflating the read. +The fault target pool includes a Silo that is Cluster-dead but driver-live. A +fault for this Silo might not run because admission rejects the Call. This can +reduce fault coverage only while the views differ. More seeds can restore +coverage without mixing the two liveness sources. ### What stays The probe loop, the vote, and `becomeDead` are production code; the skeleton does not touch them. Concurrency is not reduced. The reproduction test compares decision lines. -The decision side and the invariant side each have their own liveness reader. `checkInvariants` reads cluster liveness — owner uniqueness is an observation. The driver-owned liveness feeds only the decision side: feeding the invariant side would make a cluster-dead-but-driver-live node read its stale last view and raise a false owner-uniqueness red. +Decisions and invariants use different liveness readers. `checkInvariants` +reads Cluster liveness because Ownership uniqueness is an observation. +Decisions read driver liveness. This prevents a dead Silo with an old view from +reporting a false Ownership error. -## What a "node" is +## How simulation represents a Silo -A node = one `runtime.Runtime`. Several nodes share one `Store`. Before [step 6](../ROADMAP.md#6-multiple-nodes), nodes have no other connection. +A production Silo is a process that hosts a Grain Runtime. The simulation +represents each Silo with one `gor.Runtime`. It can run several simulated Silos +in one process. They share one `Store`. Before [step +6](../ROADMAP.md#6-multiple-silos), they have no other connection. - **Crash** — drop all in-memory state, keep the store. - **Restart** — build a new `Runtime` on the same store. -**Double activation becomes testable here.** Two Runtimes sharing one store activating the same GrainId is double activation by itself — no network partition needed to produce it. The core risk of cluster instability is already covered by assertions at step 4; step 6 only changes the way it is produced. +**Double Activation becomes testable here.** Two Runtimes can activate the +same GrainId from one Store. This does not require a network partition. Step 4 +tests the invariant. Step 6 adds Cluster fault causes. ## A crash is not Close `Close()` drains the mailbox and waits for in-flight calls to finish. That is a graceful stop. -A crash must make in-flight calls return with an error immediately, giving Grains no teardown chance. So `runtime` needs one more stop path: `Kill()` — cancel all in-flight calls' contexts, close the mailbox, do not wait for draining. - -**`Kill()` must make every goroutine exit.** This is not cleanliness: synctest panics with a deadlock report when every goroutine in the bubble blocks forever. A leaking crashed node does not leak quietly; it takes down the whole simulation test. +A crash must make in-flight Calls return with an error at once and give Grains +no teardown chance. The internal execution Runtime has a `Kill()` path. It +cancels all in-flight Call contexts, closes the mailbox, and does not wait for +**`Kill()` must make every goroutine exit.** A leaked goroutine can cause a +`synctest` deadlock. A crashed Silo must not leak a goroutine. Go cannot kill a call that is executing a user method. `Kill()` can only cancel the context; a user method that ignores its context keeps running to completion. This differs from a real process crash, but there is no other way, and the Grains in simulation tests are written by us. @@ -139,13 +184,18 @@ So the fake store must be able to report "no work in hand", and the driver waits **Wherever time is advanced, wait for it — not only before observations.** The bubble's clock only moves while every goroutine is durably blocked, and the moment `synctest.Wait()` returns, the driver itself is running again: as long as the driver does not block, injected delays never move a step. -Waiting only before observations does not cost "a bit of slowness"; a component can stay stuck in one injected delay forever: its loop never turns again, yet it looks alive. This symptom disguises itself as a bug in the system under test — last time it disguised itself as "two nodes each computing a view that contains only itself", and it took a long investigation to find the driver. +The driver must wait at each point that can advance time. Otherwise, a +component can remain in one injected delay and appear live. This can cause two +Silos to keep incomplete membership views. -A side benefit: writes the crashed node never got to answer land in this step's observation — exactly as in the real world. Step 6's fake network will hit the very same problem; the interface is settled here first. +An observation also includes a write that the crashed Silo could not answer. +The fake network uses the same rule. ## The fault injection seam -The `Store` is the storage-side I/O seam. (The cross-node `Transport` seam is its own subject below, in *The fake network's fault classes*.) The fake store implements `store.Store` and injects four kinds by seed: +The `Store` is the storage I/O boundary. The Cross-Silo `Transport` boundary is +defined below. The fake Store implements `store.Store` and injects four fault +kinds from the seed: - Read failure - Write failure that did not take effect @@ -158,9 +208,11 @@ Slow responses are not a separate kind; they are the delay. ## The fake network's fault classes -The fake `Transport` is the cross-node fault seam. Four things get conflated under "network fault"; only some are real, distinct faults under this transport model. +The fake `Transport` is the Cross-Silo fault boundary. This Transport model +has three different network fault types. -**Partition** — a set of node pairs that cannot talk. Implemented as a group map; a `Send` across groups returns a partition error at once. Deterministic: the partition is a driver decision, not a coin flip per message. This is the current capability. +**Partition** - a set of Silo pairs that cannot communicate. A `Send` across +groups returns a partition error. The driver selects the group map. **Drop** — a message that never arrives. A per-message drop keyed to the seed is the same shape; partition covers the coarse form (a whole pair goes silent). @@ -175,9 +227,12 @@ So the meaningful timing fault is delay, and delay produces every cross-message ## What network injection cannot catch -Network fault injection controls **inter-node** message timing: when a message, once handed to the transport, is delivered to the recipient. It does not control **intra-node** goroutine scheduling: which of two goroutines on the same node runs first. [testing.md](testing.md) states the boundary plainly — synctest is a unit-testing tool, not a DST framework, and does not control goroutine scheduling order. +Network faults control when Transport delivers a request between Silos. They +do not control goroutine order inside one Silo. `synctest` does not control +goroutine scheduling order. See [testing.md](testing.md). -The consequence is sharp. A bug whose window is "node A tears itself down before it finishes a local step" is a scheduling race on one node. Delaying or reordering the network message downstream of that step does not move the window — the close still races the local write regardless of when the recipient sees the result. Network injection cannot reach this class. +A local teardown race is a scheduling fault inside one Silo. A network delay +does not change that race. A contract test must control the local boundary. What does catch it: a **contract test with a blocking seam** at the teardown boundary, plus the **`-race` detector** as the statistical net. The blocking seam makes the window unavoidable — the test owns a channel that gates the in-flight work, so close cannot complete until the test releases it, and a broken close fails every run, not one in ten. This is how the graceful-close flush invariant is verified ([transport.md](transport.md) Testing); it is the pattern for teardown-ordering bugs generally, not a gap network injection could fill. @@ -188,7 +243,7 @@ Inside the bubble, `time.Now()` is already fake, and `clock.Real` works as-is. S No, for two reasons: - Unit tests outside the bubble also need to control time, and there `time.Now()` is real. -- One bubble has exactly one clock. To test clock skew between nodes at step 6, each node needs a `Clock` with an offset. +- One bubble has one clock. Each Silo uses a `Clock` offset to test clock skew. "No `time.Now()` in production code" stays. It guarantees time comes from a replaceable place; it does not guarantee any particular replacement. @@ -235,7 +290,7 @@ No JSON. When something fails, a human stares at two logs looking for the first The `sim/` package, build tag `sim`, tests named with a `TestSim` prefix (`make sim` filters with `-run TestSim`). -`sim` depends on `gor`, `runtime`, `store`, and `clock`; none of them depend on it. +`sim` depends on `gor`, `store`, and `clock`; none of them depend on it. **Invariants run over a batch of seeds, not one.** One seed walks one trajectory and cannot cover several fault combinations. The seed list is hardcoded (for example 64 consecutive seeds from some base), so failures reproduce without introducing wall-clock randomness. Reproducibility has two guards, for two distinct bug classes. A single fixed seed, run twice, catches the skeleton's classic bug — drawing the PRNG from several goroutines — because that breaks *every* seed. It does not catch a leak that breaks only *some* seeds; for that, every seed the batch runs is run twice and its decision lines compared. The single-seed test is the smoke alarm; the batch is the contract. @@ -263,23 +318,32 @@ Step 5: the Reminder table is a new fault source (scan failures, claim failures, Step 6a: the membership table is yet another fault source, shaped like the Reminder table. New invariants: -- **Membership views eventually converge.** After faults stop, all live nodes compute the same view. -- **After convergence, one key belongs to one node.** During convergence there may be more than one: that is the acknowledged double-activation window, not a bug. -- **Declaring death is irreversible.** A declared-dead node does not crawl back to `active` by itself. +- **Membership views eventually converge.** After faults stop, all live Silos + compute the same view. +- **After convergence, one GrainId belongs to one Silo.** During convergence, + two Silos can claim Ownership. +- **Declaring death is irreversible.** A declared-dead Silo does not become + active by itself. -"Live nodes" means nodes that **still consider themselves alive**, not nodes the driver did not crash. A persistently slow membership table makes nodes declare each other dead until everyone self-terminates (see [cluster.md](cluster.md)); that is 6a's known failure mode, and with no owner left at all it must not count as a broken invariant. After everyone is dead, the restart action brings nodes back: a fresh generation, a new row, and convergence must still happen. +A live Silo still considers itself active. This is different from a Silo that +the driver did not crash. A slow membership table can make all Silos declare +each other dead. This is a known Cluster limit. A restart uses a new generation +and row. The views must converge again. -**Owner uniqueness must be checked with a batch of probe GrainIds**, not just the two under test. `Owns` is pure computation; it writes nothing to storage and activates nothing, so a few dozen keys cost nothing, and checking a full batch equals comparing views: whenever two nodes' views differ, some key's owner necessarily disagrees. This also avoids opening a "hand over the view" method on the runtime for tests. +**Ownership uniqueness uses a batch of GrainIds.** `Owns` is a pure function. +It writes no State and starts no Activation. Different Silo views will disagree +on Ownership for one or more values in a sufficient batch. **After ownership filtering, `claim-lost` is no longer an event every seed batch hits.** A non-owner poller never claims; two pollers claiming the same row is only possible inside the inconsistent-view window. This is the result of [timers.md](timers.md)'s rule, not a coverage regression. Step 6b: - The fake `Transport` implementation: network partitions, dropping, and delay (the meaningful timing fault; reorder is not distinct — see *The fake network's fault classes*). -- One `Clock` with an offset per node. +- One `Clock` offset for each Silo. - Concurrent writes during a partition are blocked by the ETag — this is not a new invariant; it is step 4's invariant re-run under a new fault source. -Step 6c: probe failures and voting; the new invariant is "a healthy node is not killed by expired votes". +Step 6c adds probe failures and votes. An expired vote must not stop a healthy +Silo. ## Gap diff --git a/design/testing.md b/design/testing.md index cfd7413..6b7613a 100644 --- a/design/testing.md +++ b/design/testing.md @@ -19,7 +19,10 @@ Rule 4 is the easiest to violate and the most insidious in consequence, so [sche ## Two tracks -**Unit tests** (`make test`) — verify a single package: single-node runtime, mailbox, store, hash ring, generator. Requirements: each test under 50 ms, no network, no subprocesses. +**Unit tests** (`make test`) verify one package. A package test must not start a +network or child process. The `testcheck` release harness starts `go test` and +rejects a target with zero tests. Generator and external process tests use +separate targets. **Simulation tests** (`make sim`) — verify distributed invariants. Fake network + fake clock + fault injection + fixed seeds. Slow, on a separate target, out of the default `test`. @@ -39,19 +42,20 @@ This removes the biggest source of flakiness in tests, "waiting for an async pro ## How simulation tests are built -There is no usable off-the-shelf DST framework in Go; this must be stated clearly so later comers do not go looking: +The current dependencies do not provide the complete simulation harness. +`porcupine` checks linearizability but does not control scheduling. The `sim` +package combines it with the boundaries below. See +[simulation.md](simulation.md). -- **gosim** (`jellevandenhooff/gosim`) has the right shape (multi-machine, deterministic goroutine scheduling), but only 80 stars and **abandoned after 2024-12**. Cannot be relied on. -- **Antithesis** works; quoted 168k USD/year in 2025-09. Not in this project's cost structure. -- **porcupine** (`anishathalye/porcupine`, 1230 stars, active) is a linearizability checker: it verifies whether a history is linearizable, but does not control scheduling. Useful, but only one piece of the puzzle. +**Fake network** - implements the `Transport` interface. It uses a seed to +select delay, drop, and partition faults. A partition defines which Silo pairs +cannot communicate. Reorder is not a separate fault. A TCP connection keeps +byte order. Across connections, independent delays can change message order. +See [simulation.md](simulation.md). -Resonate's public conclusion is the same: goroutine scheduling cannot be controlled in Go, so DST must invasively constrain the whole codebase. - -So the `sim` package builds its own. Below are the pieces it needs; how they are built in [simulation.md](simulation.md): - -**Fake network** — implements the `Transport` interface. It can decide message delay, dropping, and partitioning by seed. A partition is a set of "which node pairs cannot talk"; it can change at any time. Reorder is not a separate knob: within a connection replies are matched by correlation id and a single TCP stream is ordered, and across connections two independent delays already produce every cross-message reorder. See [simulation.md](simulation.md). - -**Fake clock** — uses `synctest`'s time inside the bubble; cross-node time offsets come from a per-node offset in the `Clock` implementation, for testing clock skew. +**Fake clock** - uses `synctest`'s time inside the bubble. Cross-Silo time +offsets come from a per-Silo offset in the `Clock` implementation. This tests +clock skew. The fake clock's ticker **must drop ticks like a real `time.Ticker`**: when the receiver cannot keep up, ticks are dropped non-blockingly instead of waiting for it to read. A blocking send lets a stuck consumer drag down the hand that advances time — and the whole point of fault injection is to stick the consumer. This is the same line as [timers.md](timers.md)'s "missed windows are not made up": however many due times piled up, waking fires once. @@ -61,7 +65,13 @@ Dropping ticks imposes a hard requirement: **clock subscription must happen in t **Fake store** — an in-memory `Store` implementation that can inject write failures, slow responses, and "the write succeeded but the response was lost", the easiest case to get wrong. -**Fault injection** — node crash (drop all in-memory state, keep the store), node pause (simulating a long GC), network partition, partition recovery. +A State failure test must cover both sides of the Store result boundary. One +case fails before commit. One case commits and then loses the reply. Conflict, +cancellation, timeout, and a lost reply must all make the current Activation +unusable. The next Activation reads the record that the Store actually holds. + +**Fault injection** - Silo crash, Silo pause, network partition, and partition +recovery. A Silo crash drops memory but keeps the Store. **Invariant assertions** — checked after every step. The core ones: @@ -74,15 +84,102 @@ Dropping ticks imposes a hard requirement: **clock subscription must happen in t ## Forbidden -- No real external dependencies: real network, real processes, real databases, real filesystems (at the unit-test level). +- No real external dependencies at the unit-test level. - **The only exception is the embedded storage backend's own tests.** A SQLite backend cannot be verified to really persist data without touching disk. These tests use `t.TempDir()` and test only that one backend package. Every layer above — `runtime`, `gor`, `sim` — uses the in-memory `Store`. The exception stops here; do not let it spread. + **The only exception is the embedded storage backend's own tests.** A SQLite backend cannot be verified to really persist data without touching disk. These tests use `t.TempDir()` and test only that one backend package. Every layer above, including `internal/runtime`, `gor`, and `sim`, uses the in-memory `Store`. The exception stops here; do not let it spread. - No real time: no `time.Sleep` for synchronization, no `for now < deadline` polling for assertions. - No flakiness: no dependence on test execution order, no timestamps as seeds, no `t.Skip` to cover up intermittent failures. Treat flakiness as a bug to fix, not noise to tolerate. - No old and new tests side by side: once the migration is done, delete the old ones. ## Relationship with the ROADMAP -The DST skeleton is [step 4](../ROADMAP.md#4-deterministic-simulation-test-skeleton) of the ROADMAP, **before the cluster ([step 6](../ROADMAP.md#6-multiple-nodes))**. +The DST skeleton is [step 4](../ROADMAP.md#4-deterministic-simulation-test-skeleton) of the ROADMAP, **before the Cluster ([step 6](../ROADMAP.md#6-multiple-silos))**. The order cannot be swapped. Adding any of the four constraints above after the cluster is written means rewriting the cluster. This is the project's only "the order is non-negotiable" spot. + +## 0.1.0 release proof + +The release gate has three levels. + +### Behavior proof + +Each work batch adds focused tests for its contract. A critical state +transition also gets a targeted mutation check. The mutation must reach the +intended behavior assertion. A compile failure is not mutation proof. + +Fake-clock tests must use state or channel conditions. They must not use fixed +sleep calls, retries, or wider timing limits to become green. + +### Resource proof + +The final release tests Activation churn and a large Reminder backlog. Each +test has fixed input limits and fixed seeds. It checks goroutine cleanup, +bounded queues, bounded workers, and bounded Store pages. + +Use package-owned completion channels and resource state for goroutine leak +checks. Do not use a process-wide goroutine count for this proof. Other tests +and the Go runtime can change that count. + +Fuzz tests cover frame decode, Request Context decode, and generator marker +parse. Fuzz failures must save the smallest reproducer in the normal Go corpus. + +### Platform proof + +The 0.1.0 tested systems are: + +- Linux on amd64; +- macOS on arm64; +- Windows on amd64. + +Each system must compile all normal packages and run its supported test set. +Linux also runs the full `make ci` gate. Platform-specific test helpers must +use build-tagged files. A common test file must not import a system-only +`syscall` type. + +### Evidence rules + +Every release command must report an exit code and nonzero test work. A target +that selects zero tests is a failure. A retry does not replace the first failed +result. + +The Makefile must own the internal `testcheck` command. A caller-provided Make +variable must not replace this command. The `ci`, `external`, and +`external-tagged` entry points must run a release command contract test. The +test must compare each complete dry-run plan with and without a caller +override. The plans must be equal. Each plan must contain all required commands +in the specified order. A parallel Make option must not change this order. + +The final head must pass: + +- `make ci`; +- generated-file and `go mod tidy` diff checks; +- the three platform jobs; +- focused fuzz runs; +- targeted mutation checks; +- the bounded churn and Reminder backlog tests; +- a clean external-module build and restart run. + +`make external` is the candidate release process lane. It uses empty Go caches, +a local replacement, and real child processes. `make external-tagged` is the +published-module lane. It is fixed to the exact release version and must not +use a replacement. Both lanes must also run the upgrade proof when the release +changes generated artifacts or persistence. The old process must resolve from +the exact latest public tag without a replacement. It must write the database +that the new process opens. These commands are not part of `make test` or +`make ci`. + +Proof from an earlier commit is stale after a behavior change. + +## Gap + +The local `make ci` gate covers unit, race, simulation, generator, network, +resource, fuzz, vet, and static analysis tests. It rejects a zero-test target. +It also checks generated files and module tidy state without changing them. +The release command contract test prevents a caller-provided Make variable +from replacing the internal test command. + +The CI workflow has jobs for Linux amd64, macOS arm64, and Windows amd64. The +hosted jobs have not run on this change. `make external` passes locally on +Linux with empty Go caches and separate restart processes. Hosted external +proof and the final same-commit proof remain open. Batch 8 in +[release-0.1.0.md](release-0.1.0.md) owns these gaps. diff --git a/design/timers.md b/design/timers.md index 647d895..fe4e2e4 100644 --- a/design/timers.md +++ b/design/timers.md @@ -1,4 +1,89 @@ -# Reminders +# Grain Timers and Reminders + +Grain Timers and Reminders serve different needs. A Grain Timer belongs to one +Activation. A Reminder belongs to one GrainId and survives deactivation and +restart. + +## Grain Timers + +The public API is small: + +```go +type GrainTimer interface { + Change(dueTime time.Duration, period time.Duration) error + Stop() +} + +type GrainTimerOptions struct { + DueTime time.Duration + Period time.Duration + KeepAlive bool +} + +func RegisterGrainTimer( + grainContext *GrainContext, + callback func(context.Context) error, + options GrainTimerOptions, +) (GrainTimer, error) +``` + +A zero due time schedules the first tick now. A negative due time is invalid. +A zero period makes a one-shot timer. A positive period makes a repeating +timer. A negative period is invalid. + +Each tick enters the Activation mailbox as a local turn. It cannot overlap a +Call, another timer turn, or itself. The next period starts after the callback +finishes. A slow callback does not create a tick backlog. + +`Change` resets the next due time and period. If the callback is running, the +change takes effect after that callback. `Stop` is safe to repeat. It prevents +new ticks but does not cancel a callback that already runs. It also discards a +tick that waits in the mailbox. `Change` after `Stop` returns +`ErrGrainTimerStopped`. + +A Grain Timer uses a Runtime-owned callback context. It does not copy a Call +deadline, Call cancellation, or Request Context. `Stop` does not cancel this +context while a callback runs. Deactivation and Runtime shutdown cancel it. +They discard all timer handles for that Activation. A queued tick for an old +Activation cannot move to a new Activation. `Change` after deactivation returns +`ErrGrainTimerStopped`. + +The callback context marks the current Grain as occupied. A callback that calls +the same Grain through a Grain Reference gets `ErrCallCycle`; it does not wait +on its own mailbox turn. + +A timer does not keep the Activation active by default. If `KeepAlive` is +true, a completed tick updates the Activation's idle-use time. `KeepAlive` is +not a permanent hold. A timer whose period is longer than the idle timeout can +still end with its Activation. Deactivate on Idle also ends a keep-alive timer. + +A callback error goes to `OnError` and does not stop a repeating timer. A +callback panic also faults the Activation. The Runtime stops all timers during +fault cleanup. If Runtime cancellation makes the callback return +`context.Canceled`, the Runtime does not report that result to `OnError`. A +cancellation-shaped error from a live callback is still an Application error +and is reported. + +Each Grain Timer has one owner goroutine. The goroutine owns the timer state +and uses the injected `Clock` for one wake-up. It sends each due callback to +the Activation mailbox. It sets the next wake-up only after that callback +finishes. The `internal/timer` package does not own Grain Timers. It only owns +the persisted Reminder poller. + +After a tick enters the mailbox queue, the owner waits for the mailbox to say +that the callback ran or was discarded. Normal deactivation waits for a running +callback and then stops the owner before `OnDeactivate`. Abrupt shutdown stops +the owner and Runtime infrastructure without waiting for user callback code +that ignores cancellation. If Activation setup fails, the Runtime discards its +queued Timer turns without waiting on the Factory turn that reported the +failure. Deactivate on Idle also ends this wait after the requesting turn ends. +This lets a queued Call create the next Activation before the mailbox discards +an old Timer turn behind that Call. + +0.1.0 has no interleaving option. Grain Timer turns follow the same +non-reentrant rule as Calls. + +## Reminders One table plus one poller. The poller finds rows that have come due and delivers an ordinary call to the target Grain. @@ -12,10 +97,10 @@ type account struct { reminder gor.Reminder[Account] } -func newAccount(b *gor.Binder) *account { +func newAccount(g *gor.GrainContext) *account { return &account{ - balance: gor.NewState[int64](b, "balance"), - reminder: gor.NewReminder[Account](b), + balance: gor.NewState[int64](g, "balance"), + reminder: gor.NewReminder[Account](g), } } @@ -33,7 +118,7 @@ func (a *account) ApplyInterest(ctx context.Context, tick gor.TickStatus) error ```go type Reminder[T any] struct { /* bound to one GrainId */ } -func NewReminder[T any](b *Binder) Reminder[T] +func NewReminder[T any](g *GrainContext) Reminder[T] type ReminderTime struct { /* first delay and period */ } @@ -41,6 +126,7 @@ func After(delay time.Duration) ReminderTime func Every(period time.Duration) ReminderTime type TickStatus struct { + ReminderName string FirstTickTime time.Time Period time.Duration CurrentTickTime time.Time @@ -58,38 +144,58 @@ The public Reminder API is `Reminder[T]`, `ReminderTime`, `NewReminder`, `func(T, context.Context, TickStatus) error`. `After` creates a one-shot Reminder. `Every` creates a periodic Reminder. +`TickStatus.ReminderName` is the persisted Reminder name that caused the +Call. It supports one typed method that handles many dynamic Reminder names. +The Runtime reads the name from the claimed row. It does not ask the +Application to rebuild the name from time or State. This carries the same +input that Orleans gives to `ReceiveReminder` while keeping gor's typed method +handle. + `Account.ApplyInterest` is a Go method expression. The compiler checks that `ApplyInterest` is a method of `Account` and that its signature is `func(Account, context.Context, TickStatus) error`. A typo, rename, or signature drift is a compile error. The type parameter ties the handle to the Grain's interface. -The name the table stores is read off the method expression once, when `Handle` is called, with `reflect` and `runtime.FuncForPC`. The format of that name is not a Go-documented contract — it is empirically stable across the Go versions in use, but Go is free to change it. The implementation must therefore carry a unit test that locks the map from an interface method expression to its trailing-segment method name, so a Go upgrade that changes the encoding breaks the test instead of silently mis-naming Reminders. That runs at scheduling setup, never on the delivery path; what the poller reads at delivery is the same `method` column as before. `Handle` takes a method expression on the Grain interface — a hand-written closure of the same function type also compiles, but the name read off it is not a real method name and delivery fails with "unknown method". The contract is stated, not guarded: the type signature admits only `func(T, context.Context, TickStatus) error`, and the rest is the caller using the documented form. +`Handle` reads the method name once from the method expression. `Set` must ask +the installed generated call table to validate that name before Store I/O. A +closure or unknown method fails at Set. It cannot create a Reminder row that +will fail only after restart. The called method must be in the Grain's interface. The generated dispatch table must accept the `TickStatus` value at delivery time. The generator must also expose a typed `newReminderCall(method, TickStatus)` -factory for each Grain interface. The factory creates the normal typed request +factory for each Grain interface. The timer passes the persisted Reminder name +and tick times to this factory. The factory creates the normal typed request and reply values for the Reminder method. The timer passes those values to `Runtime.Invoke`, so local and forwarded Calls use the same path. The timer must not use reflection. The public API must not expose a string dispatch path; the stored method name is only an internal identifier used by generated code. -**Why a method expression, not a generated handle value.** Reminders are set from inside Grain methods, which live in the Grain package. The Grain package cannot import the package generated from its own interfaces: that package imports the Grain package for the interface types used in its proxies and dispatch, so the import is a cycle — the same reason generated artifacts land in their own package that the Grain package does not import (see [codegen.md](codegen.md)). A per-method handle symbol emitted by the generator therefore cannot be named from the code that sets a reminder. The method expression is the only compile-time-checked way to name a method from that code using just the `gor` package and the interface declared in the Grain package, so the handle carries no generated symbol. The generator changes nothing for this. +**Why a method expression, not a generated handle value.** The Grain package +cannot import its generated subpackage. That import would create a cycle. A +method expression gives a compile-time checked method without that import. -"One unified entry point" is rejected. Orleans has Grains implement `ReceiveReminder(name)` and switch on the name themselves — that is bringing back the hand-written dispatcher deleted at [step 3](../ROADMAP.md#3-typed-proxy-code-generation), and in user code of all places. gor's selling point is compile-time typing; it must not open a string-dispatch loophole here. +One string entry point is rejected. It would restore manual dispatch in +Application code. Step 3 removed that path with typed Grain References. See +[step 3](../ROADMAP.md#3-typed-grain-reference-generation). ## The stored identifier is the method name The `method` column holds the method name read off the handle — exactly the string the old string API held. A typed handle changes how the name is authored, not what is stored; the table, the poller, and cross-restart recovery do not change. A method rename invalidates existing Reminder rows: the stored name no longer -matches a dispatch case, and delivery returns "unknown method". The -identifier follows the method name. A rename is a breaking change to -Reminders. +matches a dispatch case. The identifier follows the method name. A rename is +a breaking change to Reminders. + +The hazard is bounded. A Grain can set its Reminders again on activation or +first use. `Set` then overwrites the row by name and stores the new method. -The hazard is bounded. Where a Grain re-asserts its Reminders on activation or first use, the rename self-heals on the next activation: `Set` overwrites the row by name, including its `method`, so the new name replaces the old. A one-shot Reminder waiting in the table across a rename is the real casualty — it fails once at delivery, the error reaches the configured sink, and the user sets it again. +If the old row becomes due first, it is an Invalid Reminder. The poller gives +it a Terminal Result. The poller uses the row ETag to remove the unchanged +row. It reports one dispatch failure only after that CAS succeeds. The +Application must set the Reminder again when it wants a new delivery. ### Migration @@ -99,9 +205,9 @@ the constructor is `NewReminder[T]`. Each `Set` call uses stored method name and table shape do not change, so existing rows survive a restart. Only source code needs migration. -## The handle comes from the Binder +## The handle comes from Grain Context -Like `State`, `gor.NewReminder[Account](b)` binds the GrainId and storage at +Like `State`, `gor.NewReminder[Account](g)` binds the GrainId and storage at Grain construction. The methods use it directly afterwards. Not fished out of `ctx`. Hiding runtime capabilities in `context.Value` makes "what this code needs" invisible and forces tests to build the right ctx before they can run. Constructor parameters are explicit; ctx is not. @@ -120,10 +226,15 @@ with a given name. The replacement resets `FirstTickTime` to the new first due time. It also resets `DueAt` to that same first due time. `Cancel(ctx, name)` deletes the Reminder. -A one-shot Reminder uses `Period = 0` in its `TickStatus`. A periodic -Reminder keeps its `FirstTickTime` when the poller claims it. The claim reports -the claimed old `DueAt` as `CurrentTickTime`. The poller computes the next -`DueAt` strictly in the future. It does not catch up missed periods. +`Set` rejects an empty name, a negative due time, a negative period, and an +invalid method handle before Store I/O. `Cancel` rejects an empty name before +Store I/O. `Every(0)` is a one-shot Reminder. + +A delivery uses the row name as `ReminderName`. A one-shot Reminder uses +`Period = 0` in its `TickStatus`. A periodic Reminder +keeps its `FirstTickTime` when the poller claims it. `CurrentTickTime` is the +time when delivery starts, from the injected `Clock`. The poller computes the +next `DueAt` strictly in the future. It does not catch up missed periods. **Missed windows are not made up.** If the process is down for three periods, it fires once on return and then tracks to the next future time. Making up @@ -150,38 +261,93 @@ several Reminders with different periods. ## ReminderStore -`ReminderStore` has four operations, aligned with what the poller and the user +`ReminderStore` has five operations, aligned with what the poller and the user need to do: -- **List due** — rows with `due_at <= now`; `now` comes in as a parameter. -- **Claim one row** — CAS with the row's etag, pushing `due_at` to the given next time; a zero time deletes the row. Exactly one claiming node wins. +- **Check**: verify Store readiness without changing Reminder data. +- **List due**: return at most `limit` rows after a cursor with `due_at <= now`. +- **Claim one row** - use CAS with the row's ETag. Set `due_at` to the next + time. A zero time deletes the row. Exactly one Silo wins the Claim. - **Write one row** — unconditional overwrite; the user's `Set` goes here. - **Delete one row** — unconditional; the user's `Cancel` goes here. +```go +type ReminderCursor struct { + DueAt time.Time + GrainId GrainId + Name string +} + +type ReminderPage struct { + Rows []Reminder + Next *ReminderCursor +} + +type ReminderStore interface { + Check(ctx context.Context) error + ListDue(ctx context.Context, now time.Time, after *ReminderCursor, limit int) (ReminderPage, error) + Claim(ctx context.Context, reminder Reminder, nextDueAt time.Time) (bool, error) + Put(ctx context.Context, reminder Reminder) error + Delete(ctx context.Context, id GrainId, name string) error +} +``` + +`Check` returns nil only when the Reminder Store can serve the configured +Reminder data. It honors context cancellation and does not change a Reminder +row. The memory implementation returns `ctx.Err()` when the context is done +and nil otherwise. The SQLite implementation checks its connection and the +required schema. Both built-in implementations must pass the same contract +tests. + `ReminderStore` persists `FirstTickTime` with each row and returns it to the poller when it lists a due row. -**The etag exists only for claiming.** The user's `Set` / `Cancel` carries no etag: the user does not have one anyway, and an explicit reschedule or cancel is his to win. The claim that got overwritten simply delivered one fewer time; at-most-once still holds. +SQLite stores one `schedule_version` row. It uses this value to assign a newer +ETag to each Put and each Claim that advances a row. The update and the row +change use one local SQLite transaction. Open sets the version to at least the +largest ETag in an old Reminder table. + +**The ETag exists only for claiming.** The user's `Set` and `Cancel` carry no +ETag. An explicit Set or Cancel is the Application change that must win. +Each Put gets an ETag that is newer than all prior Reminder ETags in that +Store. A Store must not reuse an ETag after Delete. Thus, an old Claim cannot +match a new setting with the same GrainId and name. + +An Invalid Reminder uses a zero-time Claim. A successful Claim removes the +unchanged row. A stale Claim means that Set or Cancel changed the row. The +poller must not report the old dispatch failure in that case. **The next due time is computed by the poller, not the table.** "No catch-up for missed" is policy; the table is only responsible for getting the CAS right. One-shot Reminders use the zero time for "no next" — the same convention as a zero `interval`. -No row-count limit on "list due". Add it when it is actually needed; adding it now decides for a scale that does not exist yet. +`limit` must be positive. Rows use `due_at`, GrainType, GrainKey, and name +order. `Next` is nil after the last page. The cursor lets one scan move +past an invalid row without claiming it. SQLite must have a full index that +starts with `due_at`. A partial index does not meet this contract. + +`Next` identifies the last row in its page. It must move forward from the +prior cursor. The poller reports `ReminderScan` and stops the current scan when +a Store breaks this rule. + +The default page limit is 256 rows. The default worker limit is 16. +`WithReminderPageSize` and `WithReminderWorkers` select other positive values. ## Claim first, then deliver The poller scans rows with `due_at <= now` and, for each row: -1. **Claim** — CAS to push `due_at` to the next period (delete the row for +1. Validate the installed GrainType and generated Reminder method. +2. If validation fails, acquire one worker slot and Claim the row with a zero + next time. Report the dispatch failure only when this terminal CAS succeeds. +3. For a valid row, acquire one worker slot. +4. **Claim**: use CAS to push `due_at` to the next period (delete the row for one-shot Reminders). -2. Build the typed request with the generated `newReminderCall` factory. -3. Deliver the ordinary Call through `Runtime.Invoke`, only after winning the +5. Deliver the ordinary Call through `Runtime.Invoke`, only after winning the claim. -The periodic claim keeps `FirstTickTime` and reports the claimed old `DueAt` as -`CurrentTickTime`. It pushes `due_at` to the first time strictly in the future, -not to `due_at + interval`. After three periods of downtime, adding one -interval still lands in the past; the next scan hits the same row again, and -"no catch-up" becomes catch-up. +The periodic claim keeps `FirstTickTime`. It pushes `due_at` to the first time +strictly in the future, not to `due_at + interval`. After three periods of +downtime, adding one interval still lands in the past. The next scan would hit +the same row again and turn no catch-up into catch-up. The reverse order causes repeated firing on a crash. A failure after the claim can miss one delivery. This is a deliberate trade-off: @@ -208,11 +374,41 @@ type ErrorSource interface { } type ReminderInvocation struct { - Method string + Name string + Method string + TickStatus TickStatus } func (ReminderInvocation) errorSource() {} +type GrainTimerInvocation struct{} + +func (GrainTimerInvocation) errorSource() {} + +type ReminderScan struct{} + +func (ReminderScan) errorSource() {} + +type ReminderClaim struct { + Name string +} + +func (ReminderClaim) errorSource() {} + +type ReminderDispatch struct { + Name string + Method string +} + +func (ReminderDispatch) errorSource() {} + +type ReminderTerminal struct { + Name string + Method string +} + +func (ReminderTerminal) errorSource() {} + type Deactivation struct { Reason DeactivationReason } @@ -223,39 +419,58 @@ func OnError(func(BackgroundError)) Option ``` `ErrorSource`'s unexported method seals the set inside the `gor` package. Code -outside the package cannot implement new sources. Claimed Reminder deliveries -use `ReminderInvocation{Method: ...}`. Deactivation hook failures use -`Deactivation{Reason: ...}`. Callers branch on the source type, not on a -writable string. +outside the package cannot implement new sources. Callers branch on the source +type, not on a writable string. + +**One sink, not one per timer or event kind.** It reports all background +failures that need an operator decision. These include callback, scan, +dispatch, terminal, and claim errors. A lost CAS is normal contention and does +not enter the sink. Direct Calls return errors to the caller. + +`Err` is the original callback or Store error. It follows +[errors.md](errors.md). Across a Call boundary, only the stable `Code` is usable +with `errors.Is`. -**One sink, not one per Reminder, not one per event kind.** It only reports the two kinds of application callback failures with no caller to receive them: claimed Reminder deliveries and normal deactivation hooks. Direct calls return errors to the caller as usual. Polling scans, claim failures, and losing the CAS do not enter the sink — they are operational states of the scheduler and storage, not failures of a known application action. +It does no retry, no backoff, and no alerting policy. The event carries the +Reminder name and TickStatus because they identify the failed Application +callback. It does not carry an ETag or attempt count because the Runtime has no +retry model. -`Err` is exactly the error the callback got. It follows [errors.md](errors.md): across nodes, only the stable `Code` is usable with `errors.Is`; the event does not add its own `Code` field, nor does it restore error types, fields, or wrapping. +A dispatch failure is different from a Call failure. The poller first removes +the Invalid Reminder with a zero-time Claim. Only the CAS winner reports +`ReminderDispatch`. Two pollers can resolve the same row, but only one reports +it. The removed Reminder does not return after restart. -It does no retry, no backoff, no alerting policy — those are the user's business; the runtime only delivers "this failed" into the user's hands. The event carries no reminder name, due time, interval, ETag, or attempt count: after claiming these fields may already be stale, the ETag is not an application decision, and the runtime has no retry model. `timer.Invoker` keeps receiving only GrainId and method. +If the terminal Claim returns a Store error, the poller reports +`ReminderTerminal`. It does not report `ReminderDispatch` because it does not +know if the Store removed the row. A later scan can try again when the row +remains due. The Store error can be an Unknown Result. -When unconfigured, these two kinds of errors are dropped. They are the only application callback errors the runtime drops on the user's behalf, and must be written in the docs, not hidden in the implementation. +A process can stop after the terminal CAS and before `OnError` runs. This can +lose the dispatch report. The Runtime does not retry the report. This is the +same claim-before-delivery limit as a valid Reminder. + +When unconfigured, these errors are dropped. This behavior must be in the +public docs. + +The Runtime catches an `OnError` panic. It does not call the sink again for +that panic, and it continues the cleanup that produced the original event. +The sink must return promptly and must not do blocking I/O. **A delivery canceled mid-shutdown is not a failure.** At runtime shutdown, in-flight Reminder Calls come back with a cancellation error — the method did not fail; the runtime stopped running. Sending it to `OnError` would report a false alarm to the user on every clean shutdown, and users would have to filter cancellation errors out in their own callbacks. So when the poller's context is already canceled, this error does not go out. This is not defensive special-casing; it is behavior a test must watch: cancellations during shutdown do not enter `OnError`; other callback errors matching the sink boundary above must enter. -**Claiming must be a CAS, not "read then update".** The pollers of two nodes can scan the same row at the same time; CAS is the only thing that makes one of them lose. This must be right now, not deferred until step 6 — by then, every earlier test would need rewriting. +**A Claim must use CAS.** The pollers of two Silos can scan the same row. CAS +makes exactly one Silo win. This rule is required before Cluster work. ### Migration -This is a planned v0 breaking change. Existing three-parameter error handlers -change to receive one `BackgroundError`. Reminder deliveries read -`ReminderInvocation.Method`; deactivation hook failures read -`Deactivation.Reason`. - -### Gap +This is a v0 source change. `ReminderInvocation` gains the Reminder +name and TickStatus. Grain Timer, scan, claim, and dispatch sources join the +sealed set. -The error sink is implemented in the current code. `OnError` receives a -`BackgroundError` with the Grain, original error, and source. The source set -is sealed. Reminder delivery failures and deactivation hook failures use the -sources described above; scan failures, claim failures, and shutdown -cancellations remain outside the sink by design. +### Cluster preview Cluster ownership checks and forwarding are implemented for the current optional cluster preview. Rolling upgrades and other operational cluster work @@ -263,13 +478,17 @@ remain deferred and are outside this design. ## Don't claim rows that are not yours -In a cluster, every node's poller scans the whole table, but a row's target Grain belongs to exactly one node. Before claiming a row, ask whether it is yours; if not, skip. +In a Cluster, every Silo poller scans the complete table. One Silo owns the +target Grain. A Silo must not Claim a row that it does not own. Not asking loses deliveries, not duplicates them: when a non-owner wins the claim, `due_at` has already been pushed (the row is deleted for one-shot Reminders), and then the call is rejected by routing — that due time is gone forever, and the owner's poller will not see it next round. -So the poller must ask one more thing: not just "call this method", but "do I own this GrainId". In a single node the answer is always yes — that is not a fake implementation; a single node indeed owns everything. +The poller must ask if its Silo owns the GrainId. In a Single Silo, the answer +is always yes because that Silo owns all Grains. -With an inconsistent view, two nodes may both believe they are the owner; CAS makes one lose, and at-most-once holds. If both believe it is not theirs, this due time is deferred until the view converges; at-most-once still holds. +With an inconsistent view, two Silos can both believe that they own a Grain. +CAS makes one Claim fail. If both Silos reject Ownership, delivery waits for +the view to converge. ## Coming due activates the Grain @@ -277,11 +496,15 @@ If the target Grain is not in memory, delivery activates it. This is the point o ## The poller -One goroutine, driven by the injected `Clock`. `Runtime` must make it exit whether `Close()` or `Kill()` is taken — in `sim`, a goroutine that does not exit makes the whole bubble judged deadlocked. +One coordinator goroutine uses the injected `Clock`. A fixed worker limit +bounds claim and delivery work. The coordinator must acquire worker capacity +before it claims a row. Shutdown must stop the coordinator and workers. The poller itself has only "running" and "stopped"; no state enum is invented for it: a two-state state machine is ceremony, not design. It does not distinguish draining from non-draining stops either — it carries no user state, and a delivery canceled halfway still falls within the at-most-once promise. -It gets its own package. `runtime` cannot host it — the poller reads tables, and `runtime` does not import `store`. `gor` should not host it either — that layer only assembles configuration; it does not hold algorithms. So the poller, like `mail`, is a small package: it takes a table interface, a `Clock`, and an interface for initiating calls; `gor` wires the three together. +It gets its own internal package. The execution runtime cannot host it because +the poller reads tables. The root package only assembles it. The poller takes a +table interface, a `Clock`, and an interface for initiating Calls. ## A new I/O interface @@ -316,19 +539,13 @@ The minimum failure, restart, and claim tests must cover these cases: - A failed claim does not deliver the Reminder and leaves the row available. - A process failure after a successful claim and before delivery may miss one delivery. - A failed Reminder method reaches `OnError`; the runtime does not retry it. +- Scan and claim errors reach `OnError`; a lost claim Conflict does not. - `Set` replaces by name, resets `FirstTickTime`, and resets the first due time. -- A periodic Reminder after downtime reports the old due time, keeps `FirstTickTime`, and does not replay missed periods. +- A periodic Reminder reports its actual delivery start, keeps `FirstTickTime`, and does not replay missed periods. - A one-shot Reminder reports `Period = 0`. - -## Gap - -The typed Reminder method handle, the public Reminder names, the -`first_tick_time` row field, and the generated typed Reminder-call factory are -implemented. The structured error sink is also implemented. The method name -is read from the expression once. The table, poller, and restart recovery use -the method-name string as an internal identifier. - -The remaining work is outside this single-node Reminder contract. The -optional cluster implementation is shipped as a preview; rolling upgrades -and operational cleanup remain deferred. The announced 0.1.0 release still -requires its conformance and failure-evidence work. +- An Invalid Reminder reports once after a terminal CAS. Later + polls and a process restart do not report the same row again. +- Two pollers that see one invalid row produce one dispatch report. +- A Set or Cancel that wins before the terminal CAS prevents the stale + dispatch report. +- A large due set does not exceed the row and worker limits. diff --git a/design/transport.md b/design/transport.md index 1f951b8..c06c42b 100644 --- a/design/transport.md +++ b/design/transport.md @@ -1,6 +1,7 @@ # Transport -Moves bytes between nodes. **This layer does not understand message semantics** — it does not know what a GrainId, a method, or a Grain is. +Moves bytes between Silos. **This layer does not understand Call semantics.** +It does not know what a GrainId, a method, or a Grain is. ## No gRPC @@ -22,9 +23,13 @@ type Transport interface { } ``` -An address is a string, not a node type from `cluster`. The transport does not import `cluster` — the dependency runs the other way. +An address is a string, not a `cluster.Node` value. Transport does not import +`cluster`. The dependency has the other direction. -**Binding is separated from serving.** The listen address is bound at construction, so `Addr()` can immediately return the actually bound address, and `Serve` is just the accept loop. A node must know its own address before writing its row into the membership table, and tests use `:0` to let the kernel pick a port — without the separation, this is impossible. +**Binding is separate from serving.** The constructor binds the listen +address. Thus, `Addr()` can return the bound address before `Serve` starts. +A Silo needs this address before it writes its membership row. Tests use `:0` +to let the operating system select a port. **Two stop methods, not one.** `Close` is a graceful stop and `Kill` is an abrupt stop. This matches the rest of the system — `Runtime`, `cluster.Node`, and the execution runtime all split `Close` from `Kill`, and a crash is not a `Close` ([simulation.md](simulation.md)). Serving both stops with one `Close` is what let an in-flight forwarded reply be truncated during an owner's graceful close: one method cannot mean two things. The split is named, not parameterized; a `Close(mode)` boolean or enum leaves which stop is in progress implicit at the call site and does not match the vocabulary every other component already uses. The runtime's graceful stop calls `Close`; its abrupt stop and a declared-death collapse call `Kill`. `Serve` returns when its context is canceled, when `Close` completes, or when `Kill` completes. What each stop owes an in-flight reply is defined in the Closing section below. @@ -34,7 +39,8 @@ An address is a string, not a node type from `cluster`. The transport does not i A request from A to B goes over the connection A dialed to B, and B's reply comes back over the same one. A request from B to A goes over the other connection, dialed by B. -Two nodes therefore have two TCP connections, not one. The cost is explicit; in exchange we drop arbitration logic like "can an inbound connection be used for outbound requests" — logic that only runs in the race where both sides dial at the same time, the classic "write ten lines, regret them for three years". +Two Silos have two TCP connections. This costs one more connection, but does +not need connection arbitration when both Silos dial at the same time. ## Frames @@ -54,11 +60,14 @@ Multiple requests fly on one connection at the same time, matched by correlation **Only one goroutine touches the pending table.** Each connection has three goroutines: reader, writer, owner. The owner holds the pending table and the next correlation id; its inputs are all channels — new requests, response frames read, canceled requests, a dead connection. -No mutex protects the pending table. This is not cleanliness for its own sake: blocking on a mutex is not durably blocking in `synctest`, and one mutex can keep the bubble from detecting quiescence. The same reason already appeared once with the activation placeholder in `runtime`; this is the second time. +No mutex protects the pending table. This is not cleanliness for its own sake: blocking on a mutex is not durably blocking in `synctest`, and one mutex can keep the bubble from detecting quiescence. The same reason already appeared once with the Activation placeholder in `internal/runtime`; this is the second time. `Send` hands the request together with a reply channel to the owner, then selects on the reply and `ctx.Done()`. -**Each server-side handler runs in its own goroutine; it must not run in the owner.** The owner only registers and hands over; running a handler to completion in the owner would block every other request on this connection behind it — the entire reason correlation ids exist is so requests do not wait on each other. The layer above can least afford this: `gor` packs calls of many Grains into one connection, and Grain calls are serial anyway, so one busy Grain would stall every other Grain from the same node. +**Each server-side handler runs in its own goroutine.** The connection owner +only registers and passes the request. A handler on the owner goroutine would +block all other requests on that connection. One busy Grain would then block +Calls to other Grains from the same Silo. Handlers also write responses back to the owner through a channel; only the owner ever lays out frames. @@ -80,7 +89,10 @@ When `Send`'s ctx expires, the request may already have finished executing on th Cancellation does exactly one thing: tells the owner to drop this pending entry. A response that really arrives later finds no registration and is thrown away. -**An error the caller gets does not mean the other side did not execute.** This is the same semantics as timeouts in `runtime`, and the same class of thing as "write failed but took effect". The layer above must not treat a transport error as "it did not happen". +**An error the caller gets does not mean the other side did not execute.** This +is the same rule as a timeout in `internal/runtime`, and the same class as a +write that took effect but returned an error. The layer above must not treat a +Transport error as proof that nothing happened. ## A dead connection is dead @@ -88,7 +100,8 @@ Lazy dialing: the connection is created only on the first send to an address. When the connection errors, the owner returns all pending requests with the error and closes the connection. **No reconnect loop, no backoff, no keepalive probes.** The next `Send` finds no connection and dials again; if dialing fails, it reports the error. -Deciding whether a node is really gone is the membership table's job ([cluster.md](cluster.md)). The transport having an opinion here would only produce two contradictory judgments. +The membership table decides if a Silo is dead. Transport must not make a +second decision. See [cluster.md](cluster.md). ## Encoding is not this layer's business @@ -114,4 +127,7 @@ Real-TCP close behavior is covered under `make net`: an in-flight `Send` followe The interface specified above is implemented. `Transport` exposes `Close` (graceful) and `Kill` (abrupt): a graceful close stops accepting new requests, joins in-flight handlers through their completion signals on the owner channel, flushes every queued reply frame, and only then closes the socket; a Kill cancels in-flight handlers, drops replies not yet written, and closes sockets, and it escalates a graceful close still in progress. The runtime's graceful stop calls `Close`; its abrupt stop and a declared-death collapse call `Kill`, including when the Kill escalates a close already under way. The in-memory fakes (`testTransport`, `simulationTransport`) implement both stops, and the graceful-close flush invariant is verified deterministically with a blocking handler under `synctest`. -Under the `sim` build tag, `simulationTransport` can drop individual messages by a seed-drawn decision, drop a whole node pair across a partition, and delay messages for a seeded number of fake-clock ticks. Reorder is not a distinct fault under this transport model (see [simulation.md](simulation.md)). +Under the `sim` build tag, `simulationTransport` can drop one request, drop all +traffic for one Silo pair, or delay traffic for fake-clock ticks. The seed +selects each fault. Reorder is not a separate fault. See +[simulation.md](simulation.md). diff --git a/docs/README.md b/docs/README.md index ca24808..0ef318f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,7 +18,8 @@ When a document and the implementation diverge significantly, the document gets - [writing-style.md](writing-style.md) — the English and ASD-STE100 writing rule for repository documentation. - [vision.md](vision.md) — product direction, core promises, and boundaries. - [programming-model.md](programming-model.md) — the programming model and API shape. -- [errors.md](errors.md) — call errors, stable error codes, cancellation, and the cross-node boundary. +- [errors.md](errors.md) - Call errors, stable codes, cancellation, and the + Cross-Silo boundary. - [example.md](example.md) — the device-shadow example puts Grains, State, Reminders, and cross-Grain Calls on one runnable usage path. - [compatibility.md](compatibility.md) — v0 and v1 compatibility promises to users, upgrade boundaries, and known limits. diff --git a/docs/compatibility.md b/docs/compatibility.md index 2673b29..d6f92d9 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -35,16 +35,18 @@ Production should pin to a specific v0 minor version. Moving to the next minor v ## What can already be relied on -v0's usable scope is single-process. For the published single-process capabilities, users can rely on these basic semantics: +The stable v0 scope is one Silo. Users can rely on these basic rules: - Calls for the same GrainId execute in order. -- state confirmed successfully survives a process restart. +- Confirmed State survives a process restart. - documented Reminder delivery, overload, and failure outcomes behave as described. These are product promises, not promises about implementation shape, throughput numbers, or exact execution instants. -Multi-node is a preview capability. Failure detection is based on direct probing of neighbors and death votes with expiry, but a network partition can mistake healthy nodes for failed ones, even stopping every node from serving; recovery needs a new generation. So during v0, multi-node availability, failure-detection accuracy, and upgrade experience are not stable guarantees. Related behavior may be adjusted or withdrawn in a new v0 minor version. +Cluster support is a preview. A network partition can make healthy Silos appear +dead. Recovery can need a new membership generation. Cluster availability, +failure detection, and upgrades are not stable v0 promises. The application remains responsible for evolving its own business state. gor does not understand business fields and will not convert old business data into new for the application. When the application changes method contracts or state meaning, it arranges compatible reads/writes or a downtime switch itself. @@ -60,9 +62,12 @@ With no breaking changes, the section says "None". The version number lets users ## This is not application rolling upgrades -This compatibility is a contract between gor and its users. It does not change the constraints on applications upgrading within one cluster. +This compatibility is a contract between gor and its users. It does not change +Application upgrade rules inside one Cluster. -An application's nodes must still use mutually compatible method contracts and state formats. Incompatible application changes need a downtime release, or compatible reads/writes arranged by the application itself. This constraint is in [programming-model.md](programming-model.md); it is a different matter from gor itself moving from one version to another. +An Application's Silos need compatible method contracts and State formats. +Incompatible changes need downtime or compatible reads and writes. See +[programming-model.md](programming-model.md). ## What v1 means @@ -76,4 +81,6 @@ If this v1 promise must ever be broken, gor will release a new major version wit ## Gap -The pre-announcement checklist is complete, but the first announced release also requires the 0.1.0 product contract's composition and failure-evidence gates. The broader v0 discipline and assembled release notes start at 0.1.0. The README presents single-process as the usable scope and describes multi-node failure detection as direct probing with death voting, while the reliability limits are stated in this document's "What can already be relied on" section. +The pre-announcement checklist is complete. The 0.1.0 composition, failure, +platform, and release evidence gates remain open. The v0 rules start with the +first announced release. diff --git a/docs/errors.md b/docs/errors.md index a79e5b8..49fe2d4 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -3,7 +3,11 @@ A Grain Call may complete on this Silo or on another Silo. Callers can rely on the same outcome rules. Location transparency guarantees exactly that. -Location transparency does not preserve in-process error objects. An error's concrete type, fields, wrapping, and implementation details do not become a contract because the call crosses nodes. Calls do not promise exactly-once execution either. +Cross-Silo behavior is part of the Cluster preview. It is not part of the +0.1.0 product contract. + +Location transparency does not preserve in-process error objects. Concrete +types, fields, and wrapping do not become a Cross-Silo contract. ## Stable error codes @@ -25,24 +29,35 @@ gor's own codes are a closed set. The library only uses the published `gor.*` co ## What the caller gets -| Outcome | Local call | Cross-node call | +| Outcome | Local Call | Cross-Silo Call | | --- | --- | --- | | Error with a stable code | The original error matches the code. | A new error matches the same code. Text may differ. | | Error with no determinate code | The original error object is returned as-is. | Only displayable text remains. | -| Caller cancels or times out | The caller gets its own cancellation or timeout error. | Same. The remote side may still be executing. | -| Send, connect, or reply-receive failure | This is a delivery failure. | Same. It cannot prove the remote side did not execute. | +| Caller cancels or times out | The caller gets its own cancellation or timeout error. | Same. The target Silo can still run the Call. | +| Send, connect, or reply-receive failure | This is a delivery failure. | Same. It cannot prove that the target Silo did not run the Call. | + +This parity applies only to one determinate stable code. The complete error +tree must contain exactly one unique code. No code means that no determinate +code exists. Several different codes also mean that no determinate code +exists. Merged errors use the same rule. -This parity applies only to an error's determinate stable code. An error has at most one determinate code: the only code appearing in its error tree. An error with a determinate code matches by that code both locally and across nodes. An error with no determinate code — the whole tree has none, or has several different ones — leaves only text across nodes. Merged errors follow the same rule: if the merge leaves exactly one code, it counts; several different codes are ambiguous and count as none. Parity does not promise that arbitrary sentinels, concrete types, or `errors.As` behave identically on both sides. +An error with a determinate code matches that code across Silos. An error with +no determinate code keeps only text. `errors.As` and concrete types need not +match across Silos. -An error with no determinate code can still be displayed, logged, and returned upward across nodes. Callers must not branch on its text, type, fields, or wrapping, and must not infer business state from it. +An uncoded error can still be displayed and logged across Silos. Callers must +not use its text, type, fields, or wrapping for decisions. ## Cancellation Caller cancellation or timeout means the caller is no longer waiting. It does not mean the business action did not happen. -A local call hands the caller's cancellation to the method. A cross-node call carries neither cancellation nor deadline. If the caller's cancellation happens first, the caller immediately gets its own `ctx.Err()`; a method already delivered to the remote side keeps using the remote side's own execution context. The remote side can complete, change state, and produce a result; the result is discarded at the source. +A local Call gives the caller's cancellation to the method. A Cross-Silo Call +does not send the cancellation or deadline. The target method can still finish +and change State. -This boundary keeps the caller from knowing whether the request was delivered. To avoid duplicate actions or to compensate, applications must put idempotency keys, state machines, or compensation rules into the business protocol. +This boundary gives the caller an Unknown Result. The Application must define +a Safe Repeat or a recovery rule for the Business Action. ## Reply that cannot be encoded @@ -52,14 +67,17 @@ When a method succeeds but its return value cannot be encoded, the caller gets ` ## This version's boundary -This version does not register arbitrary error types, does not restore error fields, does not preserve error chains or merge structure, and adds no codegen annotation for error codes. Applications that need business data across nodes should put it in normal return values or persistent state, not smuggle it through error objects. +This version does not restore arbitrary error types or fields. Applications +must put Cross-Silo business data in results or persistent State. ## Migration -Application code that branches on sentinels should replace the sentinel with a declared `gor.Code` and keep using `errors.Is`. For example, the device-shadow HTTP handler's HTTP 400 check should test `shadow.workshop_id_required` instead of an error object only recognizable in-process. +Application code can replace a sentinel with a declared `gor.Code`. It can +then keep using `errors.Is` across Silos. Simulators and tests must classify only by stable codes or the caller's own cancellation errors. Errors without a determinate code must be reported as unclassified, not categorized by guessing at text. ## Gap -Stable codes, cross-node `errors.Is` parity, reply-encoding priority, and the shadow and simulator migrations are implemented. Still not provided: arbitrary error type recovery, error field or chain fidelity, `errors.Join` structure fidelity, cancellation-frame or remote-deadline propagation. These are explicitly out of scope for this version. +The Single Silo error contract is implemented. The Cluster preview preserves +determinate stable codes across Silos. Cancellation remains local to each Silo. diff --git a/docs/example.md b/docs/example.md index 3e2fa97..f3876e2 100644 --- a/docs/example.md +++ b/docs/example.md @@ -1,24 +1,27 @@ # Example application -> Implemented. The runnable device-shadow service is in [../examples/shadow/](../examples/shadow/); this document explains the programming model and boundaries it demonstrates. The same service runs on one node or on several; the business code does not change between them. +> Implemented. The [device shadow service](../examples/shadow/) runs in one +> Silo. Its Cluster mode is a preview and is not part of 0.1.0. ## Why not a counter -A counter demonstrates gor's three concepts in ten lines. The problem: a counter does not show why you would use `gor` — a map plus a lock can do a counter too. +A counter can show the API. It cannot show the value of persistent Grains. -The example app answers a different question: when would you be glad you used this? +The device shadow example shows the complete programming model. ## The story -**Device shadow.** A large fleet of devices out in the field; each reports state periodically; the server must answer "what is this device like right now" at any time, and mark a device offline when it goes quiet. +**Device shadow.** Each device reports State. The service reads its current +State and marks a quiet device offline. -Chosen because it stresses four things at once — exactly the four reasons `gor` exists: +The example uses four important capabilities: **Large count, mostly idle.** A hundred thousand devices, but only a few hundred speaking at any moment. `gor` keeps each Device Grain active while it speaks and keeps its State in the store when it is idle. -**Concurrent writes to one device must be serialized.** The device reports state while operations pushes configuration. When the two collide, without serialization you write a pile of optimistic-lock retries. In `gor` this is the default; users do nothing. +**Calls for one device need serialization.** A report and a configuration +change can arrive together. The Device Grain serializes them. **Offline detection is naturally a Reminder.** "No report for thirty seconds means offline" needs an alarm that follows the Device Grain and survives a @@ -37,14 +40,17 @@ In this order, someone new to `gor` should be able to: boundaries go. 2. Know how state is stored, when it is persisted, and what a write conflict does. 3. Attach a Reminder and know it survives a process restart. -4. Know what state the world is in after a call fails — the point examples most easily gloss over. +4. Understand the Unknown Result after a Call fails. -Point 4 must be written seriously. **No ignored errors in the example.** Every failure is either handled, or a comment explains why it can be ignored here. Examples are copied; copying a `_ = err` copies a bug. +The example must handle each error. A comment must explain any error that is +safe to ignore. ## Boundaries -**The example must not modify the library.** When writing it surfaces an awkward API, a missing capability, or a doc that cannot say what it means — that is the example's most valuable output: record it. Do not work around it in the example, and do not change the library to accommodate it. +**The example must not hide product gaps.** A missing capability must change +the product plan or appear in a `Gap` section. -**No web framework, ORM, or config library.** The standard library is enough. The example should teach `gor` usage, not someone else's. +**Use no web framework, ORM, or configuration library.** The standard library +keeps the example focused on `gor`. **No frontend.** An HTTP interface you can hit with `curl` is enough. diff --git a/docs/programming-model.md b/docs/programming-model.md index f956a23..d822fe0 100644 --- a/docs/programming-model.md +++ b/docs/programming-model.md @@ -15,7 +15,8 @@ serially. the same Grain. No create or delete call is needed. The Grain starts at its first Call, may leave memory after idle time, and keeps State in the store. -**Call** — calling a method through an interface. The caller does not know and does not care whether the target is in this process or on another node. +**Call** - a method invocation through a Grain Reference. The caller does not +manage the target Activation. ## Declaring a Grain @@ -29,13 +30,32 @@ type Account interface { } ``` -The first parameter of an interface method must be `context.Context`; the last return value must be `error`. Parameters and return values in between are free-form. A method that does not comply is a generation-time error that names the line. +The Grain interface and its methods must be exported. The interface cannot +have type parameters, and a method cannot be variadic. The first parameter of +each method must be `context.Context`. The last result must be `error`. Every +contract type must be accessible from the generated subpackage. An invalid +contract is a generation error that names the source line. ### Generation prerequisite The `//gor:grain` marker says this interface gets typed Calls generated for -it. Add the generator to your module once. Run it when a marked interface -changes, before you build. See [../design/codegen.md](../design/codegen.md). +it. Add gor and its generator to your module once: + +```bash +go get github.com/suraciii/gor +go get -tool github.com/suraciii/gor/cmd/gorgen +``` + +Keep one command in the Grain package: + +```go +//go:generate go tool gorgen -pkg . +``` + +Run `go generate ./...` after a marked interface changes. CI can run +`go tool gorgen -pkg ./path/to/grains -check` to reject missing or stale +output. The default output is the `gorgen` subpackage under the Grain package. +See [../design/codegen.md](../design/codegen.md). Every Runtime must install the generated output at startup before Grains can be registered or Grain References can be obtained. @@ -69,15 +89,17 @@ that package. ```go func Register(rt *gor.Runtime) error { - return gor.Register[Account](rt, func(b *gor.Binder) Account { - return &account{balance: gor.NewState[int64](b, "balance")} + return gor.Register[Account](rt, func(g *gor.GrainContext) Account { + return &account{balance: gor.NewState[int64](g, "balance")} }) } ``` -`b` is handed in by the runtime; it connects the state cells to the store. Apart from that, the factory is an ordinary constructor. +`g` is given by the Runtime. It connects State cells to the Store. Apart from +that, the factory is an ordinary constructor. -No locks in method bodies, because none are needed — a second call on the same key is never running at the same time. +Method bodies need no lock for Call serialization. A second Call for the same +Grain does not run at the same time. ## The Grain knows its GrainId @@ -90,10 +112,10 @@ type account struct { } func Register(rt *gor.Runtime) error { - return gor.Register[Account](rt, func(b *gor.Binder) Account { + return gor.Register[Account](rt, func(g *gor.GrainContext) Account { return &account{ - id: gor.Self(b), - balance: gor.NewState[int64](b, "balance"), + id: gor.Self(g), + balance: gor.NewState[int64](g, "balance"), } }) } @@ -108,23 +130,24 @@ GrainId have the same GrainId. ## The Grain reads time -The `Binder` is given to the factory once, at activation. If method bodies need it, keep it in the factory — registration shaped as in the previous sections: +The Grain Context is given to the factory once, during activation. If method +bodies need it, keep it in the Grain. Use this registration shape: ```go type device struct { - b *gor.Binder + grain *gor.GrainContext reading gor.State[reading] } func Register(rt *gor.Runtime) error { - return gor.Register[Device](rt, func(b *gor.Binder) Device { - return &device{b: b, reading: gor.NewState[reading](b, "reading")} + return gor.Register[Device](rt, func(g *gor.GrainContext) Device { + return &device{grain: g, reading: gor.NewState[reading](g, "reading")} }) } func (d *device) Report(ctx context.Context, value float64) error { next := d.reading.Get() - next.ReportedAt = gor.Now(d.b) + next.ReportedAt = gor.Now(d.grain) ... } ``` @@ -137,11 +160,11 @@ Tests must control time, and a future Silo may have a different clock. The same function as calling from outside, with a different first argument: ```go -gor.Ref[Workshop](d.b, workshopID).DeviceOnline(ctx, deviceID) +gor.Ref[Workshop](d.grain, workshopID).DeviceOnline(ctx, deviceID) ``` -Outside, the caller holds the Runtime. Inside, the Grain holds the `Binder`. -The factory needs only `func(b *gor.Binder) T`. +Outside, the caller holds the Runtime. Inside, the Grain holds its Grain +Context. The factory needs only `func(g *gor.GrainContext) T`. Cross-Grain Calls are part of the virtual Grain model. They use the same typed reference as a local Call. @@ -157,22 +180,33 @@ balance, err := acct.Deposit(ctx, 100) compile error. This is the key difference from `any`-based APIs. See [../design/codegen.md](../design/codegen.md). -### Cluster calls and deployment limits +### Cluster preview: Calls and deployment limits -Call syntax does not change, but three things matter in a cluster: +This section is not part of the 0.1.0 product contract. -**Branchable errors must have stable error codes.** Declared codes are checkable with `errors.Is` both locally and across nodes. Undeclared codes leave only displayable text across nodes — no branching on text, type, or fields. Full contract: [errors.md](errors.md). +Call syntax does not change in a Cluster. These limits apply: -**Cancellation does not cross nodes.** In a local call, canceling `ctx` cancels the method body's `ctx` too. Across nodes, the caller gets its own `ctx.Err()` first, and the method on the other side keeps its own context and may run to completion. Full boundary: [errors.md](errors.md). +**Branchable errors need stable error codes.** `errors.Is` can check a declared +code across Silos. An undeclared error keeps only diagnostic text. See +[errors.md](errors.md). -Arguments and return values go through JSON across nodes, so they must be JSON-encodable types. Local calls skip this pass — when the same method works locally and blows up cross-node, this is usually where it comes from. +**Cancellation does not cross Silos.** Local cancellation reaches the method +context. A forwarded Call can continue on the target Silo. See +[errors.md](errors.md). -**An incompatible change cannot ride on a rolling, no-downtime upgrade within one cluster.** Nodes in a cluster must run mutually compatible application versions. When method signatures or state formats are incompatible, the application decides between a release with downtime and arranging dual writes itself. +Forwarded arguments and results use JSON. Their types must support JSON. + +**A Cluster needs compatible Application versions.** An incompatible method or +State change needs downtime or an Application migration path. ## Call outcomes and ordering One Grain processes Calls in a queue. When the queue is full, new Calls are rejected for overload. The method does not start, and State does not change. +The Silo also limits starting, active, and deactivating Activations. A Call +that needs a new Activation can be rejected before the Grain lane, mailbox, +and factory when that limit is full. Calls to an active Grain do not need a new +Activation slot. Timeout or cancellation means that the caller stopped waiting. The method may have started and may have changed State. A delivery error after a Call @@ -187,8 +221,8 @@ While a Grain handles one Call, it does not start a second Call. A Call cycle is detected and fails instead of waiting forever. The Runtime does not retry the Call. The Application decides whether a Safe Repeat is valid. -Calls from one caller to one Grain, sent locally in sequence, execute in issue -order. A future cluster does not promise network arrival order. +Calls from one caller to one Grain execute in local issue order. A future +Cluster does not promise network arrival order. ## State @@ -201,29 +235,40 @@ removes confirmed State. After `Clear()` succeeds, the next Activation sees the State as absent. A Grain can have several named State values. They are stored as one Grain -record, so any State write updates the Grain version. +record. -**When State holds a map or slice, `Get()` returns that instance, not a copy.** -The change is persisted only after `Set()`. After the Grain leaves memory, an -unsaved change is lost. +**When State holds a map or slice, `Get()` returns that value, not a copy.** +The same rule applies to pointers and interfaces that contain reference +values. Call `Set()` after you change such a value. The Runtime cannot undo a +change that the Application made through the returned value. +Each Grain State record has an ETag. A successful write changes its ETag. Every `Set()` tries to persist immediately. Only success confirms the new -value. On failure, the Runtime keeps the last confirmed value and discards -the current Activation. The next Call reads State again. +value, presence mark, and ETag. A failed write can have an Unknown +Result. A Conflict, cancellation, timeout, or lost reply all end the current +Activation. The next Call starts a new Activation and reads Confirmed State. +The Runtime does not retry the Business Action. + +A persistence error has a stable error code. Its text names the State +operation and the GrainId. An error for one named State also names that State. Multiple `Set()` calls in one method are separate State writes. An earlier write may succeed before a later write fails. Keep one business change in one State update when that result is required. -State must be JSON-encodable. The Application owns State format changes. +State must be JSON-encodable. The Runtime stores all named State values for a +Grain in one JSON object. The State names are the object keys. The Application +owns State format changes and must read old data when its format changes. +An empty, `null`, or malformed State record fails Activation with a stable +persistence error. The Runtime does not replace that record. -In a future cluster, the Runtime may have two Activations for one Grain while -ownership changes. Both may accept a Call. The State version check rejects +In the Cluster preview, the Runtime can have two Activations for one Grain while +ownership changes. Both may accept a Call. The ETag check rejects the old write instead of silently replacing newer State. The caller receives a conflict and decides whether to retry. -This behavior follows the Orleans model. A single Silo has no ownership -change, so this cluster conflict does not occur there. +This behavior follows the Orleans model. A Single Silo has no Ownership +change, so this Conflict does not occur there. ## How durable a state write is @@ -237,35 +282,92 @@ Two levels: The trade is throughput. Forcing every write to disk costs time; most services can tolerate losing the most recent changes after a hard crash, and Relaxed lets those services change state faster. Relaxed touches Grain State and nothing else. Reminders still fire at most -once after a crash. Future cluster ownership data is unaffected. +once after a crash. Future Cluster Ownership data is unaffected. If you do not choose, you get Full. The mechanism behind the trade and its exact limits are in the [persistence design](../design/persistence.md). +## SQLite cold backup and restore + +The built-in SQLite Store uses two database files. For `data/gor.db`, the +coordination file is `data/gor.db`. The State file is `data/gor-state.db`. +Each file can also have a `-wal` file. + +Use this cold backup procedure: + +1. Call `Shutdown(ctx)` and wait for it to return. +2. Close the SQLite Store. +3. Copy both database files as one backup set. +4. Copy each `-wal` file that still exists. + +For restore, replace the complete closed file set. Do not restore only one +database file. The next Runtime start checks both databases and rejects a +damaged backup. + ## Reminder -State connects to the store via `gor.State[T]`; a Reminder uses the Binder in -the same way: +State connects to the store through `gor.State[T]`. A Reminder uses Grain +Context in the same way: ```go -type account struct { +//gor:grain +type InterestAccount interface { + Open(ctx context.Context) error + ApplyInterest(ctx context.Context, tick gor.TickStatus) error +} + +type interestAccount struct { balance gor.State[int64] - reminder gor.Reminder[Account] + reminder gor.Reminder[InterestAccount] +} + +func RegisterInterestAccount(rt *gor.Runtime) error { + return gor.Register[InterestAccount](rt, func(g *gor.GrainContext) InterestAccount { + return &interestAccount{ + balance: gor.NewState[int64](g, "balance"), + reminder: gor.NewReminder[InterestAccount](g), + } + }) +} + +func (a *interestAccount) Open(ctx context.Context) error { + schedule := gor.Every(30 * 24 * time.Hour) + return a.reminder.Set(ctx, "monthly-interest", schedule, gor.Handle(InterestAccount.ApplyInterest)) } -func (a *account) Open(ctx context.Context) error { - return a.reminder.Set(ctx, "monthly-interest", gor.Every(30*24*time.Hour), gor.Handle(Account.ApplyInterest)) +func (a *interestAccount) ApplyInterest(ctx context.Context, _ gor.TickStatus) error { + balance := a.balance.Get() + return a.balance.Set(ctx, balance+(balance/100)) } -func (a *account) ApplyInterest(ctx context.Context, tick gor.TickStatus) error { ... } +account := gor.Ref[InterestAccount](rt, "alice") +if err := account.Open(ctx); err != nil { + return err +} ``` +`Open` is an ordinary Call. The Application calls it to set the Reminder. The +Runtime does not call methods only because they have a specific name. + A Reminder is persistent. After a process crash, a due Reminder can still run. If the Grain is not in memory, the Runtime starts its Activation. The Reminder is typed to the Grain interface. The Reminder method uses a method expression, so a typo or rename is a compile error. The Runtime stores the method name, not a function value. The method takes `ctx` and -`gor.TickStatus`, and returns `error`. +`gor.TickStatus`, and returns `error`. `TickStatus.ReminderName` is the +persisted Reminder name that caused the Call. One method can use it to handle +many dynamic Reminder names. + +An old persisted Reminder can contain a GrainType or method that is not +installed. This is an Invalid Reminder. The Runtime gives the unchanged +setting a Terminal Result with an ETag Claim. This removes the setting. The +Runtime then reports one `ReminderDispatch` error. Later polls and a process +restart do not report the same setting again. Setting the same Reminder name +again creates a new setting. + +`Set` rejects an empty name, a negative due time, a negative period, and an +invalid method handle. It does this before Store I/O. `Every(0)` creates a +one-shot Reminder. It is not `time.AfterFunc`. It does not promise millisecond precision. It does not replay every tick missed during downtime. @@ -281,6 +383,10 @@ method execution. The Runtime claims the due time before delivery. A crash between these actions can miss the Call. A failed method is not retried; its error goes to the background error sink. +The Runtime reads due Reminders in cursor pages. It also limits active Reminder +Calls. The defaults are 256 rows per page and 16 active Calls. Use +`WithReminderPageSize` and `WithReminderWorkers` to set other positive values. + A State change and a Reminder change are separate Runtime actions. The Application must handle a partial result when both actions are needed. @@ -289,40 +395,92 @@ Application must handle a partial result when both actions are needed. A Grain can initialize when its Activation starts. If initialization fails, that Call fails and the next Call builds a new Activation. +A Grain can request Deactivate on Idle. The Runtime ends the Activation after +the current Call. Use `gor.DeactivateOnIdle(grainContext)` in a Grain method. +Calls that already wait keep their order and enter a new Activation. A panic +in the current Call faults the old Activation instead. In that case, waiting +Calls fail and do not run. + A Grain can run a deactivation hook before it leaves. The hook receives the -reason: idle, ownership lost, normal shutdown, or an untrusted Activation. +reason: idle, Application request, ownership lost, Runtime shutdown, or an +untrusted Activation. The hook cannot prevent deactivation. A graceful stop waits for a hook that has started. An abrupt stop does not start new hooks. A hook that has started is not force-aborted. +## Grain Timers + +A Grain Timer runs a callback for the current Activation. It is not saved. The +Runtime stops it when that Activation ends. + +```go +func (g *cacheGrain) startRefreshTimer(grainContext *gor.GrainContext) error { + timer, err := gor.RegisterGrainTimer(grainContext, func(ctx context.Context) error { + return g.refreshLocalCache(ctx) + }, gor.GrainTimerOptions{ + DueTime: time.Second, + Period: time.Minute, + }) + if err != nil { + return err + } + g.refreshTimer = timer + return nil +} +``` + +Keep the returned handle when the Grain must call `Change` or `Stop`. + +The callback enters the same mailbox as a Call. It does not overlap with a +Call or another callback for that Grain. A repeating period starts after the +callback finishes. The Grain can change or stop the timer. + +`Stop` also prevents a queued callback from starting. It does not cancel a +callback that already runs. `Change` after `Stop` or Deactivation returns +`gor.ErrGrainTimerStopped`. Timer callbacks use a Runtime context. They do not +inherit Request Context or the Call deadline that created the Activation. A +callback that calls the same Grain gets `gor.ErrCallCycle`. + +A Grain Timer does not keep the Activation active by default. The Application +can select keep-alive when it registers the timer. A completed keep-alive +callback resets idle time. It does not keep the Activation forever. A long +period can still let the Activation end before the next callback. Use a +Reminder when work must survive deactivation or a process restart. + ## Failures nobody is waiting for -Two Application actions can fail with no caller waiting: a claimed Reminder -Call can fail, or a deactivation hook can fail. The Runtime can send both to -a background error sink. +Background work can fail with no caller waiting. Grain Timer callbacks, +Reminder deliveries, and deactivation hooks send these failures to one +background error sink. -Each event gives the GrainId, the original error, and a source. A Reminder -event gives the method name. A deactivation event gives the leave reason. +Each event gives the GrainId when known, the original error, and a source. +`ReminderScan` reports a Store scan failure. `ReminderDispatch` reports an +unknown stored GrainType or method. `ReminderTerminal` reports a Store failure +while the Runtime claims an Invalid Reminder for its Terminal Result. +`ReminderClaim` reports a Claim Store failure. `ReminderInvocation` gives the +Reminder name, method, and TickStatus. A deactivation event gives the reason. -Errors still follow the [Errors and cancellation](errors.md) section. Across nodes, only declared stable codes are usable for business branching; error text is for display and logging. +A lost Reminder Claim CAS is normal contention. It is not an error event. A +Claim can take effect before its Store reports an error. In this case, the +Runtime reports `ReminderClaim` and does not deliver that due time. + +Errors follow [Errors and cancellation](errors.md). Across Silos, only declared +stable codes support business decisions. Error text is diagnostic. The sink does not retry, back off, or alert. Reminder delivery is at-most-once by design. The Application owns any Safe Repeat behavior. -The handler must read the Reminder method or deactivation reason from the -event source. It must not infer the source from a method name. - -### Gap +The handler must return promptly. It must not do blocking I/O. -The background error sink and deactivation reasons are implemented. The -remaining release work is listed in [../ROADMAP.md](../ROADMAP.md). +The handler must read the Reminder source or deactivation reason from the +event source. It must not infer the source from a method name. ## Runtime observability The Runtime provides two kinds of facts. First, it provides a snapshot of this Silo's active Activations and their queued Calls. It does not aggregate -data for a future cluster. +data for a future Cluster. Second, it provides one event for each completed Call. The event gives the caller result, duration, GrainType, and method. A canceled Call has one @@ -341,76 +499,67 @@ database, err := store.OpenSQLite("data/gor.db") if err != nil { return err } defer database.Close() -rt, err := gor.New(gor.WithStore(database)) +rt, err := gor.New( + gor.WithStore(database), + gor.WithReminderStore(database), +) if err != nil { return err } -defer rt.Close() if err := gorgen.Install(rt); err != nil { return err } -``` - -`Install` hands generated proxies and dispatch functions to the Runtime. -Without this line, Grain registration and Grain References fail at startup. - -A cluster must explicitly hand the runtime the state store, the shared membership table, this node's address, this startup's generation, and the transport: +if err := domain.Register(rt); err != nil { return err } +if err := rt.Start(ctx); err != nil { return err } -```go -nodeTransport, err := transport.New(":7373") -if err != nil { return err } +// Run Application Calls. -rt, err := gor.New( - gor.WithStore(stateStore), - gor.WithMemberStore(memberStore), - gor.WithNodeAddr(nodeTransport.Addr()), - gor.WithGeneration(generation), - gor.WithTransport(nodeTransport), -) -if err != nil { - nodeTransport.Close() - return err -} -defer rt.Close() +shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) +defer cancel() +if err := rt.Shutdown(shutdownCtx); err != nil { return err } +return nil ``` -All nodes share `memberStore`; `generation` must be a fresh value on every rejoin at the same address. `Runtime.Close` closes the configured transport. The difference between a single Silo and a future cluster is configuration, not business code. +`Install` gives generated Grain Reference and dispatch functions to the Runtime. +Without this line, Grain registration and Grain References fail at startup. +`Start` checks and freezes the complete setup before it accepts a Call or +claims a Reminder. The Application calls `Shutdown(ctx)` before it closes the +database. + +The 0.1.0 product uses one Silo. Cluster configuration is not part of this +startup contract. GrainId and Call boundaries leave room for later Cluster +work. A Single Silo does not configure a Transport or membership Store. ## The runtime can stop itself -In a future cluster, a Silo can be declared dead by other Silos. After that it -must serve no Grain. +The Application uses `Shutdown(ctx)` for a normal stop. Shutdown stops new +Call admission and waits for Runtime infrastructure to end. If the context +ends first, Shutdown uses the abrupt stop path and returns the context error. -So the runtime provides a signal: +In the Cluster preview, other Silos can declare a Silo dead. That Silo must +then serve no Grain. + +The Runtime provides two signals: ```go -<-rt.Done() // closed, or declared dead +<-rt.Stopping() // Call admission has ended. +<-rt.Done() // Runtime infrastructure has ended. ``` -When it closes, the Runtime stops admitting new Grain Calls. Calls issued -after that get a stable stop error. Codes and checks are in [errors.md](errors.md). +`Stopping` also closes when a Cluster declares this Silo dead. Calls issued +after it closes get a stable stop error. Codes and checks are in +[errors.md](errors.md). -The stop signal does not rewrite results for Calls already admitted. A +`Stopping` does not rewrite results for Calls already admitted. A graceful stop lets started methods finish and rejects queued Calls. An abrupt stop cancels started methods but cannot force-abort user code that ignores cancellation. -Your process should exit, or build a new runtime and rejoin. Ignoring the signal does not silently break anything, but the service should not keep advertising itself as available. - -### Gap - -The Runtime admission boundary is implemented. Calls after stop receive the -stable stop error. Calls admitted before stop keep the result defined by the -stop mode. The code surface still needs the public Grain terminology and the -State and Reminder operations described in this target API. - -## Mental model comparison +The process should exit or build a new Runtime and rejoin. It must not keep +advertising the stopped Runtime. -If you have used other systems: +The Runtime setup and stop state machines are implemented. `Start` freezes +setup before it accepts a Call. `Shutdown` uses its context as the stop budget. +`Stopping` and `Done` report the two stop transitions shown above. -| Concept | Orleans | Temporal | Restate | gor | -|---|---|---|---|---| -| Stateful object with a GrainId | Grain | — | Virtual Object | Grain | -| GrainId | GrainId | WorkflowId | Object Key | GrainId | -| Persistent state | `[PersistentState]` | Workflow variables | built-in K/V | `State[T]` | -| Scheduled action | Reminder | Timer | — | Reminder | +## Complete example -The table only builds intuition. Semantics are not fully equivalent. The -Orleans Grain model is the reference for `gor`. +The [device shadow example](../examples/shadow/) combines Grain References, +State, Reminders, lifecycle hooks, and HTTP Calls. diff --git a/docs/release-0.1.0.md b/docs/release-0.1.0.md index 60a42ab..5a360be 100644 --- a/docs/release-0.1.0.md +++ b/docs/release-0.1.0.md @@ -1,20 +1,47 @@ # 0.1.0 Product Contract This document states what gor 0.1.0 promises. It is not a status report. -[ROADMAP.md](../ROADMAP.md) states what is done. The [design -document](../design/release-0.1.0.md) states how the release is delivered. +[ROADMAP.md](../ROADMAP.md) states what is done. The [delivery +design](../design/release-0.1.0.md) states the work order. ## Release promise -gor 0.1.0 is a reliable single-Silo Grain Runtime for Go Applications. It -starts Grains when Calls need them, serializes Calls for each Grain, and -keeps confirmed State in local storage. +gor 0.1.0 is a reliable Single Silo Grain Runtime for Go Applications. It +starts Grains when Calls need them. It serializes Calls for each Grain. It +keeps Confirmed State in a local store. -The release targets one process and one local store. It does not require a -network or a cluster. +The release targets one process and one local Store. The Single Silo path does +not need a network or a Cluster. -The release is ready only when users can understand the rules, test failure -cases, and use the public API without private runtime code. +The release is ready only when users can understand failure results, test +them, and use the public API without private runtime code. + +## Application setup + +The Application must complete setup before the Silo starts: + +1. Create the Runtime with explicit State and Reminder Stores. The built-in + SQLite Store provides both. +2. Install generated Grain code. +3. Register each Grain factory. +4. Start the Runtime. + +Start validates the complete setup. It rejects an invalid option, a missing +factory, a duplicate GrainType, and a Store that is not ready. A failed start +does not accept Calls or claim Reminders. + +Start freezes the setup. The Application must not install or register a Grain +after Start succeeds. + +The in-memory Store is an explicit test and development option. It does not +meet the release durability promise. + +A custom Reminder Store must not reuse an ETag after Delete. A later Put must +use a newer ETag. This prevents an old Claim from changing a new setting. + +The Application calls Runtime Shutdown before it closes the Stores. +`Stopping` reports when Call admission ends. `Done` reports when Runtime +infrastructure has ended. ## Supported capabilities @@ -23,134 +50,264 @@ cases, and use the public API without private runtime code. A Grain is identified by a GrainType and a GrainKey. Together they form a GrainId. -A Grain Reference names a Grain without starting it. The first Call can -start its Activation. An idle Grain can leave memory. A later Call can start -it again and load its State. +A GrainType is a stable Application name. Generated code carries this name. +Runtime Go type text does not define it. Start rejects duplicate GrainTypes. + +A Grain Reference names a Grain without starting it. The first Call can start +its Activation. An idle Grain can leave memory. A later Call can start it +again and load its State. Calls for one Grain run one at a time. Calls for different Grains may run at -the same time. The Runtime defines results for overload, timeout, cancel, -start failure, method failure, panic, and shutdown. +the same time. ### Calls -The public API uses typed Go interfaces. A caller uses a typed method call. -It does not send an untyped message. +The public API uses typed Go interfaces. A caller uses a typed method call. It +does not build an untyped request. + +A Grain may call another Grain through a Grain Reference. Local Calls and +future remote Calls use the same call model. -A Grain may call another Grain through a Grain Reference. Local and future -remote Calls use the same model. +The Runtime does not re-enter a Grain during a Call. A detected Call cycle +fails instead of waiting forever. -The Runtime does not re-enter a Grain during a Call. Reentrant and -interleaved Calls are outside 0.1.0. A call cycle fails instead of waiting -forever. +A Call that is canceled while it waits in a mailbox must not enter its method. +If cancellation happens after method entry, the caller stops waiting. The +method can continue and can change State. -The Runtime does not retry a Business Action after an unknown result. The +An overloaded mailbox rejects the Call before method entry. A method panic +fails the Call and discards the Activation. Queued Calls that did not run also +fail. The Runtime does not replay them on a new Activation. + +The Silo also has an Activation limit. A Call that needs a new Activation is +rejected before the Grain lane, mailbox, and factory when the limit is full. +Calls to current Activations can continue. The public error matches +`gor.ErrOverloaded`. + +The Runtime does not retry a Business Action after an Unknown Result. The Application must decide whether a Safe Repeat is valid. ### Request Context -A Call may carry Request Context, such as a trace ID. The called Grain can -read it during the Call. The Grain Runtime does not save it. +A Call may carry Request Context, such as a trace ID. The called Grain can read +it during the Call. The Grain Runtime does not save it. + +Request Context from the Call that starts an Activation is available during +activation. Cancellation by that caller does not cancel shared activation +work for other callers. + +### Grain Context and lifecycle + +The Runtime gives one Grain Context to each Activation. The Grain uses it to +get State, Grain References, Grain Timers, Reminders, and lifecycle controls. + +A Grain can run code when its Activation starts and when it leaves memory. +The deactivation reason tells the Application why the Activation ended. + +A Grain can use Deactivate on Idle after it changes local data that it no +longer trusts. Deactivation starts after the current Call. A later Call uses a +new Activation. + +A panic in an activation hook, a deactivation hook, or an error observer must +not stop the Silo process. The Runtime must finish its own cleanup. ### Persistent State A Grain can own named State values. A successful State write becomes the confirmed value for the Grain. -State provides these user-visible operations: +State provides these user-visible methods: -- Read the current value. -- Write a new value. -- Check whether a value exists. -- Clear the value. +- `Get` reads the current value. +- `Set` writes a new value. +- `Exists` checks whether a value exists. +- `Clear` removes the value. An absent value is different from a value that contains the type's zero value. Clear removes the confirmed value. A later Activation observes that the value is absent. -A version check stops an old Activation from replacing newer State. The -Runtime reports a conflict. +Each Grain State record has an ETag. A successful write changes its ETag. The +ETag check stops an old Activation from replacing newer State. The Runtime +reports a Conflict. + +A failed State write does not commit its candidate value, presence mark, or +ETag. The Runtime cannot undo a change made through a map, slice, pointer, or +interface returned by `Get`. The Runtime discards that Activation before it +accepts another Call. This rule includes a canceled write and a write with an +Unknown Result. State has two durability levels: - **Full**: confirmed writes are on disk before the Call returns. -- **Relaxed**: a hard machine failure may lose recent confirmed writes. - A normal process restart keeps confirmed writes. +- **Relaxed**: a hard machine failure can lose recent confirmed writes. A + normal process restart keeps confirmed writes. -State is Application data. The Application owns its meaning and its format +State is Application data. The Application owns its meaning and format changes. +### Grain Timers + +A Grain can register a Grain Timer for its current Activation. A Grain Timer +can run once or repeat. The Runtime does not save it. Deactivation stops and +discards all Grain Timers for that Activation. + +A Grain Timer callback is a serialized Grain turn. It does not overlap with +itself or with another Call for that Grain. The next tick is scheduled after +the callback finishes. + +A Grain Timer does not keep an Activation active by default. The Application +can select keep-alive behavior when it registers the timer. + +The Application can change or stop a Grain Timer. A callback error goes to the +background error sink. It does not stop later ticks. A callback panic also +discards the Activation. + ### Reminders -A Grain can set a named Reminder. A Reminder can run once or repeat on a -period. The setting, a new setting, and cancellation survive a normal -process restart. +A Grain can set a named Reminder. A Reminder can run once or repeat. Setting, +replacement, and cancellation survive a normal process restart. A due Reminder can start a Grain that is not in memory. The Runtime claims a -Reminder before it delivers the Call. A failure after the claim can miss -that delivery. The Runtime does not retry the Call automatically. +Reminder before it delivers the Call. A failure after the claim can miss that +delivery. The Runtime does not retry the Call automatically. -A periodic Reminder reports its first tick time, period, and current tick -time to the Grain. A late process does not receive every missed tick. +A Reminder method receives the Reminder name that caused the Call. This lets +one typed method handle many dynamic Reminder names. The name remains the same +after a process restart. -An Application that needs recovery must save a pending Business Action and -use a Safe Repeat handler. +A periodic Reminder reports its first tick time, period, and actual delivery +start time. A late process does not receive every missed tick. -### Lifecycle and background errors +The Runtime gives an Invalid Reminder a Terminal Result. A Reminder is invalid +when its stored GrainType or method is not installed. The Runtime +uses the Reminder ETag to remove the unchanged row. It reports one +`ReminderDispatch` error only after the Store confirms the Terminal Result. +A later process does not report the same row again. -A Grain can run code when its Activation starts and when it leaves memory. -The leave reason tells the Application why the Activation ended. +A concurrent Set or Cancel can change the row first. The old scan then +produces no error. A Store error produces a `ReminderTerminal` error. The +Runtime can try the terminal Claim again if the row remains due. + +Set can create a new Reminder with the same name. + +The terminal step has the same at-most-once limit as normal Reminder delivery. +A process failure after the Store result can prevent the background error +report. + +Reminder scans and deliveries use bounded work. A large due set must not start +an unlimited number of goroutines or load all due rows at once. + +An Application that needs recovery must save a pending Business Action and use +a Safe Repeat handler. + +The Safe Repeat handler must confirm the required State before it commits its +Application receipt. If the State result is unknown, the pending action must +remain available for a later Call. + +### Background errors and observability Failures with no waiting caller go to the configured background error sink. -Examples include a failed Reminder Call and a failed deactivation hook. The -sink reports the original error and its source. It does not add hidden -retries. +Each event contains the original error, the GrainId when known, and a stable +source. Sources include Grain Timer callbacks, Reminder work, and deactivation +hooks. The sink does not add a hidden retry. -### Observability +The Runtime catches a panic from the error sink. A failing sink must not hide +the first error or stop Runtime cleanup. The Runtime provides a snapshot of this Silo's active Activations and a -completion event for each Call. The Application chooses its own metrics, -traces, storage, and alerts. +completion event for each Call. The Application chooses its metrics, traces, +storage, and alerts. + +## Single Silo boundary + +The 0.1.0 release is a Single Silo product. It must not need a network, a +remote service, or Cluster membership. + +GrainId, Grain Reference, Call, State, Grain Timer, Reminder, and encoding +boundaries must leave room for future Cluster work. This is a design rule. It +is not a promise of reliable Cluster operation in 0.1.0. + +Multi-Silo operation and Cluster administration are outside the 0.1.0 +promise. + +## Deferred capabilities -## Single-Silo boundary +Call Filters and reentrant or interleaved Grain Calls are not part of 0.1.0. -The 0.1.0 release is a single-Silo product. The single-Silo path must not -need a network, a remote service, or cluster membership. +## Upgrade from v0.0.5 -GrainId, Grain Reference, Call, State, Reminder, and encoding boundaries -must leave room for future cluster work. This is a design rule. It is not a -promise of reliable cluster operation in 0.1.0. +The 0.1.0 public API and generated files are not source-compatible with +`v0.0.5`. Stop all Application processes before the upgrade. A rolling upgrade +between these versions is not supported. -Multi-Silo operation, ownership changes, network failure handling, rolling -upgrades, and cluster administration are outside the 0.1.0 promise. +Before the upgrade, make a backup of the SQLite database and its `-wal` and +`-shm` sidecar files. +Then update the Application source for these changes: -## Non-goals +- change `//gor:entity` to `//gor:grain`; +- use `GrainId` in place of `Identity`; +- use `GrainContext` in place of `Binder`; +- use the Reminder API in place of the old Schedule API; +- configure the State and Reminder Stores before Runtime start; +- register all Grain factories, then call Runtime Start; +- call Runtime Shutdown before the Application closes its Stores. -0.1.0 does not provide Call Filters, reentrant or interleaved Grain Calls, -cluster operation tools, incompatible rolling upgrades, cloud storage, or -unlimited scale. +Remove the old generated files. Run the 0.1.0 generator after the source update. +Do not compile old generated files with the 0.1.0 Runtime. -These limits keep the single-Silo product small and reliable. +The first 0.1.0 start migrates the gor-managed SQLite data. It keeps confirmed +State, Reminders, and Member rows. It moves State to the State database. It also +changes the old Reminder identity columns to the 0.1.0 Grain identity columns. +If migration fails, the Runtime does not start. Restore the backup before you +try a different migration. ## Acceptance standard The release is ready only when all items below are true: -1. A small example can define a Grain, get a typed Grain Reference, write - and clear State, and set a Reminder through the public API. -2. The example can stop and start the process without private runtime calls - or manual database repair. -3. Deterministic tests cover activation, serialized Calls, State conflicts, - State clearing, process failure, Reminder claims, Safe Repeats, cancel, +1. A small Application can define a Grain, get a typed Grain Reference, and use + Grain Context. It can write and clear State, register a Grain Timer, set a + Reminder, and use Deactivate on Idle. +2. The Application can stop and start the process without private Runtime + calls or manual database repair. +3. Startup rejects incomplete setup before a Call or Reminder can run. +4. Deterministic tests cover Activation, serialized Calls, queued cancel, + callback panic, State conflicts, failed State writes, Grain Timers, + Reminder claims, Invalid Reminder Terminal Results, Deactivate on Idle, shutdown, and background errors. -4. The docs state the result for timeout, cancel, delivery failure, and a - claimed Reminder whose Call did not run. -5. The full repository test gate passes. It includes unit tests, simulation, - generated-code tests, network tests, lint, and race tests. +5. The docs state the result for overload, timeout, cancel, panic, storage + failure, delivery failure, and a claimed Reminder whose Call did not run. +6. Generated files are current. Invalid Grain interfaces fail generation with + a source location and a clear error. +7. Supported operating systems compile and run their release test sets. +8. The full repository release gate passes with nonzero tests. It includes + unit, simulation, generated-code, network, race, fuzz, and resource checks. + Separate targeted mutations prove the critical behavior tests. A + caller-provided Make variable cannot replace the internal test command. +9. A clean consumer module builds the committed example with empty Go caches. + Separate processes prove normal restart and stop-after-Claim recovery. ## Gap -The conformance Application in `examples/shadow` now composes the -single-Silo Runtime, State, Reminders, typed Calls, Request Context, lifecycle, -and observations under restart and failure. It keeps business records in a -separate ApplicationStore and uses ActionID Safe Repeat. The remaining release -status is tracked in ROADMAP.md and the release gates. +The current implementation has the Single Silo core and a conformance +Application. It does not yet meet this complete contract. + +Runtime startup, GrainType declarations, Grain Context, Activation lifecycle, +Deactivate on Idle, queued Call cancellation, callback containment, and Grain +Timers are implemented. State failure and SQLite recovery rules are also +implemented. Direct proof covers a mutable State alias and a Reminder-turn +State failure. Reminder work is bounded and its failures are observable. +Reminder methods also receive the persisted Reminder name. Generator checks +and the Silo Activation admission limit are implemented. +Bounded Activation churn, bounded Reminder backlog proof, and portable +benchmark probes are also implemented. The local release gate now rejects +zero-test targets, checks generated and module files, and runs the required +fuzz and resource checks. A caller-provided Make variable cannot replace the +internal test command. Action recovery confirms requested State before it +commits the Application receipt. The clean external-module restart proof +passes locally on Linux. The exact `v0.0.5` migration and external upgrade +proof also pass locally on Linux. An Invalid Reminder now gets one Terminal +Result. Public document cleanup is complete. Hosted +supported-system proof, the exact tagged-module run, and the final same-commit +proof remain open. ROADMAP.md tracks the batches. diff --git a/docs/vision.md b/docs/vision.md index 8bd5a69..edcc896 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -9,12 +9,12 @@ stateful Grains on one machine. `gor` gives a Go program a Grain Runtime. A Grain has a GrainType, a GrainKey, State, and behavior. The runtime starts a Grain when a Call needs -it. It keeps Calls for one Grain in order and keeps confirmed State in local +it. It keeps Calls for one Grain in order and keeps Confirmed State in local storage. -The first product is one Silo on one machine. It must be useful without a -network, a sidecar, or a remote service. A later cluster extension may move -Grain ownership between Silos. It must not change the Grain model. +The first product is a Single Silo on one machine. It must be useful without a +network, a sidecar, or a remote service. A later Cluster extension may move +Grain Ownership between Silos. It must not change the Grain model. ## Core promises @@ -22,14 +22,14 @@ Grain ownership between Silos. It must not change the Grain model. `gor` uses the Orleans model and terms. A Grain Reference names a Grain without starting its Activation. The first Call can start the Activation. -The Grain may leave memory later. Its GrainId and confirmed State remain. +The Grain may leave memory later. Its GrainId and Confirmed State remain. The Go API may use Go forms where the languages differ. The runtime meaning must stay aligned with Orleans unless a product spec states a difference. -### Reliable single Silo +### Reliable Single Silo -The single-Silo product must make these results dependable: +The Single Silo product must make these results dependable: - Calls for one Grain run one at a time. - State survives a normal process restart. @@ -50,9 +50,9 @@ typed Grain Reference. A wrong argument type must fail at compile time. Tests must control time and I/O through explicit boundaries. Failure tests must use a seed that can reproduce the same decisions. This rule applies to -the single-Silo product and to the future cluster extension. +the Single Silo product and to the future Cluster extension. -## Future cluster boundary +## Future Cluster boundary Cluster support is a later extension. It may add several Silos, shared Grain ownership, routing, and transport. The 0.1.0 product does not promise these @@ -61,7 +61,7 @@ features. The public model must leave room for this extension. A Grain Reference must not depend on a local memory address. State must keep a version that can reject an old write. These are design constraints for future work, not -cluster promises in 0.1.0. +Cluster promises in 0.1.0. ## Product boundaries @@ -79,7 +79,7 @@ may add a capability only after its behavior and failure rules are defined. ## Related systems `gor` is a library. It runs inside the Application and uses a local store. -It does not require a separate runtime service for the single-Silo product. +It does not require a separate Runtime service for the Single Silo product. The project uses Orleans as its model reference. It does not promise source or binary compatibility with Orleans. It promises the Orleans Grain model in diff --git a/docs/writing-style.md b/docs/writing-style.md index edd3850..d9a99ed 100644 --- a/docs/writing-style.md +++ b/docs/writing-style.md @@ -21,12 +21,15 @@ Use American English spelling. ## Scope -This rule covers all new and changed prose in: +This rule covers all new and changed prose in repository documents. This +includes: - `docs/`; - `design/`; +- `research/`; - `README` files; - `ROADMAP.md`; +- other root Markdown files; - public API comments and examples. Use English only. Code, names, commands, logs, URLs, and quoted source text diff --git a/errors.go b/errors.go index b3b3023..3465aa6 100644 --- a/errors.go +++ b/errors.go @@ -5,8 +5,8 @@ import ( "errors" "github.com/suraciii/gor/cluster" - "github.com/suraciii/gor/mail" - runtimepkg "github.com/suraciii/gor/runtime" + "github.com/suraciii/gor/internal/mail" + runtimepkg "github.com/suraciii/gor/internal/runtime" "github.com/suraciii/gor/store" ) @@ -81,7 +81,10 @@ func CodeOf(err error) (Code, bool) { const ( ErrNoOwner Code = "gor.no_owner" ErrNodeDead Code = "gor.node_dead" + ErrRuntimeNotStarted Code = "gor.runtime_not_started" ErrRuntimeClosed Code = "gor.runtime_closed" + ErrSetupFrozen Code = "gor.setup_frozen" + ErrInvalidSetup Code = "gor.invalid_setup" ErrOverloaded Code = "gor.overloaded" ErrTypeNotInstalled Code = "gor.type_not_installed" ErrUnknownMethod Code = "gor.unknown_method" @@ -92,9 +95,9 @@ const ( ErrRequestEncodeFailed Code = "gor.request_encode_failed" ErrReplyEncodeFailed Code = "gor.reply_encode_failed" ErrTransportFailed Code = "gor.transport_failed" - // ErrCallCycle reports that a call targeted an entity that the same call - // chain already occupies, so the call could never start. The error text - // names the entities in the cycle. The cycle is detected at delivery, not + // ErrCallCycle reports that a Call targeted a Grain that the same Call + // chain already occupies, so the Call could never start. The error text + // names the Grains in the cycle. The cycle is detected at delivery, not // inferred from elapsed time: a slow call that is not a cycle still times // out as a plain timeout. ErrCallCycle Code = "gor.call_cycle" @@ -141,7 +144,7 @@ func publicError(err error) error { return withCode(ErrNodeDead, err) case errors.Is(err, runtimepkg.ErrRuntimeClosed), errors.Is(err, mail.ErrClosed): return withCode(ErrRuntimeClosed, err) - case errors.Is(err, mail.ErrOverloaded): + case errors.Is(err, mail.ErrOverloaded), errors.Is(err, runtimepkg.ErrActivationLimit): return withCode(ErrOverloaded, err) case errors.Is(err, runtimepkg.ErrTypeNotRegistered): return withCode(ErrTypeNotInstalled, err) diff --git a/errors_test.go b/errors_test.go index 00348d3..d9096e3 100644 --- a/errors_test.go +++ b/errors_test.go @@ -4,8 +4,11 @@ import ( "context" "errors" "fmt" + "sync/atomic" "testing" + "testing/synctest" + runtimepkg "github.com/suraciii/gor/internal/runtime" "github.com/suraciii/gor/store" ) @@ -92,20 +95,21 @@ func TestJoinedErrorWithMultipleCodesIsOpaqueAcrossNodes(t *testing.T) { func TestInvokePreservesApplicationCodeAndMapsFrameworkCode(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installAccountWithDispatch(t, rt, func(ctx context.Context, instance Account, method string, args any, reply any) error { if method == "Deposit" { return fmt.Errorf("application failure: %w", testApplicationCode) } return dispatchAccount(ctx, instance, method, args, reply) }) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) - err := rt.Invoke(context.Background(), GrainId{GrainType: TypeName[Account](), GrainKey: "alice"}, "Deposit", &accountDepositRequest{}, &accountDepositReply{}) + err := rt.Invoke(context.Background(), GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"}, "Deposit", &accountDepositRequest{}, &accountDepositReply{}) if !errors.Is(err, testApplicationCode) { t.Fatalf("application error = %v, want application code", err) } @@ -113,8 +117,8 @@ func TestInvokePreservesApplicationCodeAndMapsFrameworkCode(t *testing.T) { t.Fatalf("CodeOf(application error) = (%q, %v), want (%q, true)", got, ok, testApplicationCode) } - rt.Close() - err = rt.Invoke(context.Background(), GrainId{GrainType: TypeName[Account](), GrainKey: "alice"}, "Balance", &accountBalanceRequest{}, &accountBalanceReply{}) + closeRuntime(rt) + err = rt.Invoke(context.Background(), GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"}, "Balance", &accountBalanceRequest{}, &accountBalanceReply{}) if !errors.Is(err, ErrRuntimeClosed) { t.Fatalf("closed runtime error = %v, want ErrRuntimeClosed", err) } @@ -122,20 +126,21 @@ func TestInvokePreservesApplicationCodeAndMapsFrameworkCode(t *testing.T) { func TestInvokeMapsMethodPanicToErrPanic(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installAccountWithDispatch(t, rt, func(ctx context.Context, instance Account, method string, args any, reply any) error { if method == "Deposit" { panic("method exploded") } return dispatchAccount(ctx, instance, method, args, reply) }) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) - err := rt.Invoke(context.Background(), GrainId{GrainType: TypeName[Account](), GrainKey: "alice"}, "Deposit", &accountDepositRequest{}, &accountDepositReply{}) + err := rt.Invoke(context.Background(), GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"}, "Deposit", &accountDepositRequest{}, &accountDepositReply{}) if !errors.Is(err, ErrPanic) { t.Fatalf("panic error = %v, want ErrPanic", err) } @@ -144,6 +149,32 @@ func TestInvokeMapsMethodPanicToErrPanic(t *testing.T) { } } +func TestInvokeMapsOnActivatePanicToErrPanic(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + factoryCalls := new(atomic.Int32) + rt := mustNew(t, WithMaxActivations(1), WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + installLifecycleAccount(t, rt, factoryCalls, func(grain *lifecycleAccountGrain) { + grain.activatePanic = true + }) + mustStart(t, rt) + + _, err := Ref[lifecycleAccount](rt, "alice").Value(context.Background()) + if !errors.Is(err, ErrPanic) { + t.Fatalf("OnActivate panic error = %v, want ErrPanic", err) + } + if got := rt.Activations(); len(got) != 0 { + t.Fatalf("Activations after OnActivate panic = %#v, want none", got) + } + if _, err := Ref[lifecycleAccount](rt, "bob").Value(context.Background()); !errors.Is(err, ErrPanic) { + t.Fatalf("second Grain after OnActivate panic = %v, want ErrPanic", err) + } + if got := factoryCalls.Load(); got != 2 { + t.Fatalf("factory calls after OnActivate panics = %d, want 2", got) + } + }) +} + func TestPublicErrorMapsPersistenceConflict(t *testing.T) { err := publicError(store.ErrConflict) if !errors.Is(err, ErrPersistenceConflict) { @@ -154,22 +185,33 @@ func TestPublicErrorMapsPersistenceConflict(t *testing.T) { } } +func TestPublicErrorMapsActivationLimit(t *testing.T) { + err := publicError(runtimepkg.ErrActivationLimit) + if !errors.Is(err, ErrOverloaded) { + t.Fatalf("publicError(ErrActivationLimit) = %v, want ErrOverloaded", err) + } + if got, ok := CodeOf(err); !ok || got != ErrOverloaded { + t.Fatalf("CodeOf(publicError(ErrActivationLimit)) = (%q, %v), want (%q, true)", got, ok, ErrOverloaded) + } +} + func TestInvokePreservesContextDeadlineExceeded(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installAccountWithDispatch(t, rt, func(ctx context.Context, instance Account, method string, args any, reply any) error { if method == "Deposit" { return context.DeadlineExceeded } return dispatchAccount(ctx, instance, method, args, reply) }) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) - err := rt.Invoke(context.Background(), GrainId{GrainType: TypeName[Account](), GrainKey: "alice"}, "Deposit", &accountDepositRequest{}, &accountDepositReply{}) + err := rt.Invoke(context.Background(), GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"}, "Deposit", &accountDepositRequest{}, &accountDepositReply{}) if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("deadline error = %v, want context.DeadlineExceeded", err) } diff --git a/examples/shadow/README.md b/examples/shadow/README.md index 03a26e8..a5f4225 100644 --- a/examples/shadow/README.md +++ b/examples/shadow/README.md @@ -1,6 +1,7 @@ # Device shadow example -A directly runnable device-shadow service: devices report state, the service keeps the last report; after more than 30 seconds without a new message, the device goes offline and its workshop's online count updates. +This service keeps the last report for each device. A device goes offline after +30 seconds without a report. Its Workshop Grain updates the online count. ## Why these things are Grains @@ -31,33 +32,36 @@ Run from the gor repository root: go run ./examples/shadow/cmd/shadow ``` -Registering the shadow Grains only needs the Runtime: - -```go -if err := shadow.Register(rt); err != nil { - return err -} -``` - -Reminders and lifecycle hooks have no requester waiting. When starting the -Runtime, install the unified error sink, or these errors are dropped: +Create the Runtime with explicit Stores. Install the error sink if the +Application must observe background errors: ```go rt, err := gor.New( gor.WithStore(database), + gor.WithReminderStore(database), gor.OnError(shadow.LogBackgroundError), ) +if err != nil { return err } + +if err := shadow.Register(rt); err != nil { return err } +if err := rt.Start(ctx); err != nil { return err } ``` -The service listens on `:8080` and writes data to `data/gor.db`. Address and database file are configurable: +The Runtime accepts no Call before `Start` succeeds. The Application calls +`Shutdown(ctx)` before it closes the database. + +The service listens on `:8080`. It writes data to `data/gor.db`. Both values +are configurable: ```bash go run ./examples/shadow/cmd/shadow -addr :9090 -db ./data/shadow.db ``` -## Running it on multiple nodes +## Cluster preview -The same service runs as a cluster. Start one process per node, all sharing one database file; each node takes a distinct HTTP address and cluster transport address, and a fresh membership generation is taken automatically on every start: +This preview is not part of the 0.1.0 product contract. Start one process for +each Silo. The Silos share one database file. Each Silo needs a unique HTTP +address and Cluster address. ```bash go run ./examples/shadow/cmd/shadow -cluster -addr 127.0.0.1:8081 -node-addr 127.0.0.1:7371 -db ./data/cluster.db @@ -65,14 +69,14 @@ go run ./examples/shadow/cmd/shadow -cluster -addr 127.0.0.1:8082 -node-addr 127 go run ./examples/shadow/cmd/shadow -cluster -addr 127.0.0.1:8083 -node-addr 127.0.0.1:7373 -db ./data/cluster.db ``` -Run each in its own terminal. Every node serves the same HTTP API. A request -is executed on the Silo that owns its Grain. The Grain definitions and -handlers are the same as in the single-Silo service. +Run each command in its own terminal. Every Silo serves the same HTTP API. +The owning Silo runs the Call. The Grain definitions and +handlers are the same as in the Single Silo service. -A node begins serving when it joins. As Silos discover each other, Grain +A Silo begins serving when it joins. As Silos discover each other, Grain ownership settles. During that window the same Grain may be active on two Silos. One State write can then fail with a conflict. This does not happen in -the single-Silo service. The full boundary is in +the Single Silo service. The full boundary is in [../../docs/programming-model.md](../../docs/programming-model.md). ## Calling it diff --git a/examples/shadow/cluster_net_test.go b/examples/shadow/cluster_net_test.go index 8edc541..d33c7ee 100644 --- a/examples/shadow/cluster_net_test.go +++ b/examples/shadow/cluster_net_test.go @@ -15,14 +15,14 @@ import ( "github.com/suraciii/gor/transport" ) -// TestCluster_ForwardsCallAndCrossEntityCall proves, over real loopback TCP, -// that the example's business code routes through the cluster with no change: -// a call entering a node that does not own the entity is forwarded to the -// owner, and an entity's call to another entity forwards the same way. Node B +// TestCluster_ForwardsCallAndCrossGrainCall proves, over real loopback TCP, +// that the example's business code routes through the Cluster with no change: +// a Call entering a node that does not own the Grain is forwarded to the +// owner. A Grain's Call to another Grain is forwarded in the same way. Node B // is started after A, so B's initial view already contains A; every routing // decision below is therefore the converged-ring decision, with no waiting on // wall-clock convergence. -func TestCluster_ForwardsCallAndCrossEntityCall(t *testing.T) { +func TestCluster_ForwardsCallAndCrossGrainCall(t *testing.T) { shared := store.NewMemory() transportA, err := transport.New("127.0.0.1:0") if err != nil { @@ -36,8 +36,8 @@ func TestCluster_ForwardsCallAndCrossEntityCall(t *testing.T) { rtA := mustClusterRuntime(t, shared, transportA, "generation-a") rtB := mustClusterRuntime(t, shared, transportB, "generation-b") - defer rtA.Close() - defer rtB.Close() + defer shutdownRuntime(rtA) + defer shutdownRuntime(rtB) ctx := context.Background() snapshot, err := shared.ListMembers(ctx) @@ -50,8 +50,8 @@ func TestCluster_ForwardsCallAndCrossEntityCall(t *testing.T) { view := cluster.NewView(snapshot.Members) addrA, addrB := transportA.Addr(), transportB.Addr() - deviceType := gor.TypeName[domain.Device]() - workshopType := gor.TypeName[domain.Workshop]() + deviceType := gor.GrainType("domain.Device") + workshopType := gor.GrainType("domain.Workshop") deviceOnA, ok := findKey(view, deviceType, addrA) if !ok { t.Fatal("no device key owned by A") @@ -65,7 +65,7 @@ func TestCluster_ForwardsCallAndCrossEntityCall(t *testing.T) { t.Fatal("no workshop key owned by A") } - // A call entering B for an entity owned by A is forwarded: A activates it, + // A Call entering B for a Grain owned by A is forwarded. A activates it, // B does not. The read through B forwards the same way and returns the data. if err := gor.Ref[domain.Device](rtB, deviceOnA).Report(ctx, "w", "t=20"); err != nil { t.Fatalf("report A-owned device via B: %v", err) @@ -88,7 +88,7 @@ func TestCluster_ForwardsCallAndCrossEntityCall(t *testing.T) { } // A device owned by B, reported through B, runs locally on B; its - // cross-entity call to a workshop owned by A is forwarded B -> A, so the + // cross-Grain Call to a workshop owned by A is forwarded B -> A, so the // workshop activates on A and a count read through B reaches A and sees it. if err := gor.Ref[domain.Device](rtB, deviceOnB).Report(ctx, workshopOnA, "t=21"); err != nil { t.Fatalf("report B-owned device via B: %v", err) @@ -97,7 +97,7 @@ func TestCluster_ForwardsCallAndCrossEntityCall(t *testing.T) { t.Fatal("B did not activate its own device locally") } if !hasActivation(rtA, workshopType, workshopOnA) { - t.Fatal("A did not activate the A-owned workshop; the cross-entity call was not forwarded") + t.Fatal("A did not activate the A-owned workshop; the cross-Grain Call was not forwarded") } count, err := gor.Ref[domain.Workshop](rtB, workshopOnA).OnlineCount(ctx) if err != nil { @@ -112,6 +112,7 @@ func mustClusterRuntime(t *testing.T, shared *store.Memory, tr *transport.TCP, g t.Helper() rt, err := gor.New( gor.WithStore(shared), + gor.WithReminderStore(shared), gor.WithMemberStore(shared), gor.WithNodeAddr(tr.Addr()), gor.WithGeneration(generation), @@ -123,16 +124,17 @@ func mustClusterRuntime(t *testing.T, shared *store.Memory, tr *transport.TCP, g t.Fatalf("new runtime: %v", err) } if err := shadow.Register(rt); err != nil { - rt.Close() + _ = crashRuntime(rt) t.Fatalf("register: %v", err) } + startRuntime(t, rt) return rt } -func findKey(view cluster.View, entityType, owner string) (string, bool) { +func findKey(view cluster.View, grainType gor.GrainType, owner string) (string, bool) { for i := 1; i <= 100000; i++ { key := fmt.Sprintf("probe-%06d", i) - got, ok := cluster.Owner(view, store.GrainId{GrainType: entityType, GrainKey: key}) + got, ok := cluster.Owner(view, store.GrainId{GrainType: string(grainType), GrainKey: key}) if ok && got == owner { return key, true } @@ -140,9 +142,9 @@ func findKey(view cluster.View, entityType, owner string) (string, bool) { return "", false } -func hasActivation(rt *gor.Runtime, entityType, key string) bool { +func hasActivation(rt *gor.Runtime, grainType gor.GrainType, key string) bool { for _, activation := range rt.Activations() { - if activation.GrainId.GrainType == entityType && activation.GrainId.GrainKey == key { + if activation.GrainId.GrainType == grainType && activation.GrainId.GrainKey == key { return true } } diff --git a/examples/shadow/cmd/conformance/external_module_configuration_test.go b/examples/shadow/cmd/conformance/external_module_configuration_test.go new file mode 100644 index 0000000..d417b85 --- /dev/null +++ b/examples/shadow/cmd/conformance/external_module_configuration_test.go @@ -0,0 +1,177 @@ +//go:build release + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/mod/modfile" +) + +func TestExternalModuleConfiguration(t *testing.T) { + repository := findRepositoryRoot(t) + tests := []struct { + name string + tagged bool + wantVersion string + wantReplace bool + }{ + {name: "candidate", wantVersion: "v0.0.0", wantReplace: true}, + {name: "tagged", tagged: true, wantVersion: "v0.1.0"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source := externalSource(test.tagged, repository) + if source.version != test.wantVersion { + t.Fatalf("module version = %q, want %q", source.version, test.wantVersion) + } + if got := source.replacement != ""; got != test.wantReplace { + t.Fatalf("source has replacement = %t, want %t", got, test.wantReplace) + } + content := externalModuleFile(t, source) + file, err := modfile.Parse("go.mod", content, nil) + if err != nil { + t.Fatalf("parse consumer go.mod: %v", err) + } + if got := len(file.Replace); (got != 0) != test.wantReplace { + t.Fatalf("replacement count = %d, want replacement = %t", got, test.wantReplace) + } + }) + } + if query := externalPackageQuery(externalSource(false, repository)); query != conformancePackage+"@v0.0.0" { + t.Fatalf("candidate package query = %q, want exact v0.0.0 query", query) + } + if query := externalPackageQuery(externalSource(true, repository)); query != conformancePackage+"@v0.1.0" { + t.Fatalf("tagged package query = %q, want exact v0.1.0 query", query) + } + previous := externalModuleSource{version: previousModuleVersion} + if query := externalPackageQueryFor(shadowCommandPackage, previous); query != shadowCommandPackage+"@v0.0.5" { + t.Fatalf("previous package query = %q, want exact v0.0.5 query", query) + } + if content := externalModuleFile(t, previous); strings.Contains(string(content), "replace ") { + t.Fatalf("previous module file has a replacement:\n%s", content) + } + + t.Run("resolved module rules", func(t *testing.T) { + moduleCache := t.TempDir() + taggedModuleDir := filepath.Join(moduleCache, "gor@v0.1.0") + if err := os.Mkdir(taggedModuleDir, 0o755); err != nil { + t.Fatalf("create tagged module directory: %v", err) + } + outsideModuleDir := t.TempDir() + taggedSource := externalSource(true, repository) + candidateSource := externalSource(false, repository) + tests := []struct { + name string + module resolvedModule + source externalModuleSource + wantErr string + }{ + { + name: "tagged", + module: resolvedModule{Path: conformanceModule, Version: "v0.1.0", Dir: taggedModuleDir}, + source: taggedSource, + }, + { + name: "candidate", + module: resolvedModule{ + Path: conformanceModule, + Version: candidateModuleVersion, + Dir: repository, + Replace: &resolvedModule{Dir: repository}, + }, + source: candidateSource, + }, + { + name: "wrong tagged version", + module: resolvedModule{Path: conformanceModule, Version: "v0.1.1", Dir: filepath.Join(moduleCache, "gor@v0.1.1")}, + source: taggedSource, + wantErr: "version", + }, + { + name: "tagged replacement", + module: resolvedModule{ + Path: conformanceModule, + Version: "v0.1.0", + Dir: taggedModuleDir, + Replace: &resolvedModule{Dir: repository}, + }, + source: taggedSource, + wantErr: "replacement", + }, + { + name: "tagged source outside cache", + module: resolvedModule{Path: conformanceModule, Version: "v0.1.0", Dir: outsideModuleDir}, + source: taggedSource, + wantErr: "outside module cache", + }, + { + name: "candidate without replacement", + module: resolvedModule{Path: conformanceModule, Version: candidateModuleVersion, Dir: repository}, + source: candidateSource, + wantErr: "no replacement", + }, + { + name: "empty tagged directory", + module: resolvedModule{Path: conformanceModule, Version: "v0.1.0"}, + source: taggedSource, + wantErr: "directory is empty", + }, + { + name: "relative tagged directory", + module: resolvedModule{Path: conformanceModule, Version: "v0.1.0", Dir: "module-cache/gor@v0.1.0"}, + source: taggedSource, + wantErr: "not absolute", + }, + { + name: "tagged cache root", + module: resolvedModule{Path: conformanceModule, Version: "v0.1.0", Dir: moduleCache}, + source: taggedSource, + wantErr: "outside module cache", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := validateResolvedModule(test.module, test.source, repository, moduleCache) + if test.wantErr == "" { + if err != nil { + t.Fatalf("validate resolved module: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("validate resolved module = %v, want error containing %q", err, test.wantErr) + } + }) + } + }) + + t.Run("trusted module environment", func(t *testing.T) { + environment := environmentValues(externalGoEnvironment("module-cache", "build-cache")) + want := map[string]string{ + "GOENV": "off", + "GONOPROXY": "none", + "GONOSUMDB": "none", + "GOPRIVATE": "none", + "GOPROXY": "https://goproxy.cn|https://proxy.golang.org|direct", + "GOSUMDB": "sum.golang.google.cn", + } + for key, value := range want { + if environment[key] != value { + t.Errorf("%s = %q, want %q", key, environment[key], value) + } + } + }) +} + +func environmentValues(environment []string) map[string]string { + values := make(map[string]string, len(environment)) + for _, entry := range environment { + key, value, _ := strings.Cut(entry, "=") + values[strings.ToUpper(key)] = value + } + return values +} diff --git a/examples/shadow/cmd/conformance/external_release_test.go b/examples/shadow/cmd/conformance/external_release_test.go new file mode 100644 index 0000000..3596f8b --- /dev/null +++ b/examples/shadow/cmd/conformance/external_release_test.go @@ -0,0 +1,539 @@ +//go:build release + +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/suraciii/gor/examples/shadow/domain" + "golang.org/x/mod/modfile" +) + +const ( + consumerModule = "example.com/gor-release-check" + conformanceModule = "github.com/suraciii/gor" + conformancePackage = conformanceModule + "/examples/shadow/cmd/conformance" + shadowCommandPackage = conformanceModule + "/examples/shadow/cmd/shadow" + previousModuleVersion = "v0.0.5" + candidateModuleVersion = "v0.0.0" + taggedModuleVersion = "v0.1.0" +) + +var externalTagged = flag.Bool("external-tagged", false, "verify the exact v0.1.0 module without a local replacement") + +type externalModuleSource struct { + version string + replacement string +} + +type resolvedModule struct { + Path string + Version string + Dir string + Replace *resolvedModule +} + +type processFixture struct { + runtimePath string + businessPath string + deviceKey string + actionID string + state string + traceID string +} + +func TestExternalModuleRestartProof(t *testing.T) { + if err := run(context.Background(), []string{"-phase", phasePrepare, "-claim-barrier"}); err == nil || !strings.Contains(err.Error(), "requires -phase recover") { + t.Fatalf("run with prepare Claim barrier = %v, want phase error", err) + } + repository := findRepositoryRoot(t) + source := externalSource(*externalTagged, repository) + work := t.TempDir() + moduleDir := filepath.Join(work, "consumer") + moduleCache := filepath.Join(work, "module-cache") + buildCache := filepath.Join(work, "build-cache") + if relative, err := filepath.Rel(repository, moduleDir); err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(os.PathSeparator)) { + t.Fatalf("consumer module %s is inside repository %s", moduleDir, repository) + } + for _, directory := range []string{moduleDir, moduleCache, buildCache} { + if err := os.MkdirAll(directory, 0o755); err != nil { + t.Fatalf("create %s: %v", directory, err) + } + } + + if err := os.WriteFile(filepath.Join(moduleDir, "go.mod"), externalModuleFile(t, source), 0o600); err != nil { + t.Fatalf("write consumer go.mod: %v", err) + } + if source.replacement == "" { + t.Logf("resolve exact module %s without a local replacement", source.version) + } + + binary := filepath.Join(work, "gor-conformance"+executableSuffix()) + environment := externalGoEnvironment(moduleCache, buildCache) + buildContext, cancelBuild := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancelBuild() + output, err := runExternalCommand(buildContext, moduleDir, environment, "go", "get", externalPackageQuery(source)) + if err != nil { + t.Fatalf("resolve conformance command in clean consumer module: %v\n%s", err, output) + } + jsonOutput, stderr, err := runExternalJSONCommand(buildContext, moduleDir, environment, "go", "list", "-m", "-json", conformanceModule) + if err != nil { + t.Fatalf("read resolved conformance module: %v\n%s%s", err, jsonOutput, stderr) + } + if stderr != "" { + t.Logf("go list stderr:\n%s", stderr) + } + assertResolvedModule(t, jsonOutput, source, repository, moduleCache) + output, err = runExternalCommand(buildContext, moduleDir, environment, "go", "build", "-tags", "release", "-o", binary, conformancePackage) + if err != nil { + t.Fatalf("build conformance command from clean consumer module: %v\n%s", err, output) + } + if buildContext.Err() != nil { + t.Fatalf("build conformance command: %v", buildContext.Err()) + } + assertDirectoryNotEmpty(t, moduleCache) + assertDirectoryNotEmpty(t, buildCache) + + t.Run("graceful restart", func(t *testing.T) { + fixture := newProcessFixture(t, "graceful") + prepare := runConformance(t, binary, fixture, phasePrepare) + if !strings.Contains(prepare, `prepared ActionID "graceful-action"`) { + t.Fatalf("prepare output does not contain the success record:\n%s", prepare) + } + assertDurableFiles(t, fixture) + + recoverOutput := runConformance(t, binary, fixture, phaseRecover) + if !strings.Contains(recoverOutput, `recovered ActionID "graceful-action"`) { + t.Fatalf("recover output does not contain the success record:\n%s", recoverOutput) + } + }) + + t.Run("claim stop restart", func(t *testing.T) { + fixture := newProcessFixture(t, "claim-stop") + runConformance(t, binary, fixture, phasePrepare) + killAfterClaim(t, binary, fixture) + assertPendingAfterClaimStop(t, fixture) + + recoverOutput := runConformance(t, binary, fixture, phaseRecover) + if !strings.Contains(recoverOutput, `recovered ActionID "claim-stop-action"`) { + t.Fatalf("recover output does not contain the success record:\n%s", recoverOutput) + } + }) + + t.Run("receipt mismatch fails", func(t *testing.T) { + fixture := newProcessFixture(t, "mismatch") + runConformance(t, binary, fixture, phasePrepare) + wrongExpectation := fixture + wrongExpectation.state = "temperature=99" + output := runConformanceFailure(t, binary, wrongExpectation, phaseRecover) + if !strings.Contains(output, "applied record =") { + t.Fatalf("recover output does not contain the receipt mismatch:\n%s", output) + } + }) + + t.Run("fresh recover fails", func(t *testing.T) { + fixture := newProcessFixture(t, "fresh") + output := runConformanceFailure(t, binary, fixture, phaseRecover) + if !strings.Contains(output, `ActionID "fresh-action" has no applied record`) { + t.Fatalf("recover output does not contain the missing-record error:\n%s", output) + } + }) +} + +func externalSource(tagged bool, repository string) externalModuleSource { + if tagged { + return externalModuleSource{version: taggedModuleVersion} + } + return externalModuleSource{ + version: candidateModuleVersion, + replacement: filepath.ToSlash(repository), + } +} + +func externalPackageQuery(source externalModuleSource) string { + return externalPackageQueryFor(conformancePackage, source) +} + +func externalPackageQueryFor(packagePath string, source externalModuleSource) string { + return packagePath + "@" + source.version +} + +func externalModuleFile(t *testing.T, source externalModuleSource) []byte { + t.Helper() + file := new(modfile.File) + if err := file.AddModuleStmt(consumerModule); err != nil { + t.Fatalf("set consumer module: %v", err) + } + if err := file.AddGoStmt("1.25.0"); err != nil { + t.Fatalf("set consumer Go version: %v", err) + } + if err := file.AddRequire(conformanceModule, source.version); err != nil { + t.Fatalf("require conformance module: %v", err) + } + if source.replacement != "" { + if err := file.AddReplace(conformanceModule, "", source.replacement, ""); err != nil { + t.Fatalf("replace candidate module: %v", err) + } + } + content, err := file.Format() + if err != nil { + t.Fatalf("format consumer go.mod: %v", err) + } + return content +} + +func assertResolvedModule(t *testing.T, output []byte, source externalModuleSource, repository, moduleCache string) { + t.Helper() + var module resolvedModule + if err := json.Unmarshal(output, &module); err != nil { + t.Fatalf("decode resolved conformance module: %v\n%s", err, output) + } + if err := validateResolvedModule(module, source, repository, moduleCache); err != nil { + t.Fatal(err) + } +} + +func validateResolvedModule(module resolvedModule, source externalModuleSource, repository, moduleCache string) error { + if module.Path != conformanceModule { + return fmt.Errorf("resolved module path = %q, want %q", module.Path, conformanceModule) + } + if module.Version != source.version { + return fmt.Errorf("resolved module version = %q, want %q", module.Version, source.version) + } + if module.Dir == "" { + return errors.New("resolved module directory is empty") + } + if !filepath.IsAbs(module.Dir) { + return fmt.Errorf("resolved module directory %q is not absolute", module.Dir) + } + if source.replacement == "" { + if module.Replace != nil { + return fmt.Errorf("resolved tagged module has replacement: %#v", module.Replace) + } + if !pathWithin(moduleCache, module.Dir) { + return fmt.Errorf("resolved tagged module directory %q is outside module cache %q", module.Dir, moduleCache) + } + if samePath(repository, module.Dir) || pathWithin(repository, module.Dir) { + return fmt.Errorf("resolved tagged module directory %q is inside repository %q", module.Dir, repository) + } + return nil + } + if module.Replace == nil { + return errors.New("resolved candidate module has no replacement") + } + if !samePath(module.Replace.Dir, repository) || !samePath(module.Dir, repository) { + return fmt.Errorf("resolved candidate directories = Dir %q, Replace.Dir %q, want repository %q", module.Dir, module.Replace.Dir, repository) + } + return nil +} + +func samePath(left, right string) bool { + left, err := canonicalPath(left) + if err != nil { + return false + } + right, err = canonicalPath(right) + if err != nil { + return false + } + relative, err := filepath.Rel(left, right) + return err == nil && relative == "." +} + +func pathWithin(parent, child string) bool { + parent, err := canonicalPath(parent) + if err != nil { + return false + } + child, err = canonicalPath(child) + if err != nil { + return false + } + relative, err := filepath.Rel(parent, child) + return err == nil && relative != "." && relative != ".." && !strings.HasPrefix(relative, ".."+string(os.PathSeparator)) +} + +func canonicalPath(path string) (string, error) { + absolute, err := filepath.Abs(path) + if err != nil { + return "", err + } + return filepath.EvalSymlinks(absolute) +} + +func newProcessFixture(t *testing.T, name string) processFixture { + t.Helper() + directory := t.TempDir() + return processFixture{ + runtimePath: filepath.Join(directory, "runtime.db"), + businessPath: filepath.Join(directory, "application.db"), + deviceKey: name + "-device", + actionID: name + "-action", + state: "temperature=21", + traceID: name + "-trace", + } +} + +func runConformance(t *testing.T, binary string, fixture processFixture, phase string) string { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + output, err := runExternalCommand(ctx, "", nil, binary, conformanceArgs(fixture, phase)...) + if err != nil { + t.Fatalf("run %s process: %v\n%s", phase, err, output) + } + if ctx.Err() != nil { + t.Fatalf("run %s process: %v\n%s", phase, ctx.Err(), output) + } + return output +} + +func runConformanceFailure(t *testing.T, binary string, fixture processFixture, phase string) string { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + output, err := runExternalCommand(ctx, "", nil, binary, conformanceArgs(fixture, phase)...) + if err == nil { + t.Fatalf("run %s process succeeded, want nonzero exit:\n%s", phase, output) + } + if ctx.Err() != nil { + t.Fatalf("run %s process timed out: %v\n%s", phase, ctx.Err(), output) + } + var exitError *exec.ExitError + if !errors.As(err, &exitError) || exitError.ExitCode() == 0 { + t.Fatalf("run %s error = %v, want nonzero process exit\n%s", phase, err, output) + } + return output +} + +func killAfterClaim(t *testing.T, binary string, fixture processFixture) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + args := append(conformanceArgs(fixture, phaseRecover), "-timeout", "24h", "-claim-barrier") + command := exec.CommandContext(ctx, binary, args...) + stdout, err := command.StdoutPipe() + if err != nil { + t.Fatalf("open Claim process output: %v", err) + } + var stderr bytes.Buffer + command.Stderr = &stderr + if err := command.Start(); err != nil { + t.Fatalf("start Claim process: %v", err) + } + + marker := make(chan struct{}, 1) + scanDone := make(chan string, 1) + go func() { + var output strings.Builder + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + line := scanner.Text() + fmt.Fprintln(&output, line) + if line == claimBarrierReady { + marker <- struct{}{} + } + } + if err := scanner.Err(); err != nil { + fmt.Fprintf(&output, "read stdout: %v\n", err) + } + scanDone <- output.String() + }() + waitDone := make(chan error, 1) + go func() { waitDone <- command.Wait() }() + + select { + case <-marker: + case waitErr := <-waitDone: + stdoutText := <-scanDone + t.Fatalf("Claim process exited before the barrier: %v\n%s%s", waitErr, stdoutText, stderr.String()) + case <-ctx.Done(): + waitErr := <-waitDone + stdoutText := <-scanDone + t.Fatalf("wait for Claim barrier: %v; process error: %v\n%s%s", ctx.Err(), waitErr, stdoutText, stderr.String()) + } + + if err := command.Process.Kill(); err != nil { + t.Fatalf("stop process after Claim: %v", err) + } + waitErr := <-waitDone + stdoutText := <-scanDone + if ctx.Err() != nil { + t.Fatalf("Claim process was stopped by the timeout, not the parent: %v\n%s%s", ctx.Err(), stdoutText, stderr.String()) + } + if waitErr == nil { + t.Fatalf("Claim process exited successfully after Process.Kill\n%s%s", stdoutText, stderr.String()) + } + var exitError *exec.ExitError + if !errors.As(waitErr, &exitError) || exitError.ExitCode() == 0 { + t.Fatalf("Claim process error = %v, want nonzero killed exit\n%s%s", waitErr, stdoutText, stderr.String()) + } +} + +func conformanceArgs(fixture processFixture, phase string) []string { + return []string{ + "-phase", phase, + "-db", fixture.runtimePath, + "-business-db", fixture.businessPath, + "-device", fixture.deviceKey, + "-action-id", fixture.actionID, + "-state", fixture.state, + "-trace-id", fixture.traceID, + "-timeout", "15s", + } +} + +func assertDurableFiles(t *testing.T, fixture processFixture) { + t.Helper() + for _, path := range []string{fixture.runtimePath, derivedRuntimeStatePath(fixture.runtimePath), fixture.businessPath} { + info, err := os.Stat(path) + if err != nil { + t.Fatalf("read durable file %s: %v", path, err) + } + if info.Size() == 0 { + t.Fatalf("durable file %s is empty", path) + } + } +} + +func assertPendingAfterClaimStop(t *testing.T, fixture processFixture) { + t.Helper() + application, err := domain.OpenSQLiteApplicationStore(fixture.businessPath) + if err != nil { + t.Fatalf("open Application Store after Claim stop: %v", err) + } + defer func() { + if err := application.Close(); err != nil { + t.Errorf("close Application Store after Claim stop: %v", err) + } + }() + + if record, applied, err := application.ReadApplied(context.Background(), fixture.actionID); err != nil { + t.Fatalf("read receipt after Claim stop: %v", err) + } else if applied { + t.Fatalf("receipt exists after Claim stop: %#v", record) + } + pending, err := application.ListPending(context.Background()) + if err != nil { + t.Fatalf("read pending action after Claim stop: %v", err) + } + want := domain.PendingAction{ + ActionID: fixture.actionID, + DeviceKey: fixture.deviceKey, + State: fixture.state, + TraceID: fixture.traceID, + } + if len(pending) != 1 || pending[0] != want { + t.Fatalf("pending actions after Claim stop = %#v, want %#v", pending, []domain.PendingAction{want}) + } +} + +func assertDirectoryNotEmpty(t *testing.T, directory string) { + t.Helper() + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatalf("read cache directory %s: %v", directory, err) + } + if len(entries) == 0 { + t.Fatalf("cache directory %s is empty", directory) + } +} + +func runExternalCommand(ctx context.Context, directory string, environment []string, name string, args ...string) (string, error) { + command := exec.CommandContext(ctx, name, args...) + command.Dir = directory + if environment != nil { + command.Env = environment + } + output, err := command.CombinedOutput() + return string(output), err +} + +func runExternalJSONCommand(ctx context.Context, directory string, environment []string, name string, args ...string) ([]byte, string, error) { + command := exec.CommandContext(ctx, name, args...) + command.Dir = directory + command.Env = environment + var stdout bytes.Buffer + var stderr bytes.Buffer + command.Stdout = &stdout + command.Stderr = &stderr + err := command.Run() + return stdout.Bytes(), stderr.String(), err +} + +func externalGoEnvironment(moduleCache, buildCache string) []string { + blocked := map[string]bool{ + "GOCACHE": true, + "GOENV": true, + "GOFLAGS": true, + "GOMODCACHE": true, + "GONOPROXY": true, + "GONOSUMDB": true, + "GOPRIVATE": true, + "GOPROXY": true, + "GOSUMDB": true, + "GOTOOLCHAIN": true, + "GOWORK": true, + } + environment := make([]string, 0, len(os.Environ())+11) + for _, entry := range os.Environ() { + key, _, _ := strings.Cut(entry, "=") + if !blocked[strings.ToUpper(key)] { + environment = append(environment, entry) + } + } + return append(environment, + "GOCACHE="+buildCache, + "GOENV=off", + "GOFLAGS=-modcacherw", + "GOMODCACHE="+moduleCache, + "GONOPROXY=none", + "GONOSUMDB=none", + "GOPRIVATE=none", + "GOPROXY=https://goproxy.cn|https://proxy.golang.org|direct", + "GOSUMDB=sum.golang.google.cn", + "GOTOOLCHAIN=local", + "GOWORK=off", + ) +} + +func findRepositoryRoot(t *testing.T) string { + t.Helper() + _, source, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("find release test source path") + } + directory := filepath.Dir(source) + for { + goMod := filepath.Join(directory, "go.mod") + content, err := os.ReadFile(goMod) + if err == nil && strings.Contains(string(content), "module github.com/suraciii/gor") { + return directory + } + parent := filepath.Dir(directory) + if parent == directory { + t.Fatalf("find repository root from %s", source) + } + directory = parent + } +} + +func executableSuffix() string { + if runtime.GOOS == "windows" { + return ".exe" + } + return "" +} diff --git a/examples/shadow/cmd/conformance/external_upgrade_test.go b/examples/shadow/cmd/conformance/external_upgrade_test.go new file mode 100644 index 0000000..63b87ac --- /dev/null +++ b/examples/shadow/cmd/conformance/external_upgrade_test.go @@ -0,0 +1,480 @@ +//go:build release + +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/suraciii/gor/store" +) + +const upgradeReadyMarker = "device shadow listening on " + +const previousWriterSource = `package main + +import ( + "context" + "fmt" + "log" + "os" + "time" + + "github.com/suraciii/gor" + shadow "github.com/suraciii/gor/examples/shadow" + "github.com/suraciii/gor/examples/shadow/domain" + "github.com/suraciii/gor/store" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() (runErr error) { + if len(os.Args) != 2 { + return fmt.Errorf("usage: previous-writer DATABASE") + } + database, err := store.OpenSQLite(os.Args[1]) + if err != nil { + return err + } + defer func() { + if err := database.Close(); err != nil && runErr == nil { + runErr = err + } + }() + runtime, err := gor.New(gor.WithStore(database), gor.WithScheduleInterval(0)) + if err != nil { + return err + } + defer runtime.Close() + if err := shadow.Register(runtime); err != nil { + return err + } + ctx := context.Background() + device := gor.Ref[domain.Device](runtime, "device-1") + if err := device.Report(ctx, "assembly", "temperature=20"); err != nil { + return err + } + if err := device.Configure(ctx, "sample-rate=10s"); err != nil { + return err + } + if err := database.Put(ctx, store.Schedule{ + Identity: store.Identity{Type: "domain.Device", Key: "device-1"}, + Name: "offline", Method: "MarkOffline", + DueAt: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), + }); err != nil { + return err + } + fmt.Println("wrote exact v0.0.5 Shadow upgrade fixture") + return nil +} +` + +type externalShadowProcess struct { + address string + command *exec.Cmd + cancel context.CancelFunc + stdout *bytes.Buffer + waitDone <-chan error + stderrDone <-chan string + stopped bool +} + +type upgradeShadow struct { + ReportedState string `json:"reported_state"` + ReportedAt time.Time `json:"reported_at"` + Online bool `json:"online"` + WorkshopID string `json:"workshop_id"` + Configuration string `json:"configuration"` +} + +func TestExternalModuleUpgradeProof(t *testing.T) { + repository := findRepositoryRoot(t) + work := t.TempDir() + moduleDir := filepath.Join(work, "consumer") + moduleCache := filepath.Join(work, "module-cache") + buildCache := filepath.Join(work, "build-cache") + for _, directory := range []string{moduleDir, moduleCache, buildCache} { + if err := os.MkdirAll(directory, 0o755); err != nil { + t.Fatalf("create %s: %v", directory, err) + } + } + + environment := externalGoEnvironment(moduleCache, buildCache) + buildContext, cancelBuild := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancelBuild() + previousSource := externalModuleSource{version: previousModuleVersion} + t.Logf("build upgrade source from exact %s without a replacement", previousModuleVersion) + previousBinary := filepath.Join(work, "previous-writer"+executableSuffix()) + buildExternalPreviousWriter(t, buildContext, moduleDir, environment, repository, moduleCache, previousSource, previousBinary) + candidateSource := externalSource(*externalTagged, repository) + t.Logf("build upgrade target %s", candidateSource.version) + candidateBinary := filepath.Join(work, "shadow-candidate"+executableSuffix()) + buildExternalShadow(t, buildContext, moduleDir, environment, repository, moduleCache, candidateSource, candidateBinary) + + databasePath := filepath.Join(work, "data", "gor.db") + if err := os.MkdirAll(filepath.Dir(databasePath), 0o755); err != nil { + t.Fatalf("create upgrade database directory: %v", err) + } + runExternalPreviousWriter(t, previousBinary, databasePath) + client := &http.Client{Timeout: 5 * time.Second} + + candidate := startExternalShadow(t, candidateBinary, databasePath) + initial := assertUpgradeShadow(t, client, candidate.address, upgradeShadow{ + ReportedState: "temperature=20", + Online: true, + WorkshopID: "assembly", + Configuration: "sample-rate=10s", + }) + assertUpgradeOnlineCount(t, client, candidate.address, 1) + stopExternalShadow(t, candidate) + assertUpgradeReminder(t, databasePath) + + updated := startExternalShadow(t, candidateBinary, databasePath) + writeUpgradeConfiguration(t, client, updated.address, "sample-rate=30s") + assertUpgradeShadow(t, client, updated.address, upgradeShadow{ + ReportedState: "temperature=20", + ReportedAt: initial.ReportedAt, + Online: true, + WorkshopID: "assembly", + Configuration: "sample-rate=30s", + }) + assertUpgradeOnlineCount(t, client, updated.address, 1) + stopExternalShadow(t, updated) + + restarted := startExternalShadow(t, candidateBinary, databasePath) + assertUpgradeShadow(t, client, restarted.address, upgradeShadow{ + ReportedState: "temperature=20", + ReportedAt: initial.ReportedAt, + Online: true, + WorkshopID: "assembly", + Configuration: "sample-rate=30s", + }) + assertUpgradeOnlineCount(t, client, restarted.address, 1) + stopExternalShadow(t, restarted) + assertDirectoryNotEmpty(t, moduleCache) + assertDirectoryNotEmpty(t, buildCache) +} + +func buildExternalPreviousWriter(t *testing.T, ctx context.Context, moduleDir string, environment []string, repository, moduleCache string, source externalModuleSource, binary string) { + t.Helper() + resolveExternalPackage(t, ctx, moduleDir, environment, repository, moduleCache, source, shadowCommandPackage) + writerDir := filepath.Join(moduleDir, "previous-writer") + if err := os.MkdirAll(writerDir, 0o755); err != nil { + t.Fatalf("create previous writer: %v", err) + } + if err := os.WriteFile(filepath.Join(writerDir, "main.go"), []byte(previousWriterSource), 0o600); err != nil { + t.Fatalf("write previous writer: %v", err) + } + output, err := runExternalCommand(ctx, moduleDir, environment, "go", "build", "-buildvcs=false", "-o", binary, "./previous-writer") + if err != nil { + t.Fatalf("build previous writer %s: %v\n%s", source.version, err, output) + } +} + +func buildExternalShadow(t *testing.T, ctx context.Context, moduleDir string, environment []string, repository, moduleCache string, source externalModuleSource, binary string) { + t.Helper() + resolveExternalPackage(t, ctx, moduleDir, environment, repository, moduleCache, source, shadowCommandPackage) + output, err := runExternalCommand(ctx, moduleDir, environment, "go", "build", "-o", binary, shadowCommandPackage) + if err != nil { + t.Fatalf("build Shadow command %s: %v\n%s", source.version, err, output) + } + if ctx.Err() != nil { + t.Fatalf("build Shadow command %s: %v", source.version, ctx.Err()) + } +} + +func resolveExternalPackage(t *testing.T, ctx context.Context, moduleDir string, environment []string, repository, moduleCache string, source externalModuleSource, packagePath string) { + t.Helper() + if err := os.WriteFile(filepath.Join(moduleDir, "go.mod"), externalModuleFile(t, source), 0o600); err != nil { + t.Fatalf("write %s consumer go.mod: %v", source.version, err) + } + query := externalPackageQueryFor(packagePath, source) + output, err := runExternalCommand(ctx, moduleDir, environment, "go", "get", query) + if err != nil { + t.Fatalf("resolve Shadow command %s: %v\n%s", query, err, output) + } + jsonOutput, stderr, err := runExternalJSONCommand(ctx, moduleDir, environment, "go", "list", "-m", "-json", conformanceModule) + if err != nil { + t.Fatalf("read resolved Shadow module %s: %v\n%s%s", source.version, err, jsonOutput, stderr) + } + assertResolvedModule(t, jsonOutput, source, repository, moduleCache) + if ctx.Err() != nil { + t.Fatalf("resolve package %s: %v", query, ctx.Err()) + } +} + +func runExternalPreviousWriter(t *testing.T, binary, databasePath string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + output, err := runExternalCommand(ctx, "", nil, binary, databasePath) + if err != nil { + t.Fatalf("run exact %s writer: %v\n%s", previousModuleVersion, err, output) + } + if ctx.Err() != nil { + t.Fatalf("run exact %s writer: %v\n%s", previousModuleVersion, ctx.Err(), output) + } + if !strings.Contains(output, "wrote exact v0.0.5 Shadow upgrade fixture") { + t.Fatalf("exact %s writer output does not contain completion marker:\n%s", previousModuleVersion, output) + } +} + +func startExternalShadow(t *testing.T, binary, databasePath string) *externalShadowProcess { + t.Helper() + processContext, cancel := context.WithTimeout(context.Background(), 30*time.Second) + command := exec.CommandContext(processContext, binary, "-addr", "127.0.0.1:0", "-db", databasePath) + stdout := new(bytes.Buffer) + command.Stdout = stdout + stderr, err := command.StderrPipe() + if err != nil { + cancel() + t.Fatalf("open Shadow process output: %v", err) + } + if err := command.Start(); err != nil { + cancel() + t.Fatalf("start Shadow process: %v", err) + } + + ready := make(chan string, 1) + stderrDone := make(chan string, 1) + go func() { + var output strings.Builder + scanner := bufio.NewScanner(stderr) + reportedReady := false + for scanner.Scan() { + line := scanner.Text() + fmt.Fprintln(&output, line) + if !reportedReady { + index := strings.Index(line, upgradeReadyMarker) + if index >= 0 { + address := line[index+len(upgradeReadyMarker):] + if end := strings.Index(address, " with database "); end >= 0 { + reportedReady = true + ready <- address[:end] + } + } + } + } + if err := scanner.Err(); err != nil { + fmt.Fprintf(&output, "read stderr: %v\n", err) + } + stderrDone <- output.String() + }() + waitDone := make(chan error, 1) + go func() { waitDone <- command.Wait() }() + + process := &externalShadowProcess{ + command: command, + cancel: cancel, + stdout: stdout, + waitDone: waitDone, + stderrDone: stderrDone, + } + t.Cleanup(func() { + if process.stopped { + return + } + _ = process.command.Process.Kill() + <-process.waitDone + <-process.stderrDone + process.cancel() + process.stopped = true + }) + select { + case process.address = <-ready: + case waitErr := <-waitDone: + stderrText := <-stderrDone + cancel() + process.stopped = true + t.Fatalf("Shadow process exited before readiness: %v\n%s%s", waitErr, stdout.String(), stderrText) + case <-processContext.Done(): + waitErr := <-waitDone + stderrText := <-stderrDone + cancel() + process.stopped = true + t.Fatalf("wait for Shadow readiness: %v; process error: %v\n%s%s", processContext.Err(), waitErr, stdout.String(), stderrText) + } + waitForUpgradeHTTP(t, processContext, process) + return process +} + +func waitForUpgradeHTTP(t *testing.T, ctx context.Context, process *externalShadowProcess) { + t.Helper() + client := &http.Client{Timeout: 250 * time.Millisecond} + url := "http://" + process.address + "/devices/readiness/shadow" + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + response, err := client.Get(url) + if err == nil { + _, readErr := io.ReadAll(response.Body) + response.Body.Close() + if readErr == nil && response.StatusCode == http.StatusOK { + return + } + } + select { + case waitErr := <-process.waitDone: + stderrText := <-process.stderrDone + process.cancel() + process.stopped = true + t.Fatalf("Shadow process exited before HTTP readiness: %v\n%s%s", waitErr, process.stdout.String(), stderrText) + case <-ctx.Done(): + waitErr := <-process.waitDone + stderrText := <-process.stderrDone + process.cancel() + process.stopped = true + t.Fatalf("wait for Shadow HTTP readiness: %v; process error: %v\n%s%s", ctx.Err(), waitErr, process.stdout.String(), stderrText) + case <-ticker.C: + } + } +} + +func stopExternalShadow(t *testing.T, process *externalShadowProcess) { + t.Helper() + if err := process.command.Process.Kill(); err != nil { + t.Fatalf("stop Shadow process: %v", err) + } + waitErr := <-process.waitDone + stderrText := <-process.stderrDone + process.cancel() + process.stopped = true + if waitErr == nil { + t.Fatalf("Shadow process exited successfully after Process.Kill\n%s%s", process.stdout.String(), stderrText) + } + var exitError *exec.ExitError + if !errors.As(waitErr, &exitError) || exitError.ExitCode() == 0 { + t.Fatalf("Shadow process error = %v, want nonzero killed exit\n%s%s", waitErr, process.stdout.String(), stderrText) + } +} + +func writeUpgradeConfiguration(t *testing.T, client *http.Client, address, configuration string) { + t.Helper() + request, err := http.NewRequest(http.MethodPut, "http://"+address+"/devices/device-1/configuration", + strings.NewReader(fmt.Sprintf(`{"configuration":%q}`, configuration))) + if err != nil { + t.Fatalf("create configuration request: %v", err) + } + request.Header.Set("Content-Type", "application/json") + response, err := client.Do(request) + assertUpgradeResponse(t, response, err, http.StatusNoContent, "write configuration") +} + +func assertUpgradeShadow(t *testing.T, client *http.Client, address string, want upgradeShadow) upgradeShadow { + t.Helper() + response, err := client.Get("http://" + address + "/devices/device-1/shadow") + if err != nil { + t.Fatalf("read upgraded Shadow: %v", err) + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatalf("read upgraded Shadow body: %v", err) + } + if response.StatusCode != http.StatusOK { + t.Fatalf("read upgraded Shadow status = %d, want %d; body = %q", response.StatusCode, http.StatusOK, body) + } + var got upgradeShadow + if err := json.Unmarshal(body, &got); err != nil { + t.Fatalf("decode upgraded Shadow: %v; body = %q", err, body) + } + if got.ReportedAt.IsZero() { + t.Fatalf("upgraded Shadow has zero ReportedAt: %#v", got) + } + if !want.ReportedAt.IsZero() && !got.ReportedAt.Equal(want.ReportedAt) { + t.Fatalf("upgraded Shadow ReportedAt = %v, want %v", got.ReportedAt, want.ReportedAt) + } + comparableGot, comparableWant := got, want + comparableGot.ReportedAt = time.Time{} + comparableWant.ReportedAt = time.Time{} + if comparableGot != comparableWant { + t.Fatalf("upgraded Shadow = %#v, want %#v", got, want) + } + return got +} + +func assertUpgradeOnlineCount(t *testing.T, client *http.Client, address string, want int) { + t.Helper() + response, err := client.Get("http://" + address + "/workshops/assembly/online-count") + if err != nil { + t.Fatalf("read upgraded Workshop: %v", err) + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatalf("read upgraded Workshop body: %v", err) + } + if response.StatusCode != http.StatusOK { + t.Fatalf("read upgraded Workshop status = %d, want %d; body = %q", response.StatusCode, http.StatusOK, body) + } + var value struct { + OnlineCount int `json:"online_count"` + } + if err := json.Unmarshal(body, &value); err != nil { + t.Fatalf("decode upgraded Workshop: %v; body = %q", err, body) + } + if value.OnlineCount != want { + t.Fatalf("upgraded Workshop online count = %d, want %d", value.OnlineCount, want) + } +} + +func assertUpgradeReminder(t *testing.T, databasePath string) { + t.Helper() + database, err := store.OpenSQLite(databasePath) + if err != nil { + t.Fatalf("open upgraded Reminder Store: %v", err) + } + defer func() { + if err := database.Close(); err != nil { + t.Errorf("close upgraded Reminder Store: %v", err) + } + }() + page, err := database.ListDue(context.Background(), time.Date(2200, time.January, 1, 0, 0, 0, 0, time.UTC), nil, 10) + if err != nil { + t.Fatalf("read upgraded Reminders: %v", err) + } + if len(page.Rows) != 1 { + t.Fatalf("upgraded Reminders = %#v, want one row", page.Rows) + } + reminder := page.Rows[0] + if reminder.GrainId != (store.GrainId{GrainType: "domain.Device", GrainKey: "device-1"}) || + reminder.Name != "offline" || reminder.Method != "MarkOffline" || + !reminder.DueAt.Equal(time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC)) || + reminder.Interval != 0 || reminder.ETag != 2 { + t.Fatalf("upgraded Reminder = %#v, want exact v0.0.5 Device/device-1 offline row", reminder) + } + if !reminder.FirstTickTime.Equal(reminder.DueAt) { + t.Fatalf("upgraded Reminder first tick = %v, want due time %v", reminder.FirstTickTime, reminder.DueAt) + } +} + +func assertUpgradeResponse(t *testing.T, response *http.Response, err error, wantStatus int, operation string) { + t.Helper() + if err != nil { + t.Fatalf("%s: %v", operation, err) + } + defer response.Body.Close() + body, readErr := io.ReadAll(response.Body) + if readErr != nil { + t.Fatalf("%s response: %v", operation, readErr) + } + if response.StatusCode != wantStatus { + t.Fatalf("%s status = %d, want %d; body = %q", operation, response.StatusCode, wantStatus, body) + } +} diff --git a/examples/shadow/cmd/conformance/main.go b/examples/shadow/cmd/conformance/main.go index 034d9a6..f56ab5a 100644 --- a/examples/shadow/cmd/conformance/main.go +++ b/examples/shadow/cmd/conformance/main.go @@ -22,6 +22,10 @@ const ( phaseRecover = "recover" ) +type releaseProofConfig struct { + claimBarrier bool +} + func main() { if err := run(context.Background(), os.Args[1:]); err != nil { log.Fatal(err) @@ -39,6 +43,8 @@ func run(ctx context.Context, args []string) (runErr error) { state := flags.String("state", "temperature=20", "reported Device State") traceID := flags.String("trace-id", "trace-1", "Request Context trace_id for prepare") waitTimeout := flags.Duration("timeout", 10*time.Second, "maximum wait for the recovery Call") + var proof releaseProofConfig + addReleaseProofFlags(flags, &proof) if err := flags.Parse(args); err != nil { return err } @@ -51,6 +57,9 @@ func run(ctx context.Context, args []string) (runErr error) { if *waitTimeout <= 0 { return errors.New("-timeout must be positive") } + if proof.claimBarrier && *phase != phaseRecover { + return errors.New("-claim-barrier requires -phase recover") + } if err := validateDatabasePaths(*runtimePath, *businessPath); err != nil { return err } @@ -79,10 +88,11 @@ func run(ctx context.Context, args []string) (runErr error) { } }() + reminderStore := proof.reminderStore(runtimeStore) calls := make(chan gor.CallObservation, 32) options := []gor.Option{ gor.WithStore(runtimeStore), - gor.WithReminderStore(runtimeStore), + gor.WithReminderStore(reminderStore), gor.WithReminderInterval(0), gor.OnError(shadow.LogBackgroundError), gor.OnCall(func(observation gor.CallObservation) { calls <- observation }), @@ -94,10 +104,19 @@ func run(ctx context.Context, args []string) (runErr error) { if err != nil { return fmt.Errorf("create Single Silo Runtime: %w", err) } - defer rt.Close() + defer func() { + shutdownContext, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := rt.Shutdown(shutdownContext); err != nil { + runErr = errors.Join(runErr, fmt.Errorf("shutdown Runtime: %w", err)) + } + }() if err := shadow.RegisterConformance(rt, application); err != nil { return fmt.Errorf("register conformance Grains: %w", err) } + if err := rt.Start(ctx); err != nil { + return fmt.Errorf("start Single Silo Runtime: %w", err) + } coordinator := gor.Ref[domain.RecoveryCoordinator](rt, domain.RecoveryCoordinatorKey) switch *phase { @@ -128,6 +147,15 @@ func run(ctx context.Context, args []string) (runErr error) { if !applied { return fmt.Errorf("ActionID %q has no applied record", *actionID) } + want := domain.AppliedRecord{ + ActionID: *actionID, + DeviceKey: *deviceKey, + State: *state, + TraceID: *traceID, + } + if record != want { + return fmt.Errorf("applied record = %#v, want %#v", record, want) + } pending, err := application.ListPending(ctx) if err != nil { return fmt.Errorf("list pending actions: %w", err) @@ -135,6 +163,13 @@ func run(ctx context.Context, args []string) (runErr error) { if len(pending) != 0 { return fmt.Errorf("pending actions remain after recovery: %#v", pending) } + shadowState, err := gor.Ref[domain.Device](rt, *deviceKey).Shadow(ctx) + if err != nil { + return fmt.Errorf("read recovered Grain State: %w", err) + } + if shadowState.ReportedState != *state || !shadowState.Online { + return fmt.Errorf("recovered Grain State = %#v, want reported State %q and Online true", shadowState, *state) + } log.Printf("recovered ActionID %q for Device %q with receipt %#v", record.ActionID, record.DeviceKey, record) return nil } @@ -176,15 +211,13 @@ func validateDatabasePaths(runtimePath, businessPath string) error { } families := make([]databasePathFamily, len(bases)) for index, base := range bases { - absolute, err := cleanDatabasePath(base.path) - if err != nil { - return fmt.Errorf("resolve %s database path: %w", base.label, err) - } - members := []string{absolute, absolute + "-wal", absolute + "-shm"} - for _, member := range members[1:] { - if err := rejectSymlinkComponents(member); err != nil { + members := make([]string, 0, 3) + for _, suffix := range []string{"", "-wal", "-shm"} { + resolved, err := cleanDatabasePath(base.path + suffix) + if err != nil { return fmt.Errorf("resolve %s database path: %w", base.label, err) } + members = append(members, resolved) } families[index] = databasePathFamily{label: base.label, members: members} } @@ -202,8 +235,8 @@ func validateDatabasePaths(runtimePath, businessPath string) error { } for left := 0; left < len(paths); left++ { for right := left + 1; right < len(paths); right++ { - if paths[left].path == paths[right].path { - return fmt.Errorf("database paths for %s and %s must be different: both resolve to %q", paths[left].label, paths[right].label, paths[left].path) + if strings.EqualFold(paths[left].path, paths[right].path) { + return fmt.Errorf("database paths for %s and %s must be different: %q and %q are not portable as distinct paths", paths[left].label, paths[right].label, paths[left].path, paths[right].path) } same, err := sameExistingFile(paths[left].path, paths[right].path) if err != nil { @@ -236,38 +269,52 @@ func cleanDatabasePath(path string) (string, error) { if err != nil { return "", err } - if err := rejectSymlinkComponents(absolute); err != nil { - return "", err - } - return filepath.Clean(absolute), nil + return resolveDatabasePath(filepath.Clean(absolute), 0) } -func rejectSymlinkComponents(absolute string) error { - volume := filepath.VolumeName(absolute) - rest := strings.TrimPrefix(absolute, volume) - current := volume - separator := string(filepath.Separator) - if strings.HasPrefix(rest, separator) { - current = volume + separator - rest = strings.TrimPrefix(rest, separator) - } - for _, component := range strings.Split(rest, separator) { - if component == "" || component == "." { - continue +func resolveDatabasePath(absolute string, links int) (string, error) { + current := absolute + var missing []string + for { + resolved, err := filepath.EvalSymlinks(current) + if err == nil { + for index := len(missing) - 1; index >= 0; index-- { + resolved = filepath.Join(resolved, missing[index]) + } + return filepath.Clean(resolved), nil } - current = filepath.Join(current, component) - info, err := os.Lstat(current) - if errors.Is(err, os.ErrNotExist) { - return nil + if !errors.Is(err, os.ErrNotExist) { + return "", err } - if err != nil { - return err + + info, lstatErr := os.Lstat(current) + if lstatErr == nil && info.Mode()&os.ModeSymlink != 0 { + if links >= 255 { + return "", errors.New("too many database path symlinks") + } + target, readErr := os.Readlink(current) + if readErr != nil { + return "", readErr + } + if !filepath.IsAbs(target) { + target = filepath.Join(filepath.Dir(current), target) + } + for index := len(missing) - 1; index >= 0; index-- { + target = filepath.Join(target, missing[index]) + } + return resolveDatabasePath(filepath.Clean(target), links+1) } - if info.Mode()&os.ModeSymlink != 0 { - return fmt.Errorf("database path contains symlink component %q", current) + if lstatErr != nil && !errors.Is(lstatErr, os.ErrNotExist) { + return "", lstatErr } + + parent := filepath.Dir(current) + if parent == current { + return "", err + } + missing = append(missing, filepath.Base(current)) + current = parent } - return nil } func sameExistingFile(leftPath, rightPath string) (bool, error) { diff --git a/examples/shadow/cmd/conformance/main_symlink_test.go b/examples/shadow/cmd/conformance/main_symlink_test.go new file mode 100644 index 0000000..89efc8a --- /dev/null +++ b/examples/shadow/cmd/conformance/main_symlink_test.go @@ -0,0 +1,65 @@ +//go:build !windows + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestValidateDatabasePathsRejectsSymlinkAliases(t *testing.T) { + directory := t.TempDir() + runtimePath := filepath.Join(directory, "runtime.db") + businessPath := filepath.Join(directory, "business.db") + if err := os.WriteFile(runtimePath, []byte("runtime"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(runtimePath, businessPath); err != nil { + t.Fatalf("create symlink alias: %v", err) + } + if err := validateDatabasePaths(runtimePath, businessPath); err == nil || !strings.Contains(err.Error(), "must be different") { + t.Fatalf("validateDatabasePaths(%q, %q) = %v, want resolved alias error", runtimePath, businessPath, err) + } + + danglingTarget := filepath.Join(directory, "not-created.db") + danglingAlias := filepath.Join(directory, "dangling-business.db") + if err := os.Symlink(danglingTarget, danglingAlias); err != nil { + t.Fatalf("create dangling symlink alias: %v", err) + } + if err := validateDatabasePaths(danglingTarget, danglingAlias); err == nil || !strings.Contains(err.Error(), "must be different") { + t.Fatalf("validateDatabasePaths(%q, %q) = %v, want dangling alias error", danglingTarget, danglingAlias, err) + } +} + +func TestValidateDatabasePathsRejectsMultiHopSymlinkAliases(t *testing.T) { + directory := t.TempDir() + targetPath := filepath.Join(directory, "target.db") + if err := os.Symlink(targetPath, filepath.Join(directory, "link-one.db")); err != nil { + t.Fatalf("create first symlink: %v", err) + } + if err := os.Symlink(filepath.Join(directory, "link-one.db"), filepath.Join(directory, "link-two.db")); err != nil { + t.Fatalf("create second symlink: %v", err) + } + if err := validateDatabasePaths(filepath.Join(directory, "link-two.db"), targetPath); err == nil || !strings.Contains(err.Error(), "must be different") { + t.Fatalf("multi-hop symlink validation = %v, want resolved alias error", err) + } +} + +func TestValidateDatabasePathsAcceptsDistinctPathsBelowSymlinkedDirectory(t *testing.T) { + directory := t.TempDir() + targetDirectory := filepath.Join(directory, "target") + if err := os.Mkdir(targetDirectory, 0o700); err != nil { + t.Fatal(err) + } + linkedDirectory := filepath.Join(directory, "linked") + if err := os.Symlink(targetDirectory, linkedDirectory); err != nil { + t.Fatalf("create directory symlink: %v", err) + } + runtimePath := filepath.Join(linkedDirectory, "runtime.db") + businessPath := filepath.Join(linkedDirectory, "business.db") + if err := validateDatabasePaths(runtimePath, businessPath); err != nil { + t.Fatalf("validateDatabasePaths(%q, %q) = %v, want separate identities below symlinked directory", runtimePath, businessPath, err) + } +} diff --git a/examples/shadow/cmd/conformance/main_test.go b/examples/shadow/cmd/conformance/main_test.go index 9d00699..90312a2 100644 --- a/examples/shadow/cmd/conformance/main_test.go +++ b/examples/shadow/cmd/conformance/main_test.go @@ -23,6 +23,7 @@ func TestValidateDatabasePathsRejectsEquivalentPaths(t *testing.T) { {name: "same absolute", runtime: absolute, business: absolute}, {name: "relative and absolute", runtime: "runtime.db", business: absolute}, {name: "cleaned relative and absolute", runtime: filepath.Join("data", "..", "runtime.db"), business: absolute}, + {name: "case-only names", runtime: filepath.Join(workingDir, "Runtime.db"), business: absolute}, } { t.Run(test.name, func(t *testing.T) { err := validateDatabasePaths(test.runtime, test.business) @@ -51,6 +52,9 @@ func TestValidateDatabasePathsRejectsSQLiteSidecarAliases(t *testing.T) { if err := validateDatabasePaths("runtime.db", "runtime-state.db-wal"); err == nil || !strings.Contains(err.Error(), "must be different") { t.Fatalf("validateDatabasePaths for Runtime State sidecar = %v, want sidecar collision error", err) } + if err := validateDatabasePaths("runtime.db", "RUNTIME.DB-WAL"); err == nil || !strings.Contains(err.Error(), "must be different") { + t.Fatalf("validateDatabasePaths for case-only sidecar = %v, want portable sidecar collision error", err) + } } func TestValidateDatabasePathsRejectsDerivedRuntimeStatePath(t *testing.T) { @@ -62,30 +66,6 @@ func TestValidateDatabasePathsRejectsDerivedRuntimeStatePath(t *testing.T) { } } -func TestValidateDatabasePathsRejectsSymlinkAliases(t *testing.T) { - directory := t.TempDir() - runtimePath := filepath.Join(directory, "runtime.db") - businessPath := filepath.Join(directory, "business.db") - if err := os.WriteFile(runtimePath, []byte("runtime"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.Symlink(runtimePath, businessPath); err != nil { - t.Fatalf("create symlink alias: %v", err) - } - if err := validateDatabasePaths(runtimePath, businessPath); err == nil || !strings.Contains(err.Error(), "symlink") { - t.Fatalf("validateDatabasePaths(%q, %q) = %v, want symlink error", runtimePath, businessPath, err) - } - - danglingTarget := filepath.Join(directory, "not-created.db") - danglingAlias := filepath.Join(directory, "dangling-business.db") - if err := os.Symlink(danglingTarget, danglingAlias); err != nil { - t.Fatalf("create dangling symlink alias: %v", err) - } - if err := validateDatabasePaths(danglingTarget, danglingAlias); err == nil || !strings.Contains(err.Error(), "symlink") { - t.Fatalf("validateDatabasePaths(%q, %q) = %v, want dangling symlink error", danglingTarget, danglingAlias, err) - } -} - func TestValidateDatabasePathsRejectsHardLinkAliases(t *testing.T) { directory := t.TempDir() runtimePath := filepath.Join(directory, "runtime.db") @@ -141,30 +121,11 @@ func TestValidateDatabasePathsRejectsRuntimeStateHardLinkAlias(t *testing.T) { } } -func TestValidateDatabasePathsRejectsParentTraversalAndSymlinkAliases(t *testing.T) { +func TestValidateDatabasePathsRejectsParentTraversal(t *testing.T) { directory := t.TempDir() - runtimePath := filepath.Join(directory, "runtime.db") businessPath := filepath.Join(directory, "business.db") - if err := os.Symlink(filepath.Join(directory, "target.db"), filepath.Join(directory, "link-one.db")); err != nil { - t.Fatalf("create first symlink: %v", err) - } - if err := os.Symlink(filepath.Join(directory, "link-one.db"), filepath.Join(directory, "link-two.db")); err != nil { - t.Fatalf("create second symlink: %v", err) - } - if err := validateDatabasePaths(filepath.Join(directory, "link-two.db"), businessPath); err == nil || !strings.Contains(err.Error(), "symlink") { - t.Fatalf("multi-hop symlink validation = %v, want symlink error", err) - } - - traversalPath := filepath.Join(directory, "link-one.db") + string(os.PathSeparator) + ".." + string(os.PathSeparator) + "runtime.db" + traversalPath := filepath.Join(directory, "data") + string(os.PathSeparator) + ".." + string(os.PathSeparator) + "runtime.db" if err := validateDatabasePaths(traversalPath, businessPath); err == nil || !strings.Contains(err.Error(), "must not contain '..'") { - t.Fatalf("symlink-aware parent traversal validation = %v, want parent traversal error", err) - } - - traversalRuntime := filepath.Join(directory, "runtime-alias.db") - if err := os.Symlink(runtimePath, traversalRuntime); err != nil { - t.Fatalf("create runtime symlink: %v", err) - } - if err := validateDatabasePaths(traversalRuntime, businessPath); err == nil || !strings.Contains(err.Error(), "symlink") { - t.Fatalf("runtime self-alias validation = %v, want symlink error", err) + t.Fatalf("parent traversal validation = %v, want parent traversal error", err) } } diff --git a/examples/shadow/cmd/conformance/proof_default.go b/examples/shadow/cmd/conformance/proof_default.go new file mode 100644 index 0000000..cf97ed7 --- /dev/null +++ b/examples/shadow/cmd/conformance/proof_default.go @@ -0,0 +1,15 @@ +//go:build !release + +package main + +import ( + "flag" + + "github.com/suraciii/gor/store" +) + +func addReleaseProofFlags(*flag.FlagSet, *releaseProofConfig) {} + +func (releaseProofConfig) reminderStore(base store.ReminderStore) store.ReminderStore { + return base +} diff --git a/examples/shadow/cmd/conformance/proof_default_test.go b/examples/shadow/cmd/conformance/proof_default_test.go new file mode 100644 index 0000000..7b3ca40 --- /dev/null +++ b/examples/shadow/cmd/conformance/proof_default_test.go @@ -0,0 +1,16 @@ +//go:build !release + +package main + +import ( + "context" + "strings" + "testing" +) + +func TestRunRejectsReleaseOnlyClaimBarrier(t *testing.T) { + err := run(context.Background(), []string{"-phase", phaseRecover, "-claim-barrier"}) + if err == nil || !strings.Contains(err.Error(), "flag provided but not defined") { + t.Fatalf("run with release-only Claim barrier = %v, want unknown flag", err) + } +} diff --git a/examples/shadow/cmd/conformance/proof_release.go b/examples/shadow/cmd/conformance/proof_release.go new file mode 100644 index 0000000..e6e1417 --- /dev/null +++ b/examples/shadow/cmd/conformance/proof_release.go @@ -0,0 +1,47 @@ +//go:build release + +package main + +import ( + "context" + "flag" + "fmt" + "os" + "sync" + "time" + + "github.com/suraciii/gor/store" +) + +const claimBarrierReady = "GOR_CONFORMANCE_CLAIM_READY" + +func addReleaseProofFlags(flags *flag.FlagSet, proof *releaseProofConfig) { + flags.BoolVar(&proof.claimBarrier, "claim-barrier", false, "block after a successful Reminder Claim") +} + +func (proof releaseProofConfig) reminderStore(base store.ReminderStore) store.ReminderStore { + if !proof.claimBarrier { + return base + } + return &claimBarrierStore{ReminderStore: base} +} + +type claimBarrierStore struct { + store.ReminderStore + once sync.Once +} + +func (s *claimBarrierStore) Claim(ctx context.Context, reminder store.Reminder, nextDueAt time.Time) (bool, error) { + won, err := s.ReminderStore.Claim(ctx, reminder, nextDueAt) + if err != nil || !won { + return won, err + } + s.once.Do(func() { + fmt.Fprintln(os.Stdout, claimBarrierReady) + <-ctx.Done() + }) + if err := ctx.Err(); err != nil { + return false, err + } + return true, nil +} diff --git a/examples/shadow/cmd/load/main.go b/examples/shadow/cmd/load/main.go index 657ac59..661c4f9 100644 --- a/examples/shadow/cmd/load/main.go +++ b/examples/shadow/cmd/load/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "flag" "fmt" "log" @@ -66,6 +67,7 @@ func run(ctx context.Context, args []string) (runErr error) { lifecycleEvents := make(chan domain.LifecycleEvent, 2*(*deviceCount)+4) rt, err := gor.New( gor.WithStore(database), + gor.WithReminderStore(database), gor.WithClock(sourceClock), gor.WithIdleTimeout(idleTimeout), gor.WithEvictionInterval(evictionInterval), @@ -75,9 +77,18 @@ func run(ctx context.Context, args []string) (runErr error) { if err != nil { return fmt.Errorf("create runtime: %w", err) } - defer rt.Close() + defer func() { + shutdownContext, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := rt.Shutdown(shutdownContext); err != nil { + runErr = errors.Join(runErr, fmt.Errorf("shutdown runtime: %w", err)) + } + }() if err := shadow.RegisterWithLifecycle(rt, lifecycleEvents); err != nil { - return fmt.Errorf("register shadow entities: %w", err) + return fmt.Errorf("register shadow Grains: %w", err) + } + if err := rt.Start(ctx); err != nil { + return fmt.Errorf("start runtime: %w", err) } if err := reportDevices(ctx, rt, *deviceCount); err != nil { diff --git a/examples/shadow/cmd/load/main_test.go b/examples/shadow/cmd/load/main_test.go index f0b194c..3537c0d 100644 --- a/examples/shadow/cmd/load/main_test.go +++ b/examples/shadow/cmd/load/main_test.go @@ -12,8 +12,10 @@ import ( ) func TestReportDevicesReturnsAfterSuccess(t *testing.T) { + backend := store.NewMemory() rt, err := gor.New( - gor.WithStore(store.NewMemory()), + gor.WithStore(backend), + gor.WithReminderStore(backend), gor.WithClock(clock.NewFake(time.Unix(0, 0).UTC())), gor.WithIdleTimeout(0), gor.WithEvictionInterval(0), @@ -23,10 +25,14 @@ func TestReportDevicesReturnsAfterSuccess(t *testing.T) { t.Fatal(err) } if err := shadow.Register(rt); err != nil { - rt.Close() + _ = rt.Shutdown(context.Background()) t.Fatal(err) } - defer rt.Close() + if err := rt.Start(context.Background()); err != nil { + _ = rt.Shutdown(context.Background()) + t.Fatal(err) + } + defer rt.Shutdown(context.Background()) if err := reportDevices(context.Background(), rt, 1); err != nil { t.Fatalf("reportDevices returned error: %v", err) diff --git a/examples/shadow/cmd/shadow/main.go b/examples/shadow/cmd/shadow/main.go index b31ef0b..3bbaac5 100644 --- a/examples/shadow/cmd/shadow/main.go +++ b/examples/shadow/cmd/shadow/main.go @@ -8,6 +8,7 @@ import ( "flag" "fmt" "log" + "net" "net/http" "os" "os/signal" @@ -33,9 +34,9 @@ func run(ctx context.Context, args []string) (runErr error) { flags := flag.NewFlagSet("shadow", flag.ContinueOnError) flags.SetOutput(os.Stderr) address := flags.String("addr", ":8080", "HTTP listen address") - databasePath := flags.String("db", "data/gor.db", "SQLite database path (shared by every node in cluster mode)") - clusterEnabled := flags.Bool("cluster", false, "run as one node of a cluster; all nodes share -db") - nodeAddr := flags.String("node-addr", "", "cluster transport bind address; defaults to 127.0.0.1:0 when -cluster") + databasePath := flags.String("db", "data/gor.db", "SQLite database path (shared by every Silo in Cluster mode)") + clusterEnabled := flags.Bool("cluster", false, "run as one Silo in a Cluster; all Silos share -db") + nodeAddr := flags.String("node-addr", "", "Silo transport bind address; defaults to 127.0.0.1:0 when -cluster") generation := flags.String("generation", "", "cluster membership generation; defaults to a fresh value when -cluster") if err := flags.Parse(args); err != nil { return err @@ -68,20 +69,30 @@ func run(ctx context.Context, args []string) (runErr error) { } return fmt.Errorf("create runtime: %w", err) } - defer rt.Close() + defer func() { + shutdownContext, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := rt.Shutdown(shutdownContext); err != nil { + runErr = errors.Join(runErr, fmt.Errorf("shutdown runtime: %w", err)) + } + }() if err := shadow.Register(rt); err != nil { - return fmt.Errorf("register shadow entities: %w", err) + return fmt.Errorf("register shadow Grains: %w", err) + } + if err := rt.Start(ctx); err != nil { + return fmt.Errorf("start runtime: %w", err) } - server := &http.Server{ - Addr: *address, - Handler: shadow.NewHandler(rt), + listener, err := net.Listen("tcp", *address) + if err != nil { + return fmt.Errorf("listen HTTP: %w", err) } + server := &http.Server{Handler: shadow.NewHandler(rt)} serverErrors := make(chan error, 1) go func() { - serverErrors <- server.ListenAndServe() + serverErrors <- server.Serve(listener) }() - log.Printf("device shadow listening on %s with database %s", *address, *databasePath) + log.Printf("device shadow listening on %s with database %s", listener.Addr(), *databasePath) select { case err := <-serverErrors: @@ -99,11 +110,11 @@ func run(ctx context.Context, args []string) (runErr error) { } } -// configureCluster builds the cluster options for this node when clustering is -// enabled, or returns nil when running single-node. The same database backs -// both entity state and the shared membership table: every node opens the same +// configureCluster builds the Cluster options for this Silo when Cluster mode +// is enabled, or returns nil when running one Silo. The same database backs +// both Grain State and the shared membership table: every Silo opens the same // SQLite file. nodeAddr defaults to an OS-chosen loopback port; a fresh -// generation is taken on every start, so a node that rejoins at the same +// generation is taken on every start, so a Silo that rejoins at the same // address never reuses its previous incarnation's row. The returned transport // is owned by the runtime once New succeeds; the caller closes it only on the // early-error path. @@ -125,7 +136,7 @@ func configureCluster(enabled bool, members store.MemberStore, nodeAddr, generat return nil, nil, fmt.Errorf("generate membership generation: %w", err) } } - log.Printf("cluster node %s generation %s", nodeTransport.Addr(), generation) + log.Printf("Cluster Silo %s generation %s", nodeTransport.Addr(), generation) options := []gor.Option{ gor.WithMemberStore(members), gor.WithNodeAddr(nodeTransport.Addr()), @@ -136,7 +147,7 @@ func configureCluster(enabled bool, members store.MemberStore, nodeAddr, generat } // newGeneration returns a fresh membership generation. A generation must be -// new on every rejoin at the same address, so a restarted node does not claim +// new on every rejoin at the same address, so a restarted Silo does not claim // the row of its previous incarnation while others may still be voting on it. func newGeneration() (string, error) { var raw [8]byte @@ -146,7 +157,16 @@ func newGeneration() (string, error) { return hex.EncodeToString(raw[:]), nil } -func newRuntime(database store.Store, options ...gor.Option) (*gor.Runtime, error) { - options = append([]gor.Option{gor.WithStore(database), gor.OnError(shadow.LogBackgroundError)}, options...) +type runtimeStore interface { + store.Store + store.ReminderStore +} + +func newRuntime(database runtimeStore, options ...gor.Option) (*gor.Runtime, error) { + options = append([]gor.Option{ + gor.WithStore(database), + gor.WithReminderStore(database), + gor.OnError(shadow.LogBackgroundError), + }, options...) return gor.New(options...) } diff --git a/examples/shadow/cmd/shadow/main_test.go b/examples/shadow/cmd/shadow/main_test.go index a6a2bf3..6faa634 100644 --- a/examples/shadow/cmd/shadow/main_test.go +++ b/examples/shadow/cmd/shadow/main_test.go @@ -12,7 +12,6 @@ import ( "github.com/suraciii/gor" "github.com/suraciii/gor/clock" shadow "github.com/suraciii/gor/examples/shadow" - "github.com/suraciii/gor/examples/shadow/domain" "github.com/suraciii/gor/store" ) @@ -38,17 +37,22 @@ func TestNewRuntimeReportsScheduledFailure(t *testing.T) { t.Fatal(err) } if err := shadow.Register(rt); err != nil { - rt.Close() + _ = rt.Shutdown(context.Background()) + logger.SetOutput(previousWriter) + t.Fatal(err) + } + if err := rt.Start(context.Background()); err != nil { + _ = rt.Shutdown(context.Background()) logger.SetOutput(previousWriter) t.Fatal(err) } defer func() { - rt.Close() + _ = rt.Shutdown(context.Background()) logger.SetOutput(previousWriter) }() if err := backend.Put(context.Background(), store.Reminder{ - GrainId: store.GrainId{GrainType: gor.TypeName[domain.Device](), GrainKey: "device-1"}, + GrainId: store.GrainId{GrainType: "domain.Device", GrainKey: "device-1"}, Name: "broken", Method: "NotAMethod", DueAt: start, diff --git a/examples/shadow/cmd/shadow/quick_start_release_test.go b/examples/shadow/cmd/shadow/quick_start_release_test.go new file mode 100644 index 0000000..1c2a91e --- /dev/null +++ b/examples/shadow/cmd/shadow/quick_start_release_test.go @@ -0,0 +1,166 @@ +//go:build release + +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +const quickStartMarker = "device shadow listening on " + +func TestQuickStartHTTPProcess(t *testing.T) { + work := t.TempDir() + binary := filepath.Join(work, "shadow"+shadowExecutableSuffix()) + buildContext, cancelBuild := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancelBuild() + build := exec.CommandContext(buildContext, "go", "build", "-o", binary, ".") + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build Quick Start command: %v\n%s", err, output) + } + if buildContext.Err() != nil { + t.Fatalf("build Quick Start command: %v", buildContext.Err()) + } + + processContext, cancelProcess := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelProcess() + command := exec.CommandContext(processContext, binary, + "-addr", "127.0.0.1:0", + "-db", filepath.Join(work, "data", "gor.db"), + ) + var stdout bytes.Buffer + command.Stdout = &stdout + stderr, err := command.StderrPipe() + if err != nil { + t.Fatalf("open Quick Start process output: %v", err) + } + if err := command.Start(); err != nil { + t.Fatalf("start Quick Start process: %v", err) + } + + addressReady := make(chan string, 1) + stderrDone := make(chan string, 1) + go func() { + var output strings.Builder + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + line := scanner.Text() + fmt.Fprintln(&output, line) + if index := strings.Index(line, quickStartMarker); index >= 0 { + address := line[index+len(quickStartMarker):] + if end := strings.Index(address, " with database "); end >= 0 { + addressReady <- address[:end] + } + } + } + if err := scanner.Err(); err != nil { + fmt.Fprintf(&output, "read stderr: %v\n", err) + } + stderrDone <- output.String() + }() + waitDone := make(chan error, 1) + go func() { waitDone <- command.Wait() }() + + stopped := false + defer func() { + if stopped { + return + } + if command.ProcessState == nil { + _ = command.Process.Kill() + } + <-waitDone + <-stderrDone + }() + + var address string + select { + case address = <-addressReady: + case waitErr := <-waitDone: + stopped = true + stderrText := <-stderrDone + t.Fatalf("Quick Start process exited before listening: %v\n%s%s", waitErr, stdout.String(), stderrText) + case <-processContext.Done(): + waitErr := <-waitDone + stopped = true + stderrText := <-stderrDone + t.Fatalf("wait for Quick Start listener: %v; process error: %v\n%s%s", processContext.Err(), waitErr, stdout.String(), stderrText) + } + + client := &http.Client{Timeout: 5 * time.Second} + reportBody := strings.NewReader(`{"workshop_id":"assembly","state":"temperature=20"}`) + report, err := client.Post("http://"+address+"/devices/device-1/reports", "application/json", reportBody) + if err != nil { + t.Fatalf("run Quick Start report request: %v", err) + } + reportText := readQuickStartBody(t, report) + if report.StatusCode != http.StatusNoContent { + t.Fatalf("report status = %d, want %d; body = %q", report.StatusCode, http.StatusNoContent, reportText) + } + + read, err := client.Get("http://" + address + "/devices/device-1/shadow") + if err != nil { + t.Fatalf("run Quick Start read request: %v", err) + } + readText := readQuickStartBody(t, read) + if read.StatusCode != http.StatusOK { + t.Fatalf("read status = %d, want %d; body = %q", read.StatusCode, http.StatusOK, readText) + } + var shadow struct { + ReportedState string `json:"reported_state"` + Online bool `json:"online"` + WorkshopID string `json:"workshop_id"` + } + if err := json.Unmarshal([]byte(readText), &shadow); err != nil { + t.Fatalf("decode Quick Start shadow: %v; body = %q", err, readText) + } + if shadow.ReportedState != "temperature=20" || !shadow.Online || shadow.WorkshopID != "assembly" { + t.Fatalf("Quick Start shadow = %#v, want the reported device state", shadow) + } + + if err := command.Process.Kill(); err != nil { + t.Fatalf("stop Quick Start process: %v", err) + } + waitErr := <-waitDone + stderrText := <-stderrDone + stopped = true + if processContext.Err() != nil { + t.Fatalf("Quick Start process was stopped by timeout, not the parent: %v\n%s%s", processContext.Err(), stdout.String(), stderrText) + } + if waitErr == nil { + t.Fatalf("Quick Start process exited successfully after Process.Kill\n%s%s", stdout.String(), stderrText) + } + var exitError *exec.ExitError + if !errors.As(waitErr, &exitError) || exitError.ExitCode() == 0 { + t.Fatalf("Quick Start process error = %v, want nonzero killed exit\n%s%s", waitErr, stdout.String(), stderrText) + } +} + +func readQuickStartBody(t *testing.T, response *http.Response) string { + t.Helper() + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatalf("read Quick Start response: %v", err) + } + return string(body) +} + +func shadowExecutableSuffix() string { + if runtime.GOOS == "windows" { + return ".exe" + } + return "" +} diff --git a/examples/shadow/conformance_core_test.go b/examples/shadow/conformance_core_test.go index 2756740..2b29fba 100644 --- a/examples/shadow/conformance_core_test.go +++ b/examples/shadow/conformance_core_test.go @@ -59,9 +59,9 @@ func TestConformance_TypedGrainsStatePresenceAndClear(t *testing.T) { t.Fatalf("second ClearShadow: %v", err) } - rt.Kill() + _ = crashRuntime(rt) rt = newConformanceRuntime(t, sourceClock, stateStore, reminderStore, application, nil, nil) - defer rt.Kill() + defer crashRuntime(rt) if exists, err := gor.Ref[domain.Device](rt, "device-1").ShadowExists(ctx); err != nil || exists { t.Fatalf("ShadowExists after reactivation = (%v, %v), want (false, nil)", exists, err) } @@ -77,7 +77,7 @@ func TestConformance_RequestContextIsCopiedAndNotPersisted(t *testing.T) { application := domain.NewMemoryApplicationStore() observed := make(chan domain.RecoveryObservation, 4) rt := newConformanceRuntime(t, sourceClock, stateStore, reminderStore, application, observed, nil) - defer rt.Kill() + defer crashRuntime(rt) ctx, err := gor.WithRequestContext(context.Background(), "trace_id", "trace-1") if err != nil { t.Fatal(err) @@ -98,18 +98,15 @@ func TestConformance_RequestContextIsCopiedAndNotPersisted(t *testing.T) { t.Fatalf("pending actions = %#v, want one copied trace ID", pending) } - stateRecord, err := stateStore.Read(context.Background(), store.GrainId{GrainType: gor.TypeName[domain.Device](), GrainKey: "device-1"}) + stateRecord, err := stateStore.Read(context.Background(), store.GrainId{GrainType: "domain.Device", GrainKey: "device-1"}) if err != nil { t.Fatalf("read Device State: %v", err) } if bytes.Contains(stateRecord.Data, []byte("trace_id")) || bytes.Contains(stateRecord.Data, []byte("trace-1")) { t.Fatalf("Device State contains Request Context: %s", stateRecord.Data) } - rows, err := reminderStore.ListDue(context.Background(), start.Add(2*time.Second)) - if err != nil { - t.Fatalf("ListDue: %v", err) - } - if !hasReminder(rows, gor.TypeName[domain.RecoveryCoordinator](), domain.RecoveryCoordinatorKey, domain.RecoveryReminderName) { + rows := listDueReminders(t, reminderStore, start.Add(2*time.Second)) + if !hasReminder(rows, "domain.RecoveryCoordinator", domain.RecoveryCoordinatorKey, domain.RecoveryReminderName) { t.Fatalf("Reminders = %#v, want fixed-key recovery Reminder", rows) } for _, row := range rows { @@ -128,7 +125,7 @@ func TestConformance_RequestContextIsCopiedAndNotPersisted(t *testing.T) { if observation.TracePresent || observation.TraceID != nil { t.Fatalf("Reminder Request Context = %#v, want absent", observation) } - if !observation.Tick.FirstTickTime.Equal(start.Add(domain.RecoveryInterval)) || observation.Tick.Period != domain.RecoveryInterval { + if observation.Tick.ReminderName != domain.RecoveryReminderName || !observation.Tick.FirstTickTime.Equal(start.Add(domain.RecoveryInterval)) || observation.Tick.Period != domain.RecoveryInterval { t.Fatalf("Reminder TickStatus = %#v, want fixed first tick and period", observation.Tick) } }) @@ -154,11 +151,11 @@ func TestConformance_RestartRecoversPendingAction(t *testing.T) { if err := gor.Ref[domain.Device](rt, "device-1").ReportAction(ctx, "action-restart", "temperature=21"); err != nil { t.Fatalf("ReportAction: %v", err) } - rt.Kill() + _ = crashRuntime(rt) calls := make(chan gor.CallObservation, 16) rt = newConformanceRuntime(t, sourceClock, stateStore, reminderStore, application, observed, calls) - defer rt.Kill() + defer crashRuntime(rt) if err := gor.Ref[domain.RecoveryCoordinator](rt, domain.RecoveryCoordinatorKey).Start(context.Background()); err != nil { t.Fatalf("restart Start: %v", err) } diff --git a/examples/shadow/conformance_failures_test.go b/examples/shadow/conformance_failures_test.go index bcd4a4e..482eda7 100644 --- a/examples/shadow/conformance_failures_test.go +++ b/examples/shadow/conformance_failures_test.go @@ -17,7 +17,7 @@ func TestConformance_ReminderClaimHasOneWinner(t *testing.T) { start := time.Unix(1300, 0).UTC() reminderStore := store.NewMemory() row := store.Reminder{ - GrainId: store.GrainId{GrainType: gor.TypeName[shadowdomain.RecoveryCoordinator](), GrainKey: shadowdomain.RecoveryCoordinatorKey}, + GrainId: store.GrainId{GrainType: "domain.RecoveryCoordinator", GrainKey: shadowdomain.RecoveryCoordinatorKey}, Name: shadowdomain.RecoveryReminderName, Method: "Recover", FirstTickTime: start, @@ -27,9 +27,9 @@ func TestConformance_ReminderClaimHasOneWinner(t *testing.T) { if err := reminderStore.Put(context.Background(), row); err != nil { t.Fatal(err) } - due, err := reminderStore.ListDue(context.Background(), start) - if err != nil || len(due) != 1 { - t.Fatalf("ListDue = (%#v, %v), want one row", due, err) + due := listDueReminders(t, reminderStore, start) + if len(due) != 1 { + t.Fatalf("ListDue = %#v, want one row", due) } startClaims := make(chan struct{}) results := make(chan claimResult, 2) @@ -56,6 +56,55 @@ func TestConformance_ReminderClaimHasOneWinner(t *testing.T) { } } +func TestConformance_InvalidReminderHasOneTerminalReport(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(1350, 0).UTC() + sourceClock := clock.NewFake(start) + stateStore := store.NewMemory() + reminderStore := store.NewMemory() + application := shadowdomain.NewMemoryApplicationStore() + errorsSeen := make(chan gor.BackgroundError, 4) + row := store.Reminder{ + GrainId: store.GrainId{GrainType: "domain.RecoveryCoordinator", GrainKey: shadowdomain.RecoveryCoordinatorKey}, + Name: "old-recovery", + Method: "Missing", + FirstTickTime: start, + DueAt: start, + Interval: shadowdomain.RecoveryInterval, + } + if err := reminderStore.Put(context.Background(), row); err != nil { + t.Fatal(err) + } + + first := newConformanceRuntimeWithErrors(t, sourceClock, stateStore, reminderStore, application, nil, nil, errorsSeen) + for range 3 { + sourceClock.Advance(shadowdomain.RecoveryInterval) + synctest.Wait() + } + background := <-errorsSeen + source, ok := background.Source.(gor.ReminderDispatch) + wantID := gor.GrainId{GrainType: "domain.RecoveryCoordinator", GrainKey: shadowdomain.RecoveryCoordinatorKey} + if !ok || background.GrainId != wantID || source.Name != row.Name || source.Method != row.Method || !errors.Is(background.Err, gor.ErrUnknownMethod) { + t.Fatalf("OnError = %#v, want one ReminderDispatch for old-recovery/Missing", background) + } + if got := len(errorsSeen); got != 0 { + t.Fatalf("later dispatch failures = %d, want 0", got) + } + if rows := listDueReminders(t, reminderStore, sourceClock.Now()); hasReminder(rows, row.GrainId.GrainType, row.GrainId.GrainKey, row.Name) { + t.Fatalf("terminal Reminder returned by ListDue: %#v", rows) + } + _ = crashRuntime(first) + + second := newConformanceRuntimeWithErrors(t, sourceClock, stateStore, reminderStore, application, nil, nil, errorsSeen) + defer crashRuntime(second) + sourceClock.Advance(shadowdomain.RecoveryInterval) + synctest.Wait() + if got := len(errorsSeen); got != 0 { + t.Fatalf("dispatch failures after Runtime restart = %d, want 0", got) + } + }) +} + func TestConformance_StopAfterClaimLeavesPendingForNextTick(t *testing.T) { synctest.Test(t, func(t *testing.T) { start := time.Unix(1400, 0).UTC() @@ -81,7 +130,7 @@ func TestConformance_StopAfterClaimLeavesPendingForNextTick(t *testing.T) { killDone := make(chan struct{}) go func() { - rt.Kill() + _ = crashRuntime(rt) close(killDone) }() <-rt.Done() @@ -92,7 +141,7 @@ func TestConformance_StopAfterClaimLeavesPendingForNextTick(t *testing.T) { } rt = newConformanceRuntime(t, sourceClock, stateStore, baseReminders, application, nil, nil) - defer rt.Kill() + defer crashRuntime(rt) if err := gor.Ref[shadowdomain.RecoveryCoordinator](rt, shadowdomain.RecoveryCoordinatorKey).Start(context.Background()); err != nil { t.Fatalf("restart Start: %v", err) } @@ -116,7 +165,7 @@ func TestConformance_SafeRepeatAndUnknownResult(t *testing.T) { errorsSeen := make(chan gor.BackgroundError, 4) calls := make(chan gor.CallObservation, 16) rt := newConformanceRuntimeWithErrors(t, sourceClock, stateStore, reminderStore, application, nil, calls, errorsSeen) - defer rt.Kill() + defer crashRuntime(rt) if err := gor.Ref[shadowdomain.RecoveryCoordinator](rt, shadowdomain.RecoveryCoordinatorKey).Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } @@ -130,8 +179,14 @@ func TestConformance_SafeRepeatAndUnknownResult(t *testing.T) { t.Fatalf("OnError = %#v, want unknown result", background) } source, ok := background.Source.(gor.ReminderInvocation) - if !ok || source.Method != "Recover" { - t.Fatalf("OnError source = %#v, want ReminderInvocation{Recover}", background.Source) + wantTick := gor.TickStatus{ + ReminderName: shadowdomain.RecoveryReminderName, + FirstTickTime: start.Add(shadowdomain.RecoveryInterval), + Period: shadowdomain.RecoveryInterval, + CurrentTickTime: start.Add(shadowdomain.RecoveryInterval), + } + if !ok || source.Name != shadowdomain.RecoveryReminderName || source.Method != "Recover" || source.TickStatus != wantTick { + t.Fatalf("OnError source = %#v, want recovery/Recover with TickStatus %#v", background.Source, wantTick) } if !hasCall(calls, "Recover", unknownErr) { t.Fatalf("OnCall observations = %v, want Recover error", drainCalls(calls)) @@ -143,6 +198,10 @@ func TestConformance_SafeRepeatAndUnknownResult(t *testing.T) { if pending, err := baseApplication.ListPending(context.Background()); err != nil || len(pending) != 0 { t.Fatalf("pending after unknown result = (%#v, %v), want none", pending, err) } + shadow, err := gor.Ref[shadowdomain.Device](rt, "device-1").Shadow(context.Background()) + if err != nil || shadow.ReportedState != "temperature=23" || !shadow.Online { + t.Fatalf("Shadow after unknown Application result = (%#v, %v), want requested State", shadow, err) + } if err := gor.Ref[shadowdomain.Device](rt, "device-1").ApplyPending(context.Background(), "action-unknown"); err != nil { t.Fatalf("Safe Repeat ApplyPending: %v", err) } @@ -164,7 +223,7 @@ func TestConformance_ReminderFailureRetriesPendingAndReportsError(t *testing.T) application := &faultApplicationStore{ApplicationStore: baseApplication, before: beforeErr} errorsSeen := make(chan gor.BackgroundError, 4) rt := newConformanceRuntimeWithErrors(t, sourceClock, stateStore, reminderStore, application, nil, nil, errorsSeen) - defer rt.Kill() + defer crashRuntime(rt) if err := gor.Ref[shadowdomain.RecoveryCoordinator](rt, shadowdomain.RecoveryCoordinatorKey).Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } @@ -180,6 +239,13 @@ func TestConformance_ReminderFailureRetriesPendingAndReportsError(t *testing.T) if pending, err := baseApplication.ListPending(context.Background()); err != nil || len(pending) != 1 { t.Fatalf("pending after failed Reminder = (%#v, %v), want one action", pending, err) } + if record, applied, err := baseApplication.ReadApplied(context.Background(), "action-before"); err != nil || applied { + t.Fatalf("receipt after failed Application commit = (%#v, %v, %v), want absent", record, applied, err) + } + shadow, err := gor.Ref[shadowdomain.Device](rt, "device-1").Shadow(context.Background()) + if err != nil || shadow.ReportedState != "temperature=24" || !shadow.Online { + t.Fatalf("Shadow before Application retry = (%#v, %v), want requested State", shadow, err) + } sourceClock.Advance(shadowdomain.RecoveryInterval) synctest.Wait() if _, applied, err := baseApplication.ReadApplied(context.Background(), "action-before"); err != nil || !applied { @@ -197,7 +263,7 @@ func TestConformance_CancelRecoveryReminderClearsScheduleAndState(t *testing.T) application := shadowdomain.NewMemoryApplicationStore() observed := make(chan shadowdomain.RecoveryObservation, 4) rt := newConformanceRuntime(t, sourceClock, stateStore, reminderStore, application, observed, nil) - defer rt.Kill() + defer crashRuntime(rt) coordinator := gor.Ref[shadowdomain.RecoveryCoordinator](rt, shadowdomain.RecoveryCoordinatorKey) if err := coordinator.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) @@ -205,14 +271,17 @@ func TestConformance_CancelRecoveryReminderClearsScheduleAndState(t *testing.T) if err := coordinator.Stop(context.Background()); err != nil { t.Fatalf("Stop: %v", err) } - rows, err := reminderStore.ListDue(context.Background(), start.Add(2*shadowdomain.RecoveryInterval)) - if err != nil { - t.Fatal(err) + synctest.Wait() + for _, activation := range rt.Activations() { + if activation.GrainId == (gor.GrainId{GrainType: "domain.RecoveryCoordinator", GrainKey: shadowdomain.RecoveryCoordinatorKey}) { + t.Fatalf("recovery coordinator stayed active after Stop: %#v", activation) + } } - if hasReminder(rows, gor.TypeName[shadowdomain.RecoveryCoordinator](), shadowdomain.RecoveryCoordinatorKey, shadowdomain.RecoveryReminderName) { + rows := listDueReminders(t, reminderStore, start.Add(2*shadowdomain.RecoveryInterval)) + if hasReminder(rows, "domain.RecoveryCoordinator", shadowdomain.RecoveryCoordinatorKey, shadowdomain.RecoveryReminderName) { t.Fatalf("recovery Reminder remains after Stop: %#v", rows) } - record, err := stateStore.Read(context.Background(), store.GrainId{GrainType: gor.TypeName[shadowdomain.RecoveryCoordinator](), GrainKey: shadowdomain.RecoveryCoordinatorKey}) + record, err := stateStore.Read(context.Background(), store.GrainId{GrainType: "domain.RecoveryCoordinator", GrainKey: shadowdomain.RecoveryCoordinatorKey}) if err != nil { t.Fatal(err) } diff --git a/examples/shadow/conformance_grain_timer_test.go b/examples/shadow/conformance_grain_timer_test.go new file mode 100644 index 0000000..caeca26 --- /dev/null +++ b/examples/shadow/conformance_grain_timer_test.go @@ -0,0 +1,97 @@ +package shadow_test + +import ( + "context" + "testing" + "testing/synctest" + "time" + + "github.com/suraciii/gor" + "github.com/suraciii/gor/clock" + shadow "github.com/suraciii/gor/examples/shadow" + "github.com/suraciii/gor/examples/shadow/domain" + "github.com/suraciii/gor/store" +) + +type blockingPutReminderStore struct { + store.ReminderStore + started chan struct{} + release chan struct{} +} + +func (s *blockingPutReminderStore) Put(ctx context.Context, reminder store.Reminder) error { + close(s.started) + select { + case <-s.release: + return s.ReminderStore.Put(ctx, reminder) + case <-ctx.Done(): + return ctx.Err() + } +} + +func TestConformance_GrainTimerSerializesAndEndsWithActivation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(1800, 0).UTC() + sourceClock := clock.NewFake(start) + stateStore := store.NewMemory() + baseReminders := store.NewMemory() + reminderStore := &blockingPutReminderStore{ + ReminderStore: baseReminders, + started: make(chan struct{}), + release: make(chan struct{}), + } + application := domain.NewMemoryApplicationStore() + timerCallbacks := make(chan time.Time, 4) + + rt, err := gor.New( + gor.WithStore(stateStore), + gor.WithReminderStore(reminderStore), + gor.WithClock(sourceClock), + gor.WithIdleTimeout(0), + gor.WithEvictionInterval(0), + gor.WithReminderInterval(0), + ) + if err != nil { + t.Fatal(err) + } + if err := shadow.RegisterConformanceWithGrainTimerObservation(rt, application, timerCallbacks); err != nil { + t.Fatal(err) + } + startRuntime(t, rt) + defer crashRuntime(rt) + + coordinator := gor.Ref[domain.RecoveryCoordinator](rt, domain.RecoveryCoordinatorKey) + startDone := make(chan error, 1) + go func() { startDone <- coordinator.Start(context.Background()) }() + synctest.Wait() + <-reminderStore.started + sourceClock.Advance(domain.RecoveryInterval) + synctest.Wait() + select { + case <-timerCallbacks: + t.Fatal("Grain Timer callback overlapped the blocked Start Call") + default: + } + + close(reminderStore.release) + synctest.Wait() + if err := <-startDone; err != nil { + t.Fatal(err) + } + if at := <-timerCallbacks; !at.Equal(start.Add(domain.RecoveryInterval)) { + t.Fatalf("Grain Timer callback time = %s", at) + } + + if err := coordinator.Stop(context.Background()); err != nil { + t.Fatal(err) + } + synctest.Wait() + sourceClock.Advance(domain.RecoveryInterval) + synctest.Wait() + select { + case at := <-timerCallbacks: + t.Fatalf("old Activation Grain Timer ran at %s", at) + default: + } + }) +} diff --git a/examples/shadow/conformance_helpers_test.go b/examples/shadow/conformance_helpers_test.go index 0ecbada..959dce5 100644 --- a/examples/shadow/conformance_helpers_test.go +++ b/examples/shadow/conformance_helpers_test.go @@ -31,8 +31,12 @@ func (s *blockingReminderStore) Claim(ctx context.Context, reminder store.Remind return won, err } close(s.claimed) - <-s.release - return won, nil + select { + case <-s.release: + return won, nil + case <-ctx.Done(): + return false, ctx.Err() + } } type faultApplicationStore struct { @@ -43,11 +47,11 @@ type faultApplicationStore struct { afterUsed atomic.Bool } -func (s *faultApplicationStore) ApplyPending(ctx context.Context, actionID string) error { +func (s *faultApplicationStore) CompletePending(ctx context.Context, actionID string) error { if s.before != nil && s.beforeUsed.CompareAndSwap(false, true) { return s.before } - if err := s.ApplicationStore.ApplyPending(ctx, actionID); err != nil { + if err := s.ApplicationStore.CompletePending(ctx, actionID); err != nil { return err } if s.afterCommit != nil && s.afterUsed.CompareAndSwap(false, true) { @@ -84,16 +88,35 @@ func newConformanceRuntimeWithErrors(t *testing.T, sourceClock *clock.Fake, stat } if observed == nil { if err := shadow.RegisterConformance(rt, application); err != nil { - rt.Kill() + _ = crashRuntime(rt) t.Fatal(err) } } else if err := shadow.RegisterConformanceWithObservation(rt, application, observed); err != nil { - rt.Kill() + _ = crashRuntime(rt) t.Fatal(err) } + startRuntime(t, rt) return rt } +func startRuntime(t *testing.T, rt *gor.Runtime) { + t.Helper() + if err := rt.Start(context.Background()); err != nil { + _ = crashRuntime(rt) + t.Fatalf("start Runtime: %v", err) + } +} + +func shutdownRuntime(rt *gor.Runtime) error { + return rt.Shutdown(context.Background()) +} + +func crashRuntime(rt *gor.Runtime) error { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return rt.Shutdown(ctx) +} + func hasReminder(rows []store.Reminder, grainType, grainKey, name string) bool { for _, row := range rows { if row.GrainId.GrainType == grainType && row.GrainId.GrainKey == grainKey && row.Name == name { @@ -103,6 +126,18 @@ func hasReminder(rows []store.Reminder, grainType, grainKey, name string) bool { return false } +func listDueReminders(t *testing.T, reminderStore store.ReminderStore, now time.Time) []store.Reminder { + t.Helper() + page, err := reminderStore.ListDue(context.Background(), now, nil, 1024) + if err != nil { + t.Fatalf("ListDue: %v", err) + } + if page.Next != nil { + t.Fatal("Reminder test helper limit is too small") + } + return page.Rows +} + func hasCall(calls chan gor.CallObservation, method string, wantErr error) bool { found := false for len(calls) > 0 { diff --git a/examples/shadow/conformance_report_action_test.go b/examples/shadow/conformance_report_action_test.go index cad6b24..722b5e3 100644 --- a/examples/shadow/conformance_report_action_test.go +++ b/examples/shadow/conformance_report_action_test.go @@ -18,7 +18,7 @@ func TestConformance_ReportActionConflictLeavesShadowUnchanged(t *testing.T) { sourceClock := clock.NewFake(time.Unix(1900, 0).UTC()) application := domain.NewMemoryApplicationStore() rt := newConformanceRuntime(t, sourceClock, store.NewMemory(), store.NewMemory(), application, nil, nil) - defer rt.Kill() + defer crashRuntime(rt) device := gor.Ref[domain.Device](rt, "device-1") if err := device.ReportAction(context.Background(), "action-existing", "temperature=20"); err != nil { @@ -50,3 +50,88 @@ func TestConformance_ReportActionConflictLeavesShadowUnchanged(t *testing.T) { } }) } + +func TestConformance_ApplyPendingRejectsDifferentDevice(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + application := domain.NewMemoryApplicationStore() + if err := application.SavePending(context.Background(), domain.PendingAction{ + ActionID: "action-target", + DeviceKey: "device-1", + State: "temperature=28", + }); err != nil { + t.Fatal(err) + } + rt := newConformanceRuntime(t, clock.NewFake(time.Unix(1925, 0).UTC()), store.NewMemory(), store.NewMemory(), application, nil, nil) + defer crashRuntime(rt) + + err := gor.Ref[domain.Device](rt, "device-2").ApplyPending(context.Background(), "action-target") + if !errors.Is(err, domain.ErrPendingActionTarget) { + t.Fatalf("ApplyPending error = %v, want ErrPendingActionTarget", err) + } + if record, applied, err := application.ReadApplied(context.Background(), "action-target"); err != nil || applied { + t.Fatalf("receipt after wrong Device = (%#v, %v, %v), want absent", record, applied, err) + } + if pending, err := application.ListPending(context.Background()); err != nil || len(pending) != 1 { + t.Fatalf("pending after wrong Device = (%#v, %v), want one action", pending, err) + } + if exists, err := gor.Ref[domain.Device](rt, "device-2").ShadowExists(context.Background()); err != nil || exists { + t.Fatalf("wrong Device State exists = (%v, %v), want absent", exists, err) + } + if err := gor.Ref[domain.Device](rt, "device-1").ApplyPending(context.Background(), "action-target"); err != nil { + t.Fatalf("ApplyPending on target Device: %v", err) + } + err = gor.Ref[domain.Device](rt, "device-2").ApplyPending(context.Background(), "action-target") + if !errors.Is(err, domain.ErrPendingActionTarget) { + t.Fatalf("applied action on wrong Device error = %v, want ErrPendingActionTarget", err) + } + }) +} + +func TestConformance_RecoveryUsesPendingSaveOrder(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(1935, 0).UTC()) + application := domain.NewMemoryApplicationStore() + rt := newConformanceRuntime(t, sourceClock, store.NewMemory(), store.NewMemory(), application, nil, nil) + defer crashRuntime(rt) + if err := gor.Ref[domain.RecoveryCoordinator](rt, domain.RecoveryCoordinatorKey).Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + device := gor.Ref[domain.Device](rt, "device-1") + if err := device.ReportAction(context.Background(), "z-first", "temperature=28"); err != nil { + t.Fatalf("first ReportAction: %v", err) + } + if err := device.ReportAction(context.Background(), "a-second", "temperature=29"); err != nil { + t.Fatalf("second ReportAction: %v", err) + } + + sourceClock.Advance(domain.RecoveryInterval) + synctest.Wait() + shadow, err := device.Shadow(context.Background()) + if err != nil || shadow.ReportedState != "temperature=29" || !shadow.Online { + t.Fatalf("Shadow after ordered recovery = (%#v, %v), want second saved action", shadow, err) + } + for _, actionID := range []string{"z-first", "a-second"} { + if record, applied, err := application.ReadApplied(context.Background(), actionID); err != nil || !applied { + t.Fatalf("receipt %q after ordered recovery = (%#v, %v, %v), want applied", actionID, record, applied, err) + } + } + if pending, err := application.ListPending(context.Background()); err != nil || len(pending) != 0 { + t.Fatalf("pending after ordered recovery = (%#v, %v), want none", pending, err) + } + + if err := device.ApplyPending(context.Background(), "z-first"); err != nil { + t.Fatalf("repeat old action: %v", err) + } + shadow, err = device.Shadow(context.Background()) + if err != nil || shadow.ReportedState != "temperature=29" { + t.Fatalf("Shadow after old receipt repeat = (%#v, %v), want unchanged", shadow, err) + } + if err := device.ReportAction(context.Background(), "z-first", "temperature=28"); err != nil { + t.Fatalf("repeat completed ReportAction: %v", err) + } + shadow, err = device.Shadow(context.Background()) + if err != nil || shadow.ReportedState != "temperature=29" { + t.Fatalf("Shadow after completed ReportAction repeat = (%#v, %v), want unchanged", shadow, err) + } + }) +} diff --git a/examples/shadow/conformance_state_failure_test.go b/examples/shadow/conformance_state_failure_test.go new file mode 100644 index 0000000..44af180 --- /dev/null +++ b/examples/shadow/conformance_state_failure_test.go @@ -0,0 +1,247 @@ +package shadow_test + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/suraciii/gor" + "github.com/suraciii/gor/clock" + "github.com/suraciii/gor/examples/shadow" + "github.com/suraciii/gor/examples/shadow/domain" + "github.com/suraciii/gor/store" +) + +type appliedStateFailureStore struct { + *store.Memory + err error + failed atomic.Bool + writes atomic.Int64 +} + +func (s *appliedStateFailureStore) Write(ctx context.Context, id store.GrainId, data []byte, expect store.ETag) (store.ETag, error) { + if id.GrainType == "domain.Device" && id.GrainKey == "device-1" { + s.writes.Add(1) + } + etag, err := s.Memory.Write(ctx, id, data, expect) + if err != nil { + return 0, err + } + if id.GrainType == "domain.Device" && id.GrainKey == "device-1" && s.failed.CompareAndSwap(false, true) { + return 0, s.err + } + return etag, nil +} + +type rejectedStateFailureStore struct { + *store.Memory + err error + failed atomic.Bool +} + +func (s *rejectedStateFailureStore) Write(ctx context.Context, id store.GrainId, data []byte, expect store.ETag) (store.ETag, error) { + if id.GrainType == "domain.Device" && id.GrainKey == "device-1" && s.failed.CompareAndSwap(false, true) { + return 0, s.err + } + return s.Memory.Write(ctx, id, data, expect) +} + +func TestConformance_RecoveryStateFailureDoesNotCompleteAction(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + stateErr := errors.New("recovery State write failed before commit") + stateStore := &rejectedStateFailureStore{Memory: store.NewMemory(), err: stateErr} + application := domain.NewMemoryApplicationStore() + if err := application.SavePending(context.Background(), domain.PendingAction{ + ActionID: "action-recovery-failure", + DeviceKey: "device-1", + State: "temperature=25", + }); err != nil { + t.Fatal(err) + } + sourceClock := clock.NewFake(time.Unix(1940, 0).UTC()) + errorsSeen := make(chan gor.BackgroundError, 4) + rt := newConformanceRuntimeWithErrors(t, sourceClock, stateStore, store.NewMemory(), application, nil, nil, errorsSeen) + defer crashRuntime(rt) + if err := gor.Ref[domain.RecoveryCoordinator](rt, domain.RecoveryCoordinatorKey).Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + + sourceClock.Advance(domain.RecoveryInterval) + synctest.Wait() + background := <-errorsSeen + if !errors.Is(background.Err, stateErr) || !errors.Is(background.Err, gor.ErrPersistenceFailed) { + t.Fatalf("OnError = %#v, want recovery State failure", background) + } + if record, applied, err := application.ReadApplied(context.Background(), "action-recovery-failure"); err != nil || applied { + t.Fatalf("receipt after recovery State failure = (%#v, %v, %v), want absent", record, applied, err) + } + if pending, err := application.ListPending(context.Background()); err != nil || len(pending) != 1 { + t.Fatalf("pending after recovery State failure = (%#v, %v), want one action", pending, err) + } + + sourceClock.Advance(domain.RecoveryInterval) + synctest.Wait() + shadow, err := gor.Ref[domain.Device](rt, "device-1").Shadow(context.Background()) + if err != nil || shadow.ReportedState != "temperature=25" || !shadow.Online { + t.Fatalf("Shadow after recovery retry = (%#v, %v), want requested State", shadow, err) + } + if record, applied, err := application.ReadApplied(context.Background(), "action-recovery-failure"); err != nil || !applied || record.State != "temperature=25" { + t.Fatalf("receipt after recovery retry = (%#v, %v, %v), want applied action", record, applied, err) + } + }) +} + +func TestConformance_StateFailureKeepsActionPendingUntilRecovery(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + stateErr := errors.New("State write failed before commit") + stateStore := &rejectedStateFailureStore{Memory: store.NewMemory(), err: stateErr} + reminderStore := store.NewMemory() + application := domain.NewMemoryApplicationStore() + sourceClock := clock.NewFake(time.Unix(1950, 0).UTC()) + rt := newConformanceRuntime(t, sourceClock, stateStore, reminderStore, application, nil, nil) + coordinator := gor.Ref[domain.RecoveryCoordinator](rt, domain.RecoveryCoordinatorKey) + if err := coordinator.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + + err := gor.Ref[domain.Device](rt, "device-1").ReportAction(context.Background(), "action-state-failure", "temperature=26") + if !errors.Is(err, stateErr) || !errors.Is(err, gor.ErrPersistenceFailed) { + t.Fatalf("ReportAction error = %v, want State persistence failure", err) + } + if record, applied, err := application.ReadApplied(context.Background(), "action-state-failure"); err != nil || applied { + t.Fatalf("receipt after State failure = (%#v, %v, %v), want absent", record, applied, err) + } + if pending, err := application.ListPending(context.Background()); err != nil || len(pending) != 1 { + t.Fatalf("pending after State failure = (%#v, %v), want one action", pending, err) + } + if err := crashRuntime(rt); err != nil && !errors.Is(err, context.Canceled) { + t.Fatalf("stop after State failure: %v", err) + } + + rt = newConformanceRuntime(t, sourceClock, stateStore, reminderStore, application, nil, nil) + defer crashRuntime(rt) + if err := gor.Ref[domain.RecoveryCoordinator](rt, domain.RecoveryCoordinatorKey).Start(context.Background()); err != nil { + t.Fatalf("restart Start: %v", err) + } + sourceClock.Advance(domain.RecoveryInterval) + synctest.Wait() + + shadow, err := gor.Ref[domain.Device](rt, "device-1").Shadow(context.Background()) + if err != nil { + t.Fatalf("Shadow after recovery: %v", err) + } + if shadow.ReportedState != "temperature=26" || !shadow.Online { + t.Fatalf("Shadow after recovery = %#v, want requested State", shadow) + } + if record, applied, err := application.ReadApplied(context.Background(), "action-state-failure"); err != nil || !applied || record.State != "temperature=26" { + t.Fatalf("receipt after State recovery = (%#v, %v, %v), want applied action", record, applied, err) + } + if pending, err := application.ListPending(context.Background()); err != nil || len(pending) != 0 { + t.Fatalf("pending after State recovery = (%#v, %v), want none", pending, err) + } + }) +} + +func TestConformance_StateUnknownResultRepeatsAfterRestart(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + unknownErr := errors.New("State write reply was lost") + stateStore := &appliedStateFailureStore{Memory: store.NewMemory(), err: unknownErr} + reminderStore := store.NewMemory() + application := domain.NewMemoryApplicationStore() + sourceClock := clock.NewFake(time.Unix(1975, 0).UTC()) + rt := newConformanceRuntime(t, sourceClock, stateStore, reminderStore, application, nil, nil) + if err := gor.Ref[domain.RecoveryCoordinator](rt, domain.RecoveryCoordinatorKey).Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + + err := gor.Ref[domain.Device](rt, "device-1").ReportAction(context.Background(), "action-state-unknown", "temperature=27") + if !errors.Is(err, unknownErr) || !errors.Is(err, gor.ErrPersistenceFailed) { + t.Fatalf("ReportAction error = %v, want unknown State result", err) + } + if err := crashRuntime(rt); err != nil && !errors.Is(err, context.Canceled) { + t.Fatalf("stop after unknown State result: %v", err) + } + + rt = newConformanceRuntime(t, sourceClock, stateStore, reminderStore, application, nil, nil) + defer crashRuntime(rt) + if err := gor.Ref[domain.RecoveryCoordinator](rt, domain.RecoveryCoordinatorKey).Start(context.Background()); err != nil { + t.Fatalf("restart Start: %v", err) + } + if err := gor.Ref[domain.Device](rt, "device-1").ReportAction(context.Background(), "action-state-unknown", "temperature=27"); err != nil { + t.Fatalf("Safe Repeat ReportAction: %v", err) + } + sourceClock.Advance(domain.RecoveryInterval) + synctest.Wait() + + shadow, err := gor.Ref[domain.Device](rt, "device-1").Shadow(context.Background()) + if err != nil || shadow.ReportedState != "temperature=27" || !shadow.Online { + t.Fatalf("Shadow after unknown State result = (%#v, %v), want requested State", shadow, err) + } + if stateStore.writes.Load() != 1 { + t.Fatalf("Device State writes = %d, want one committed attempt", stateStore.writes.Load()) + } + if record, applied, err := application.ReadApplied(context.Background(), "action-state-unknown"); err != nil || !applied || record.State != "temperature=27" { + t.Fatalf("receipt after unknown State result = (%#v, %v, %v), want applied action", record, applied, err) + } + }) +} + +func TestConformance_StateUnknownResultDiscardsActivationAndReloadsStore(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + unknownErr := errors.New("State write reply was lost") + stateStore := &appliedStateFailureStore{Memory: store.NewMemory(), err: unknownErr} + reminderStore := store.NewMemory() + sourceClock := clock.NewFake(time.Unix(2000, 0).UTC()) + lifecycle := make(chan domain.LifecycleEvent, 4) + + rt, err := gor.New( + gor.WithStore(stateStore), + gor.WithReminderStore(reminderStore), + gor.WithClock(sourceClock), + gor.WithIdleTimeout(0), + gor.WithEvictionInterval(0), + ) + if err != nil { + t.Fatal(err) + } + if err := shadow.RegisterWithLifecycle(rt, lifecycle); err != nil { + t.Fatal(err) + } + startRuntime(t, rt) + defer crashRuntime(rt) + + device := gor.Ref[domain.Device](rt, "device-1") + if err := device.Configure(context.Background(), "sample-rate=5s"); !errors.Is(err, unknownErr) { + t.Fatalf("Configure error = %v, want %v", err, unknownErr) + } else if !errors.Is(err, gor.ErrPersistenceFailed) { + t.Fatalf("Configure error = %v, want ErrPersistenceFailed", err) + } + synctest.Wait() + assertLifecycleEvent(t, lifecycle, domain.LifecycleActivated) + assertLifecycleEvent(t, lifecycle, domain.LifecycleDeactivated) + + value, err := device.Shadow(context.Background()) + if err != nil { + t.Fatalf("Shadow after unknown result: %v", err) + } + if value.Configuration != "sample-rate=5s" { + t.Fatalf("restored Configuration = %q, want sample-rate=5s", value.Configuration) + } + assertLifecycleEvent(t, lifecycle, domain.LifecycleActivated) + }) +} + +func assertLifecycleEvent(t *testing.T, events <-chan domain.LifecycleEvent, kind string) { + t.Helper() + select { + case event := <-events: + if event.Kind != kind { + t.Fatalf("Lifecycle event = %#v, want kind %q", event, kind) + } + default: + t.Fatalf("Lifecycle event %q was not ready", kind) + } +} diff --git a/examples/shadow/conformance_stop_test.go b/examples/shadow/conformance_stop_test.go index 302a9b6..34a5960 100644 --- a/examples/shadow/conformance_stop_test.go +++ b/examples/shadow/conformance_stop_test.go @@ -31,18 +31,12 @@ func TestConformance_StopInterruptionRestartsCoordinator(t *testing.T) { if err := coordinator.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } - before, err := baseReminders.ListDue(context.Background(), start.Add(2*domain.RecoveryInterval)) - if err != nil { - t.Fatal(err) - } + before := listDueReminders(t, baseReminders, start.Add(2*domain.RecoveryInterval)) beforeReminder := findRecoveryReminder(t, before) if err := coordinator.Start(context.Background()); err != nil { t.Fatalf("idempotent Start: %v", err) } - after, err := baseReminders.ListDue(context.Background(), start.Add(2*domain.RecoveryInterval)) - if err != nil { - t.Fatal(err) - } + after := listDueReminders(t, baseReminders, start.Add(2*domain.RecoveryInterval)) afterReminder := findRecoveryReminder(t, after) if !afterReminder.FirstTickTime.Equal(beforeReminder.FirstTickTime) { t.Fatalf("idempotent Start changed FirstTickTime from %s to %s", beforeReminder.FirstTickTime, afterReminder.FirstTickTime) @@ -58,7 +52,7 @@ func TestConformance_StopInterruptionRestartsCoordinator(t *testing.T) { <-blockingReminders.deleted coordinatorRecord, err := stateStore.Read(context.Background(), store.GrainId{ - GrainType: gor.TypeName[domain.RecoveryCoordinator](), + GrainType: "domain.RecoveryCoordinator", GrainKey: domain.RecoveryCoordinatorKey, }) if err != nil { @@ -67,17 +61,14 @@ func TestConformance_StopInterruptionRestartsCoordinator(t *testing.T) { if string(coordinatorRecord.Data) != `{}` { t.Fatalf("Coordinator State while Cancel is blocked = %s, want cleared state", coordinatorRecord.Data) } - remaining, err := baseReminders.ListDue(context.Background(), start.Add(2*domain.RecoveryInterval)) - if err != nil { - t.Fatal(err) - } - if hasReminder(remaining, gor.TypeName[domain.RecoveryCoordinator](), domain.RecoveryCoordinatorKey, domain.RecoveryReminderName) { + remaining := listDueReminders(t, baseReminders, start.Add(2*domain.RecoveryInterval)) + if hasReminder(remaining, "domain.RecoveryCoordinator", domain.RecoveryCoordinatorKey, domain.RecoveryReminderName) { t.Fatalf("recovery Reminder remains after Delete completed: %#v", remaining) } killDone := make(chan struct{}) go func() { - rt.Kill() + _ = crashRuntime(rt) close(killDone) }() <-rt.Done() @@ -88,14 +79,11 @@ func TestConformance_StopInterruptionRestartsCoordinator(t *testing.T) { } rt = newConformanceRuntime(t, sourceClock, stateStore, baseReminders, application, observed, nil) - defer rt.Kill() + defer crashRuntime(rt) if err := gor.Ref[domain.RecoveryCoordinator](rt, domain.RecoveryCoordinatorKey).Start(context.Background()); err != nil { t.Fatalf("restart Start: %v", err) } - rows, err := baseReminders.ListDue(context.Background(), start.Add(2*domain.RecoveryInterval)) - if err != nil { - t.Fatal(err) - } + rows := listDueReminders(t, baseReminders, start.Add(2*domain.RecoveryInterval)) findRecoveryReminder(t, rows) sourceClock.Advance(domain.RecoveryInterval) synctest.Wait() @@ -123,7 +111,7 @@ func (s *blockingDeleteReminderStore) Delete(ctx context.Context, id store.Grain func findRecoveryReminder(t *testing.T, rows []store.Reminder) store.Reminder { t.Helper() for _, row := range rows { - if row.GrainId.GrainType == gor.TypeName[domain.RecoveryCoordinator]() && row.GrainId.GrainKey == domain.RecoveryCoordinatorKey && row.Name == domain.RecoveryReminderName { + if row.GrainId.GrainType == "domain.RecoveryCoordinator" && row.GrainId.GrainKey == domain.RecoveryCoordinatorKey && row.Name == domain.RecoveryReminderName { return row } } diff --git a/examples/shadow/domain/application.go b/examples/shadow/domain/application.go index 7b2189e..068c41b 100644 --- a/examples/shadow/domain/application.go +++ b/examples/shadow/domain/application.go @@ -7,7 +7,7 @@ import ( "fmt" "net/url" "path/filepath" - "sort" + "strings" "sync" _ "modernc.org/sqlite" @@ -18,7 +18,8 @@ import ( type ApplicationStore interface { SavePending(context.Context, PendingAction) error ListPending(context.Context) ([]PendingAction, error) - ApplyPending(context.Context, string) error + ReadPending(context.Context, string) (PendingAction, bool, error) + CompletePending(context.Context, string) error ReadApplied(context.Context, string) (AppliedRecord, bool, error) Close() error } @@ -44,14 +45,17 @@ var ( ErrPendingActionConflict = errors.New("application action ID has a different payload") // ErrPendingActionNotFound reports an action that is neither pending nor applied. ErrPendingActionNotFound = errors.New("application pending action was not found") + // ErrPendingActionTarget reports a pending action for a different Device. + ErrPendingActionTarget = errors.New("application pending action targets a different Device") ) // MemoryApplicationStore is an in-memory ApplicationStore for deterministic // example tests. type MemoryApplicationStore struct { - mu sync.Mutex - pending map[string]PendingAction - applied map[string]AppliedRecord + mu sync.Mutex + pending map[string]PendingAction + pendingOrder []string + applied map[string]AppliedRecord } var _ ApplicationStore = (*MemoryApplicationStore)(nil) @@ -87,10 +91,11 @@ func (s *MemoryApplicationStore) SavePending(ctx context.Context, action Pending return nil } s.pending[action.ActionID] = action + s.pendingOrder = append(s.pendingOrder, action.ActionID) return nil } -// ListPending returns pending actions in ActionID order. +// ListPending returns pending actions in save order. func (s *MemoryApplicationStore) ListPending(ctx context.Context) ([]PendingAction, error) { if err := ctx.Err(); err != nil { return nil, err @@ -98,16 +103,28 @@ func (s *MemoryApplicationStore) ListPending(ctx context.Context) ([]PendingActi s.mu.Lock() defer s.mu.Unlock() result := make([]PendingAction, 0, len(s.pending)) - for _, action := range s.pending { - result = append(result, action) + for _, actionID := range s.pendingOrder { + if action, ok := s.pending[actionID]; ok { + result = append(result, action) + } } - sort.Slice(result, func(i, j int) bool { return result[i].ActionID < result[j].ActionID }) return result, nil } -// ApplyPending applies one action and creates one receipt. Repeating an -// applied ActionID succeeds without changing the receipt. -func (s *MemoryApplicationStore) ApplyPending(ctx context.Context, actionID string) error { +// ReadPending reads one pending action. +func (s *MemoryApplicationStore) ReadPending(ctx context.Context, actionID string) (PendingAction, bool, error) { + if err := ctx.Err(); err != nil { + return PendingAction{}, false, err + } + s.mu.Lock() + defer s.mu.Unlock() + action, ok := s.pending[actionID] + return action, ok, nil +} + +// CompletePending creates one receipt and removes the pending action. +// Repeating an applied ActionID succeeds without changing the receipt. +func (s *MemoryApplicationStore) CompletePending(ctx context.Context, actionID string) error { if err := ctx.Err(); err != nil { return err } @@ -122,6 +139,12 @@ func (s *MemoryApplicationStore) ApplyPending(ctx context.Context, actionID stri } s.applied[actionID] = AppliedRecord(action) delete(s.pending, actionID) + for index, pendingID := range s.pendingOrder { + if pendingID == actionID { + s.pendingOrder = append(s.pendingOrder[:index], s.pendingOrder[index+1:]...) + break + } + } return nil } @@ -183,7 +206,11 @@ CREATE TABLE IF NOT EXISTS applied_records ( func applicationSQLiteDSN(path string) string { absolute, _ := filepath.Abs(path) - fileURI := (&url.URL{Scheme: "file", Path: filepath.ToSlash(absolute)}).String() + slashPath := filepath.ToSlash(absolute) + if filepath.VolumeName(absolute) != "" && !strings.HasPrefix(slashPath, "/") { + slashPath = "/" + slashPath + } + fileURI := (&url.URL{Scheme: "file", Path: slashPath}).String() return fmt.Sprintf("%s?_pragma=journal_mode(WAL)&_pragma=synchronous(FULL)&_pragma=busy_timeout(5000)", fileURI) } @@ -230,12 +257,12 @@ func (s *SQLiteApplicationStore) SavePending(ctx context.Context, action Pending return tx.Commit() } -// ListPending returns pending actions in ActionID order. +// ListPending returns pending actions in save order. func (s *SQLiteApplicationStore) ListPending(ctx context.Context) ([]PendingAction, error) { rows, err := s.db.QueryContext(ctx, ` SELECT action_id, device_key, state, trace_id FROM pending_actions -ORDER BY action_id`) +ORDER BY rowid`) if err != nil { return nil, err } @@ -254,9 +281,14 @@ ORDER BY action_id`) return result, nil } -// ApplyPending applies one action in one application transaction. The unique -// ActionID makes a repeat a successful no-op. -func (s *SQLiteApplicationStore) ApplyPending(ctx context.Context, actionID string) error { +// ReadPending reads one pending action. +func (s *SQLiteApplicationStore) ReadPending(ctx context.Context, actionID string) (PendingAction, bool, error) { + return readPending(ctx, s.db, actionID) +} + +// CompletePending creates one receipt and removes the pending action in one +// Application transaction. The unique ActionID makes a repeat a no-op. +func (s *SQLiteApplicationStore) CompletePending(ctx context.Context, actionID string) error { if err := ctx.Err(); err != nil { return err } diff --git a/examples/shadow/domain/application_test.go b/examples/shadow/domain/application_test.go index 42c7a11..df92b3b 100644 --- a/examples/shadow/domain/application_test.go +++ b/examples/shadow/domain/application_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "runtime" "testing" ) @@ -34,14 +35,20 @@ func TestApplicationStore_SafeRepeatAndActionIDConflict(t *testing.T) { t.Fatalf("conflicting SavePending = %v, want conflict", err) } pending, err := application.ListPending(ctx) - if err != nil || len(pending) != 2 || pending[0].ActionID != "a" || pending[1].ActionID != "b" { - t.Fatalf("ListPending = (%#v, %v), want ActionID order", pending, err) + if err != nil || len(pending) != 2 || pending[0].ActionID != "b" || pending[1].ActionID != "a" { + t.Fatalf("ListPending = (%#v, %v), want save order", pending, err) } - if err := application.ApplyPending(ctx, "a"); err != nil { + if action, ok, err := application.ReadPending(ctx, "a"); err != nil || !ok || action != second { + t.Fatalf("ReadPending = (%#v, %v, %v), want second action", action, ok, err) + } + if err := application.CompletePending(ctx, "a"); err != nil { t.Fatal(err) } - if err := application.ApplyPending(ctx, "a"); err != nil { - t.Fatalf("Safe Repeat ApplyPending: %v", err) + if err := application.CompletePending(ctx, "a"); err != nil { + t.Fatalf("Safe Repeat CompletePending: %v", err) + } + if action, ok, err := application.ReadPending(ctx, "a"); err != nil || ok || action != (PendingAction{}) { + t.Fatalf("ReadPending after completion = (%#v, %v, %v), want absent", action, ok, err) } record, ok, err := application.ReadApplied(ctx, "a") if err != nil || !ok || record.State != "one" || record.TraceID != "trace-a" { @@ -61,6 +68,10 @@ func TestSQLiteApplicationStore_PersistsPendingAndAppliedRecords(t *testing.T) { first.Close() t.Fatal(err) } + if err := first.SavePending(context.Background(), PendingAction{ActionID: "persisted-second", DeviceKey: "device-1", State: "second", TraceID: "trace-2"}); err != nil { + first.Close() + t.Fatal(err) + } if err := first.Close(); err != nil { t.Fatal(err) } @@ -68,11 +79,14 @@ func TestSQLiteApplicationStore_PersistsPendingAndAppliedRecords(t *testing.T) { if err != nil { t.Fatal(err) } - if err := second.ApplyPending(context.Background(), "persisted"); err != nil { - second.Close() + defer second.Close() + pending, err := second.ListPending(context.Background()) + if err != nil || len(pending) != 2 || pending[0].ActionID != "persisted" || pending[1].ActionID != "persisted-second" { + t.Fatalf("ListPending after reopen = (%#v, %v), want persisted save order", pending, err) + } + if err := second.CompletePending(context.Background(), "persisted"); err != nil { t.Fatal(err) } - defer second.Close() record, ok, err := second.ReadApplied(context.Background(), "persisted") if err != nil || !ok || record.ActionID != "persisted" { t.Fatalf("ReadApplied after reopen = (%#v, %v, %v), want persisted receipt", record, ok, err) @@ -80,7 +94,11 @@ func TestSQLiteApplicationStore_PersistsPendingAndAppliedRecords(t *testing.T) { } func TestSQLiteApplicationStore_PathWithURICharactersUsesRequestedFile(t *testing.T) { - path := filepath.Join(t.TempDir(), "business?#.db") + name := "business?#.db" + if runtime.GOOS == "windows" { + name = "business%.db" + } + path := filepath.Join(t.TempDir(), name) first, err := OpenSQLiteApplicationStore(path) if err != nil { t.Fatal(err) diff --git a/examples/shadow/domain/domain.go b/examples/shadow/domain/domain.go index 08b1879..4ed2b81 100644 --- a/examples/shadow/domain/domain.go +++ b/examples/shadow/domain/domain.go @@ -1,5 +1,7 @@ package domain +//go:generate go tool gorgen -pkg . + import ( "context" "errors" @@ -76,7 +78,7 @@ type Workshop interface { } type device struct { - binder *gor.Binder + grainContext *gor.GrainContext id gor.GrainId shadow gor.State[Shadow] schedule gor.Reminder[Device] @@ -84,25 +86,25 @@ type device struct { lifecycleEvents chan<- LifecycleEvent } -func NewDevice(b *gor.Binder) Device { +func NewDevice(b *gor.GrainContext) Device { return newDevice(b, nil, nil) } -func NewDeviceWithLifecycle(b *gor.Binder, events chan<- LifecycleEvent) Device { +func NewDeviceWithLifecycle(b *gor.GrainContext, events chan<- LifecycleEvent) Device { return newDevice(b, events, nil) } -func NewDeviceWithApplication(b *gor.Binder, application ApplicationStore) Device { +func NewDeviceWithApplication(b *gor.GrainContext, application ApplicationStore) Device { return newDevice(b, nil, application) } -func NewDeviceWithApplicationAndLifecycle(b *gor.Binder, application ApplicationStore, events chan<- LifecycleEvent) Device { +func NewDeviceWithApplicationAndLifecycle(b *gor.GrainContext, application ApplicationStore, events chan<- LifecycleEvent) Device { return newDevice(b, events, application) } -func newDevice(b *gor.Binder, events chan<- LifecycleEvent, application ApplicationStore) Device { +func newDevice(b *gor.GrainContext, events chan<- LifecycleEvent, application ApplicationStore) Device { return &device{ - binder: b, + grainContext: b, id: gor.Self(b), shadow: gor.NewState[Shadow](b, "shadow"), schedule: gor.NewReminder[Device](b), @@ -118,7 +120,7 @@ func (d *device) Report(ctx context.Context, workshopID string, state string) er previous := d.shadow.Get() next := previous next.ReportedState = state - next.ReportedAt = gor.Now(d.binder) + next.ReportedAt = gor.Now(d.grainContext) next.Online = true next.WorkshopID = workshopID if err := d.shadow.Set(ctx, next); err != nil { @@ -129,12 +131,12 @@ func (d *device) Report(ctx context.Context, workshopID string, state string) er } if previous.Online && previous.WorkshopID != workshopID { - if err := gor.Ref[Workshop](d.binder, previous.WorkshopID).DeviceOffline(ctx, d.id.GrainKey); err != nil { + if err := gor.Ref[Workshop](d.grainContext, previous.WorkshopID).DeviceOffline(ctx, d.id.GrainKey); err != nil { return err } } if !previous.Online || previous.WorkshopID != workshopID { - if err := gor.Ref[Workshop](d.binder, workshopID).DeviceOnline(ctx, d.id.GrainKey); err != nil { + if err := gor.Ref[Workshop](d.grainContext, workshopID).DeviceOnline(ctx, d.id.GrainKey); err != nil { return err } } @@ -160,11 +162,12 @@ func (d *device) ReportAction(ctx context.Context, actionID string, state string }); err != nil { return err } - shadow := d.shadow.Get() - shadow.ReportedState = state - shadow.ReportedAt = gor.Now(d.binder) - shadow.Online = true - return d.shadow.Set(ctx, shadow) + if _, applied, err := d.application.ReadApplied(ctx, actionID); err != nil { + return err + } else if applied { + return nil + } + return d.confirmReportedState(ctx, state) } func (d *device) Configure(ctx context.Context, configuration string) error { @@ -189,7 +192,48 @@ func (d *device) ApplyPending(ctx context.Context, actionID string) error { if d.application == nil { return errors.New("application store is not configured") } - return d.application.ApplyPending(ctx, actionID) + if record, applied, err := d.application.ReadApplied(ctx, actionID); err != nil { + return err + } else if applied { + return d.checkActionTarget(actionID, record.DeviceKey) + } + action, pending, err := d.application.ReadPending(ctx, actionID) + if err != nil { + return err + } + if !pending { + if record, applied, err := d.application.ReadApplied(ctx, actionID); err != nil { + return err + } else if applied { + return d.checkActionTarget(actionID, record.DeviceKey) + } + return ErrPendingActionNotFound + } + if err := d.checkActionTarget(actionID, action.DeviceKey); err != nil { + return err + } + if err := d.confirmReportedState(ctx, action.State); err != nil { + return err + } + return d.application.CompletePending(ctx, actionID) +} + +func (d *device) confirmReportedState(ctx context.Context, state string) error { + shadow := d.shadow.Get() + if shadow.ReportedState == state && shadow.Online { + return nil + } + shadow.ReportedState = state + shadow.ReportedAt = gor.Now(d.grainContext) + shadow.Online = true + return d.shadow.Set(ctx, shadow) +} + +func (d *device) checkActionTarget(actionID string, deviceKey string) error { + if deviceKey != d.id.GrainKey { + return fmt.Errorf("%w: action %q targets %q, not %q", ErrPendingActionTarget, actionID, deviceKey, d.id.GrainKey) + } + return nil } func (d *device) OnActivate(context.Context) error { @@ -228,7 +272,7 @@ func (d *device) MarkOffline(ctx context.Context, _ gor.TickStatus) error { if err := d.shadow.Set(ctx, shadow); err != nil { return err } - return gor.Ref[Workshop](d.binder, shadow.WorkshopID).DeviceOffline(ctx, d.id.GrainKey) + return gor.Ref[Workshop](d.grainContext, shadow.WorkshopID).DeviceOffline(ctx, d.id.GrainKey) } type workshop struct { @@ -237,15 +281,15 @@ type workshop struct { lifecycleEvents chan<- LifecycleEvent } -func NewWorkshop(b *gor.Binder) Workshop { +func NewWorkshop(b *gor.GrainContext) Workshop { return newWorkshop(b, nil) } -func NewWorkshopWithLifecycle(b *gor.Binder, events chan<- LifecycleEvent) Workshop { +func NewWorkshopWithLifecycle(b *gor.GrainContext, events chan<- LifecycleEvent) Workshop { return newWorkshop(b, events) } -func newWorkshop(b *gor.Binder, events chan<- LifecycleEvent) Workshop { +func newWorkshop(b *gor.GrainContext, events chan<- LifecycleEvent) Workshop { return &workshop{ id: gor.Self(b), online: gor.NewState[map[string]struct{}](b, "online"), diff --git a/examples/shadow/domain/gorgen/generated.go b/examples/shadow/domain/gorgen/generated.go index 9528eff..ee59a92 100644 --- a/examples/shadow/domain/gorgen/generated.go +++ b/examples/shadow/domain/gorgen/generated.go @@ -1,3 +1,5 @@ +// Code generated by gorgen. DO NOT EDIT. + package gorgen import ( @@ -8,131 +10,135 @@ import ( "github.com/suraciii/gor/examples/shadow/domain" ) -type deviceProxy struct { +const generatedCodeVersion = 1 + +const gorgen6_DeviceGrainType gor.GrainType = "domain.Device" + +type gorgen6_DeviceProxy struct { id gor.GrainId rt gor.Invoker } -type deviceApplyPendingRequest struct { +type gorgen6_Device12_ApplyPendingRequest struct { A0 string } -type deviceApplyPendingReply struct{} +type gorgen6_Device12_ApplyPendingReply struct{} -func (p *deviceProxy) ApplyPending(ctx context.Context, actionID string) error { - var reply deviceApplyPendingReply - err := p.rt.Invoke(ctx, p.id, "ApplyPending", &deviceApplyPendingRequest{A0: actionID}, &reply) +func (p *gorgen6_DeviceProxy) ApplyPending(ctx context.Context, arg0 string) error { + var reply gorgen6_Device12_ApplyPendingReply + err := p.rt.Invoke(ctx, p.id, "ApplyPending", &gorgen6_Device12_ApplyPendingRequest{A0: arg0}, &reply) return err } -type deviceClearShadowRequest struct{} -type deviceClearShadowReply struct{} +type gorgen6_Device11_ClearShadowRequest struct{} +type gorgen6_Device11_ClearShadowReply struct{} -func (p *deviceProxy) ClearShadow(ctx context.Context) error { - var reply deviceClearShadowReply - err := p.rt.Invoke(ctx, p.id, "ClearShadow", &deviceClearShadowRequest{}, &reply) +func (p *gorgen6_DeviceProxy) ClearShadow(ctx context.Context) error { + var reply gorgen6_Device11_ClearShadowReply + err := p.rt.Invoke(ctx, p.id, "ClearShadow", &gorgen6_Device11_ClearShadowRequest{}, &reply) return err } -type deviceConfigureRequest struct { +type gorgen6_Device9_ConfigureRequest struct { A0 string } -type deviceConfigureReply struct{} +type gorgen6_Device9_ConfigureReply struct{} -func (p *deviceProxy) Configure(ctx context.Context, configuration string) error { - var reply deviceConfigureReply - err := p.rt.Invoke(ctx, p.id, "Configure", &deviceConfigureRequest{A0: configuration}, &reply) +func (p *gorgen6_DeviceProxy) Configure(ctx context.Context, arg0 string) error { + var reply gorgen6_Device9_ConfigureReply + err := p.rt.Invoke(ctx, p.id, "Configure", &gorgen6_Device9_ConfigureRequest{A0: arg0}, &reply) return err } -type deviceMarkOfflineRequest struct { +type gorgen6_Device11_MarkOfflineRequest struct { A0 gor.TickStatus } -type deviceMarkOfflineReply struct{} +type gorgen6_Device11_MarkOfflineReply struct{} -func (p *deviceProxy) MarkOffline(ctx context.Context, tick gor.TickStatus) error { - var reply deviceMarkOfflineReply - err := p.rt.Invoke(ctx, p.id, "MarkOffline", &deviceMarkOfflineRequest{A0: tick}, &reply) +func (p *gorgen6_DeviceProxy) MarkOffline(ctx context.Context, arg0 gor.TickStatus) error { + var reply gorgen6_Device11_MarkOfflineReply + err := p.rt.Invoke(ctx, p.id, "MarkOffline", &gorgen6_Device11_MarkOfflineRequest{A0: arg0}, &reply) return err } -type deviceReportRequest struct { +type gorgen6_Device6_ReportRequest struct { A0 string A1 string } -type deviceReportReply struct{} +type gorgen6_Device6_ReportReply struct{} -func (p *deviceProxy) Report(ctx context.Context, workshopID string, state string) error { - var reply deviceReportReply - err := p.rt.Invoke(ctx, p.id, "Report", &deviceReportRequest{A0: workshopID, A1: state}, &reply) +func (p *gorgen6_DeviceProxy) Report(ctx context.Context, arg0 string, arg1 string) error { + var reply gorgen6_Device6_ReportReply + err := p.rt.Invoke(ctx, p.id, "Report", &gorgen6_Device6_ReportRequest{A0: arg0, A1: arg1}, &reply) return err } -type deviceReportActionRequest struct { +type gorgen6_Device12_ReportActionRequest struct { A0 string A1 string } -type deviceReportActionReply struct{} +type gorgen6_Device12_ReportActionReply struct{} -func (p *deviceProxy) ReportAction(ctx context.Context, actionID string, state string) error { - var reply deviceReportActionReply - err := p.rt.Invoke(ctx, p.id, "ReportAction", &deviceReportActionRequest{A0: actionID, A1: state}, &reply) +func (p *gorgen6_DeviceProxy) ReportAction(ctx context.Context, arg0 string, arg1 string) error { + var reply gorgen6_Device12_ReportActionReply + err := p.rt.Invoke(ctx, p.id, "ReportAction", &gorgen6_Device12_ReportActionRequest{A0: arg0, A1: arg1}, &reply) return err } -type deviceShadowRequest struct{} -type deviceShadowReply struct { +type gorgen6_Device6_ShadowRequest struct{} +type gorgen6_Device6_ShadowReply struct { R0 domain.Shadow } -func (p *deviceProxy) Shadow(ctx context.Context) (domain.Shadow, error) { - var reply deviceShadowReply - err := p.rt.Invoke(ctx, p.id, "Shadow", &deviceShadowRequest{}, &reply) +func (p *gorgen6_DeviceProxy) Shadow(ctx context.Context) (domain.Shadow, error) { + var reply gorgen6_Device6_ShadowReply + err := p.rt.Invoke(ctx, p.id, "Shadow", &gorgen6_Device6_ShadowRequest{}, &reply) return reply.R0, err } -type deviceShadowExistsRequest struct{} -type deviceShadowExistsReply struct { +type gorgen6_Device12_ShadowExistsRequest struct{} +type gorgen6_Device12_ShadowExistsReply struct { R0 bool } -func (p *deviceProxy) ShadowExists(ctx context.Context) (bool, error) { - var reply deviceShadowExistsReply - err := p.rt.Invoke(ctx, p.id, "ShadowExists", &deviceShadowExistsRequest{}, &reply) +func (p *gorgen6_DeviceProxy) ShadowExists(ctx context.Context) (bool, error) { + var reply gorgen6_Device12_ShadowExistsReply + err := p.rt.Invoke(ctx, p.id, "ShadowExists", &gorgen6_Device12_ShadowExistsRequest{}, &reply) return reply.R0, err } -func dispatchDevice(ctx context.Context, instance domain.Device, method string, args any, reply any) error { +func gorgen6_DeviceDispatch(ctx context.Context, instance domain.Device, method string, args any, reply any) error { switch method { case "ApplyPending": - typedArgs := args.(*deviceApplyPendingRequest) + typedArgs := args.(*gorgen6_Device12_ApplyPendingRequest) err := instance.ApplyPending(ctx, typedArgs.A0) return err case "ClearShadow": err := instance.ClearShadow(ctx) return err case "Configure": - typedArgs := args.(*deviceConfigureRequest) + typedArgs := args.(*gorgen6_Device9_ConfigureRequest) err := instance.Configure(ctx, typedArgs.A0) return err case "MarkOffline": - typedArgs := args.(*deviceMarkOfflineRequest) + typedArgs := args.(*gorgen6_Device11_MarkOfflineRequest) err := instance.MarkOffline(ctx, typedArgs.A0) return err case "Report": - typedArgs := args.(*deviceReportRequest) + typedArgs := args.(*gorgen6_Device6_ReportRequest) err := instance.Report(ctx, typedArgs.A0, typedArgs.A1) return err case "ReportAction": - typedArgs := args.(*deviceReportActionRequest) + typedArgs := args.(*gorgen6_Device12_ReportActionRequest) err := instance.ReportAction(ctx, typedArgs.A0, typedArgs.A1) return err case "Shadow": - typedReply := reply.(*deviceShadowReply) + typedReply := reply.(*gorgen6_Device6_ShadowReply) r0, err := instance.Shadow(ctx) typedReply.R0 = r0 return err case "ShadowExists": - typedReply := reply.(*deviceShadowExistsReply) + typedReply := reply.(*gorgen6_Device12_ShadowExistsReply) r0, err := instance.ShadowExists(ctx) typedReply.R0 = r0 return err @@ -141,80 +147,82 @@ func dispatchDevice(ctx context.Context, instance domain.Device, method string, } } -func newDeviceCall(method string) (args any, reply any) { +func gorgen6_DeviceNewCall(method string) (args any, reply any) { switch method { case "ApplyPending": - return &deviceApplyPendingRequest{}, &deviceApplyPendingReply{} + return &gorgen6_Device12_ApplyPendingRequest{}, &gorgen6_Device12_ApplyPendingReply{} case "ClearShadow": - return &deviceClearShadowRequest{}, &deviceClearShadowReply{} + return &gorgen6_Device11_ClearShadowRequest{}, &gorgen6_Device11_ClearShadowReply{} case "Configure": - return &deviceConfigureRequest{}, &deviceConfigureReply{} + return &gorgen6_Device9_ConfigureRequest{}, &gorgen6_Device9_ConfigureReply{} case "MarkOffline": - return &deviceMarkOfflineRequest{}, &deviceMarkOfflineReply{} + return &gorgen6_Device11_MarkOfflineRequest{}, &gorgen6_Device11_MarkOfflineReply{} case "Report": - return &deviceReportRequest{}, &deviceReportReply{} + return &gorgen6_Device6_ReportRequest{}, &gorgen6_Device6_ReportReply{} case "ReportAction": - return &deviceReportActionRequest{}, &deviceReportActionReply{} + return &gorgen6_Device12_ReportActionRequest{}, &gorgen6_Device12_ReportActionReply{} case "Shadow": - return &deviceShadowRequest{}, &deviceShadowReply{} + return &gorgen6_Device6_ShadowRequest{}, &gorgen6_Device6_ShadowReply{} case "ShadowExists": - return &deviceShadowExistsRequest{}, &deviceShadowExistsReply{} + return &gorgen6_Device12_ShadowExistsRequest{}, &gorgen6_Device12_ShadowExistsReply{} default: return nil, nil } } -func newDeviceReminderCall(method string, status gor.TickStatus) (args any, reply any) { +func gorgen6_DeviceNewReminderCall(method string, status gor.TickStatus) (args any, reply any) { switch method { case "MarkOffline": - return &deviceMarkOfflineRequest{A0: status}, &deviceMarkOfflineReply{} + return &gorgen6_Device11_MarkOfflineRequest{A0: status}, &gorgen6_Device11_MarkOfflineReply{} default: return nil, nil } } -func newDeviceProxy(rt gor.Invoker, id gor.GrainId) domain.Device { - return &deviceProxy{id: id, rt: rt} +func gorgen6_DeviceNewProxy(rt gor.Invoker, id gor.GrainId) domain.Device { + return &gorgen6_DeviceProxy{id: id, rt: rt} } -type recoveryCoordinatorProxy struct { +const gorgen19_RecoveryCoordinatorGrainType gor.GrainType = "domain.RecoveryCoordinator" + +type gorgen19_RecoveryCoordinatorProxy struct { id gor.GrainId rt gor.Invoker } -type recoveryCoordinatorRecoverRequest struct { +type gorgen19_RecoveryCoordinator7_RecoverRequest struct { A0 gor.TickStatus } -type recoveryCoordinatorRecoverReply struct{} +type gorgen19_RecoveryCoordinator7_RecoverReply struct{} -func (p *recoveryCoordinatorProxy) Recover(ctx context.Context, tick gor.TickStatus) error { - var reply recoveryCoordinatorRecoverReply - err := p.rt.Invoke(ctx, p.id, "Recover", &recoveryCoordinatorRecoverRequest{A0: tick}, &reply) +func (p *gorgen19_RecoveryCoordinatorProxy) Recover(ctx context.Context, arg0 gor.TickStatus) error { + var reply gorgen19_RecoveryCoordinator7_RecoverReply + err := p.rt.Invoke(ctx, p.id, "Recover", &gorgen19_RecoveryCoordinator7_RecoverRequest{A0: arg0}, &reply) return err } -type recoveryCoordinatorStartRequest struct{} -type recoveryCoordinatorStartReply struct{} +type gorgen19_RecoveryCoordinator5_StartRequest struct{} +type gorgen19_RecoveryCoordinator5_StartReply struct{} -func (p *recoveryCoordinatorProxy) Start(ctx context.Context) error { - var reply recoveryCoordinatorStartReply - err := p.rt.Invoke(ctx, p.id, "Start", &recoveryCoordinatorStartRequest{}, &reply) +func (p *gorgen19_RecoveryCoordinatorProxy) Start(ctx context.Context) error { + var reply gorgen19_RecoveryCoordinator5_StartReply + err := p.rt.Invoke(ctx, p.id, "Start", &gorgen19_RecoveryCoordinator5_StartRequest{}, &reply) return err } -type recoveryCoordinatorStopRequest struct{} -type recoveryCoordinatorStopReply struct{} +type gorgen19_RecoveryCoordinator4_StopRequest struct{} +type gorgen19_RecoveryCoordinator4_StopReply struct{} -func (p *recoveryCoordinatorProxy) Stop(ctx context.Context) error { - var reply recoveryCoordinatorStopReply - err := p.rt.Invoke(ctx, p.id, "Stop", &recoveryCoordinatorStopRequest{}, &reply) +func (p *gorgen19_RecoveryCoordinatorProxy) Stop(ctx context.Context) error { + var reply gorgen19_RecoveryCoordinator4_StopReply + err := p.rt.Invoke(ctx, p.id, "Stop", &gorgen19_RecoveryCoordinator4_StopRequest{}, &reply) return err } -func dispatchRecoveryCoordinator(ctx context.Context, instance domain.RecoveryCoordinator, method string, args any, reply any) error { +func gorgen19_RecoveryCoordinatorDispatch(ctx context.Context, instance domain.RecoveryCoordinator, method string, args any, reply any) error { switch method { case "Recover": - typedArgs := args.(*recoveryCoordinatorRecoverRequest) + typedArgs := args.(*gorgen19_RecoveryCoordinator7_RecoverRequest) err := instance.Recover(ctx, typedArgs.A0) return err case "Start": @@ -228,82 +236,84 @@ func dispatchRecoveryCoordinator(ctx context.Context, instance domain.RecoveryCo } } -func newRecoveryCoordinatorCall(method string) (args any, reply any) { +func gorgen19_RecoveryCoordinatorNewCall(method string) (args any, reply any) { switch method { case "Recover": - return &recoveryCoordinatorRecoverRequest{}, &recoveryCoordinatorRecoverReply{} + return &gorgen19_RecoveryCoordinator7_RecoverRequest{}, &gorgen19_RecoveryCoordinator7_RecoverReply{} case "Start": - return &recoveryCoordinatorStartRequest{}, &recoveryCoordinatorStartReply{} + return &gorgen19_RecoveryCoordinator5_StartRequest{}, &gorgen19_RecoveryCoordinator5_StartReply{} case "Stop": - return &recoveryCoordinatorStopRequest{}, &recoveryCoordinatorStopReply{} + return &gorgen19_RecoveryCoordinator4_StopRequest{}, &gorgen19_RecoveryCoordinator4_StopReply{} default: return nil, nil } } -func newRecoveryCoordinatorReminderCall(method string, status gor.TickStatus) (args any, reply any) { +func gorgen19_RecoveryCoordinatorNewReminderCall(method string, status gor.TickStatus) (args any, reply any) { switch method { case "Recover": - return &recoveryCoordinatorRecoverRequest{A0: status}, &recoveryCoordinatorRecoverReply{} + return &gorgen19_RecoveryCoordinator7_RecoverRequest{A0: status}, &gorgen19_RecoveryCoordinator7_RecoverReply{} default: return nil, nil } } -func newRecoveryCoordinatorProxy(rt gor.Invoker, id gor.GrainId) domain.RecoveryCoordinator { - return &recoveryCoordinatorProxy{id: id, rt: rt} +func gorgen19_RecoveryCoordinatorNewProxy(rt gor.Invoker, id gor.GrainId) domain.RecoveryCoordinator { + return &gorgen19_RecoveryCoordinatorProxy{id: id, rt: rt} } -type workshopProxy struct { +const gorgen8_WorkshopGrainType gor.GrainType = "domain.Workshop" + +type gorgen8_WorkshopProxy struct { id gor.GrainId rt gor.Invoker } -type workshopDeviceOfflineRequest struct { +type gorgen8_Workshop13_DeviceOfflineRequest struct { A0 string } -type workshopDeviceOfflineReply struct{} +type gorgen8_Workshop13_DeviceOfflineReply struct{} -func (p *workshopProxy) DeviceOffline(ctx context.Context, deviceID string) error { - var reply workshopDeviceOfflineReply - err := p.rt.Invoke(ctx, p.id, "DeviceOffline", &workshopDeviceOfflineRequest{A0: deviceID}, &reply) +func (p *gorgen8_WorkshopProxy) DeviceOffline(ctx context.Context, arg0 string) error { + var reply gorgen8_Workshop13_DeviceOfflineReply + err := p.rt.Invoke(ctx, p.id, "DeviceOffline", &gorgen8_Workshop13_DeviceOfflineRequest{A0: arg0}, &reply) return err } -type workshopDeviceOnlineRequest struct { +type gorgen8_Workshop12_DeviceOnlineRequest struct { A0 string } -type workshopDeviceOnlineReply struct{} +type gorgen8_Workshop12_DeviceOnlineReply struct{} -func (p *workshopProxy) DeviceOnline(ctx context.Context, deviceID string) error { - var reply workshopDeviceOnlineReply - err := p.rt.Invoke(ctx, p.id, "DeviceOnline", &workshopDeviceOnlineRequest{A0: deviceID}, &reply) +func (p *gorgen8_WorkshopProxy) DeviceOnline(ctx context.Context, arg0 string) error { + var reply gorgen8_Workshop12_DeviceOnlineReply + err := p.rt.Invoke(ctx, p.id, "DeviceOnline", &gorgen8_Workshop12_DeviceOnlineRequest{A0: arg0}, &reply) return err } -type workshopOnlineCountRequest struct{} -type workshopOnlineCountReply struct { +type gorgen8_Workshop11_OnlineCountRequest struct{} +type gorgen8_Workshop11_OnlineCountReply struct { R0 int } -func (p *workshopProxy) OnlineCount(ctx context.Context) (int, error) { - var reply workshopOnlineCountReply - err := p.rt.Invoke(ctx, p.id, "OnlineCount", &workshopOnlineCountRequest{}, &reply) +func (p *gorgen8_WorkshopProxy) OnlineCount(ctx context.Context) (int, error) { + var reply gorgen8_Workshop11_OnlineCountReply + err := p.rt.Invoke(ctx, p.id, "OnlineCount", &gorgen8_Workshop11_OnlineCountRequest{}, &reply) return reply.R0, err } -func dispatchWorkshop(ctx context.Context, instance domain.Workshop, method string, args any, reply any) error { +func gorgen8_WorkshopDispatch(ctx context.Context, instance domain.Workshop, method string, args any, reply any) error { switch method { case "DeviceOffline": - typedArgs := args.(*workshopDeviceOfflineRequest) + typedArgs := args.(*gorgen8_Workshop13_DeviceOfflineRequest) err := instance.DeviceOffline(ctx, typedArgs.A0) return err case "DeviceOnline": - typedArgs := args.(*workshopDeviceOnlineRequest) + typedArgs := args.(*gorgen8_Workshop12_DeviceOnlineRequest) err := instance.DeviceOnline(ctx, typedArgs.A0) return err case "OnlineCount": - typedReply := reply.(*workshopOnlineCountReply) + typedReply := reply.(*gorgen8_Workshop11_OnlineCountReply) r0, err := instance.OnlineCount(ctx) typedReply.R0 = r0 return err @@ -312,28 +322,43 @@ func dispatchWorkshop(ctx context.Context, instance domain.Workshop, method stri } } -func newWorkshopCall(method string) (args any, reply any) { +func gorgen8_WorkshopNewCall(method string) (args any, reply any) { switch method { case "DeviceOffline": - return &workshopDeviceOfflineRequest{}, &workshopDeviceOfflineReply{} + return &gorgen8_Workshop13_DeviceOfflineRequest{}, &gorgen8_Workshop13_DeviceOfflineReply{} case "DeviceOnline": - return &workshopDeviceOnlineRequest{}, &workshopDeviceOnlineReply{} + return &gorgen8_Workshop12_DeviceOnlineRequest{}, &gorgen8_Workshop12_DeviceOnlineReply{} case "OnlineCount": - return &workshopOnlineCountRequest{}, &workshopOnlineCountReply{} + return &gorgen8_Workshop11_OnlineCountRequest{}, &gorgen8_Workshop11_OnlineCountReply{} default: return nil, nil } } -func newWorkshopReminderCall(method string, status gor.TickStatus) (args any, reply any) { +func gorgen8_WorkshopNewReminderCall(method string, status gor.TickStatus) (args any, reply any) { switch method { default: return nil, nil } } -func newWorkshopProxy(rt gor.Invoker, id gor.GrainId) domain.Workshop { - return &workshopProxy{id: id, rt: rt} +func gorgen8_WorkshopNewProxy(rt gor.Invoker, id gor.GrainId) domain.Workshop { + return &gorgen8_WorkshopProxy{id: id, rt: rt} +} + +// InstallDevice installs the generated bindings for Device in rt. +func InstallDevice(rt *gor.Runtime) error { + return gor.InstallType[domain.Device](rt, generatedCodeVersion, gorgen6_DeviceGrainType, gorgen6_DeviceDispatch, gorgen6_DeviceNewProxy, gorgen6_DeviceNewCall, gorgen6_DeviceNewReminderCall) +} + +// InstallRecoveryCoordinator installs the generated bindings for RecoveryCoordinator in rt. +func InstallRecoveryCoordinator(rt *gor.Runtime) error { + return gor.InstallType[domain.RecoveryCoordinator](rt, generatedCodeVersion, gorgen19_RecoveryCoordinatorGrainType, gorgen19_RecoveryCoordinatorDispatch, gorgen19_RecoveryCoordinatorNewProxy, gorgen19_RecoveryCoordinatorNewCall, gorgen19_RecoveryCoordinatorNewReminderCall) +} + +// InstallWorkshop installs the generated bindings for Workshop in rt. +func InstallWorkshop(rt *gor.Runtime) error { + return gor.InstallType[domain.Workshop](rt, generatedCodeVersion, gorgen8_WorkshopGrainType, gorgen8_WorkshopDispatch, gorgen8_WorkshopNewProxy, gorgen8_WorkshopNewCall, gorgen8_WorkshopNewReminderCall) } // Install installs the generated Grain bindings in rt. @@ -341,13 +366,13 @@ func newWorkshopProxy(rt gor.Invoker, id gor.GrainId) domain.Workshop { // the generated Grain types. After it returns nil, gor.Register and gor.Ref // can use those types with rt. func Install(rt *gor.Runtime) error { - if err := gor.InstallType[domain.Device](rt, dispatchDevice, newDeviceProxy, newDeviceCall, newDeviceReminderCall); err != nil { + if err := InstallDevice(rt); err != nil { return err } - if err := gor.InstallType[domain.RecoveryCoordinator](rt, dispatchRecoveryCoordinator, newRecoveryCoordinatorProxy, newRecoveryCoordinatorCall, newRecoveryCoordinatorReminderCall); err != nil { + if err := InstallRecoveryCoordinator(rt); err != nil { return err } - if err := gor.InstallType[domain.Workshop](rt, dispatchWorkshop, newWorkshopProxy, newWorkshopCall, newWorkshopReminderCall); err != nil { + if err := InstallWorkshop(rt); err != nil { return err } return nil diff --git a/examples/shadow/domain/recovery.go b/examples/shadow/domain/recovery.go index e5161d1..78ca33c 100644 --- a/examples/shadow/domain/recovery.go +++ b/examples/shadow/domain/recovery.go @@ -3,36 +3,52 @@ package domain import ( "context" "errors" + "time" "github.com/suraciii/gor" ) type recoveryCoordinator struct { - binder *gor.Binder - status gor.State[CoordinatorState] - reminder gor.Reminder[RecoveryCoordinator] - application ApplicationStore - observed chan<- RecoveryObservation + grainContext *gor.GrainContext + status gor.State[CoordinatorState] + reminder gor.Reminder[RecoveryCoordinator] + application ApplicationStore + observed chan<- RecoveryObservation } // NewRecoveryCoordinator creates the fixed-key recovery Grain. -func NewRecoveryCoordinator(b *gor.Binder, application ApplicationStore) RecoveryCoordinator { +func NewRecoveryCoordinator(b *gor.GrainContext, application ApplicationStore) RecoveryCoordinator { return newRecoveryCoordinator(b, application, nil) } // NewRecoveryCoordinatorWithObservation creates the recovery Grain and sends // each Recover context and tick to observed. The channel is for example tests. -func NewRecoveryCoordinatorWithObservation(b *gor.Binder, application ApplicationStore, observed chan<- RecoveryObservation) RecoveryCoordinator { +func NewRecoveryCoordinatorWithObservation(b *gor.GrainContext, application ApplicationStore, observed chan<- RecoveryObservation) RecoveryCoordinator { return newRecoveryCoordinator(b, application, observed) } -func newRecoveryCoordinator(b *gor.Binder, application ApplicationStore, observed chan<- RecoveryObservation) RecoveryCoordinator { +// NewRecoveryCoordinatorWithGrainTimerObservation creates the recovery Grain +// and reports each Activation-local Grain Timer callback. It supports the +// public conformance test. +func NewRecoveryCoordinatorWithGrainTimerObservation(b *gor.GrainContext, application ApplicationStore, observed chan<- time.Time) RecoveryCoordinator { + coordinator := newRecoveryCoordinator(b, application, nil) + _, err := gor.RegisterGrainTimer(b, func(context.Context) error { + observed <- gor.Now(b) + return nil + }, gor.GrainTimerOptions{DueTime: RecoveryInterval, Period: RecoveryInterval}) + if err != nil { + panic(err) + } + return coordinator +} + +func newRecoveryCoordinator(b *gor.GrainContext, application ApplicationStore, observed chan<- RecoveryObservation) RecoveryCoordinator { return &recoveryCoordinator{ - binder: b, - status: gor.NewState[CoordinatorState](b, "running"), - reminder: gor.NewReminder[RecoveryCoordinator](b), - application: application, - observed: observed, + grainContext: b, + status: gor.NewState[CoordinatorState](b, "running"), + reminder: gor.NewReminder[RecoveryCoordinator](b), + application: application, + observed: observed, } } @@ -52,7 +68,11 @@ func (c *recoveryCoordinator) Stop(ctx context.Context) error { if err := c.status.Clear(ctx); err != nil { return err } - return c.reminder.Cancel(ctx, RecoveryReminderName) + if err := c.reminder.Cancel(ctx, RecoveryReminderName); err != nil { + return err + } + gor.DeactivateOnIdle(c.grainContext) + return nil } func (c *recoveryCoordinator) Recover(ctx context.Context, tick gor.TickStatus) error { @@ -68,7 +88,7 @@ func (c *recoveryCoordinator) Recover(ctx context.Context, tick gor.TickStatus) return err } for _, action := range actions { - if err := gor.Ref[Device](c.binder, action.DeviceKey).ApplyPending(ctx, action.ActionID); err != nil { + if err := gor.Ref[Device](c.grainContext, action.DeviceKey).ApplyPending(ctx, action.ActionID); err != nil { return err } } diff --git a/examples/shadow/http_test.go b/examples/shadow/http_test.go index 868522f..f829940 100644 --- a/examples/shadow/http_test.go +++ b/examples/shadow/http_test.go @@ -16,8 +16,10 @@ import ( func TestHTTPReportsConfiguresAndReadsShadow(t *testing.T) { sourceClock := clock.NewFake(time.Unix(0, 0).UTC()) + backend := store.NewMemory() rt, err := gor.New( - gor.WithStore(store.NewMemory()), + gor.WithStore(backend), + gor.WithReminderStore(backend), gor.WithClock(sourceClock), gor.WithIdleTimeout(0), gor.WithEvictionInterval(0), @@ -27,10 +29,11 @@ func TestHTTPReportsConfiguresAndReadsShadow(t *testing.T) { t.Fatal(err) } if err := shadow.Register(rt); err != nil { - rt.Close() + _ = crashRuntime(rt) t.Fatal(err) } - defer rt.Close() + startRuntime(t, rt) + defer shutdownRuntime(rt) handler := shadow.NewHandler(rt) report := serve(handler, http.MethodPost, "/devices/device-1/reports", `{"workshop_id":"assembly","state":"temperature=20"}`) @@ -78,15 +81,17 @@ func TestHTTPReportsConfiguresAndReadsShadow(t *testing.T) { func TestHTTPRejectsMalformedJSON(t *testing.T) { sourceClock := clock.NewFake(time.Unix(0, 0).UTC()) - rt, err := gor.New(gor.WithStore(store.NewMemory()), gor.WithClock(sourceClock), gor.WithReminderInterval(time.Second), gor.WithIdleTimeout(0), gor.WithEvictionInterval(0)) + backend := store.NewMemory() + rt, err := gor.New(gor.WithStore(backend), gor.WithReminderStore(backend), gor.WithClock(sourceClock), gor.WithReminderInterval(time.Second), gor.WithIdleTimeout(0), gor.WithEvictionInterval(0)) if err != nil { t.Fatal(err) } if err := shadow.Register(rt); err != nil { - rt.Close() + _ = crashRuntime(rt) t.Fatal(err) } - defer rt.Close() + startRuntime(t, rt) + defer shutdownRuntime(rt) response := serve(shadow.NewHandler(rt), http.MethodPost, "/devices/device-1/reports", `{not-json}`) if response.Code != http.StatusBadRequest { @@ -104,6 +109,7 @@ func TestHTTPReportWriteFailureIsNotBadRequest(t *testing.T) { sourceClock := clock.NewFake(time.Unix(0, 0).UTC()) rt, err := gor.New( gor.WithStore(backend), + gor.WithReminderStore(backend), gor.WithClock(sourceClock), gor.WithReminderInterval(time.Second), gor.WithIdleTimeout(0), @@ -113,10 +119,11 @@ func TestHTTPReportWriteFailureIsNotBadRequest(t *testing.T) { t.Fatal(err) } if err := shadow.Register(rt); err != nil { - rt.Close() + _ = crashRuntime(rt) t.Fatal(err) } - defer rt.Close() + startRuntime(t, rt) + defer shutdownRuntime(rt) backend.failWorkshopWrites.Store(true) response := serve(shadow.NewHandler(rt), http.MethodPost, "/devices/device-1/reports", `{"workshop_id":"assembly","state":"temperature=20"}`) diff --git a/examples/shadow/runtime.go b/examples/shadow/runtime.go index 7905c61..17c7c09 100644 --- a/examples/shadow/runtime.go +++ b/examples/shadow/runtime.go @@ -2,6 +2,7 @@ package shadow import ( "log" + "time" "github.com/suraciii/gor" "github.com/suraciii/gor/examples/shadow/domain" @@ -12,6 +13,16 @@ func LogBackgroundError(event gor.BackgroundError) { switch source := event.Source.(type) { case gor.ReminderInvocation: log.Printf("%s/%s.%s failed: %v", event.GrainId.GrainType, event.GrainId.GrainKey, source.Method, event.Err) + case gor.ReminderScan: + log.Printf("Reminder scan failed: %v", event.Err) + case gor.ReminderClaim: + log.Printf("%s/%s Reminder %q claim failed: %v", event.GrainId.GrainType, event.GrainId.GrainKey, source.Name, event.Err) + case gor.ReminderDispatch: + log.Printf("%s/%s.%s failed: %v", event.GrainId.GrainType, event.GrainId.GrainKey, source.Method, event.Err) + case gor.ReminderTerminal: + log.Printf("%s/%s Reminder %q method %s Terminal Result failed: %v", event.GrainId.GrainType, event.GrainId.GrainKey, source.Name, source.Method, event.Err) + case gor.GrainTimerInvocation: + log.Printf("%s/%s Grain Timer failed: %v", event.GrainId.GrainType, event.GrainId.GrainKey, event.Err) case gor.Deactivation: log.Printf("%s/%s deactivation (%v) failed: %v", event.GrainId.GrainType, event.GrainId.GrainKey, source.Reason, event.Err) } @@ -23,10 +34,10 @@ func Register(rt *gor.Runtime) error { func RegisterWithLifecycle(rt *gor.Runtime, events chan<- domain.LifecycleEvent) error { return register(rt, - func(b *gor.Binder) domain.Device { + func(b *gor.GrainContext) domain.Device { return domain.NewDeviceWithLifecycle(b, events) }, - func(b *gor.Binder) domain.Workshop { + func(b *gor.GrainContext) domain.Workshop { return domain.NewWorkshopWithLifecycle(b, events) }, ) @@ -44,16 +55,38 @@ func RegisterConformanceWithObservation(rt *gor.Runtime, application domain.Appl return registerConformance(rt, application, observed) } +// RegisterConformanceWithGrainTimerObservation installs the conformance +// Application and reports Activation-local Grain Timer callbacks. +func RegisterConformanceWithGrainTimerObservation(rt *gor.Runtime, application domain.ApplicationStore, observed chan<- time.Time) error { + if err := register(rt, + func(b *gor.GrainContext) domain.Device { + return domain.NewDeviceWithApplication(b, application) + }, + domain.NewWorkshop, + ); err != nil { + return err + } + if err := gorgen.InstallRecoveryCoordinator(rt); err != nil { + return err + } + return gor.Register[domain.RecoveryCoordinator](rt, func(b *gor.GrainContext) domain.RecoveryCoordinator { + return domain.NewRecoveryCoordinatorWithGrainTimerObservation(b, application, observed) + }) +} + func registerConformance(rt *gor.Runtime, application domain.ApplicationStore, observed chan<- domain.RecoveryObservation) error { if err := register(rt, - func(b *gor.Binder) domain.Device { + func(b *gor.GrainContext) domain.Device { return domain.NewDeviceWithApplication(b, application) }, domain.NewWorkshop, ); err != nil { return err } - return gor.Register[domain.RecoveryCoordinator](rt, func(b *gor.Binder) domain.RecoveryCoordinator { + if err := gorgen.InstallRecoveryCoordinator(rt); err != nil { + return err + } + return gor.Register[domain.RecoveryCoordinator](rt, func(b *gor.GrainContext) domain.RecoveryCoordinator { if observed == nil { return domain.NewRecoveryCoordinator(b, application) } @@ -61,8 +94,11 @@ func registerConformance(rt *gor.Runtime, application domain.ApplicationStore, o }) } -func register(rt *gor.Runtime, deviceFactory func(*gor.Binder) domain.Device, workshopFactory func(*gor.Binder) domain.Workshop) error { - if err := gorgen.Install(rt); err != nil { +func register(rt *gor.Runtime, deviceFactory func(*gor.GrainContext) domain.Device, workshopFactory func(*gor.GrainContext) domain.Workshop) error { + if err := gorgen.InstallDevice(rt); err != nil { + return err + } + if err := gorgen.InstallWorkshop(rt); err != nil { return err } if err := gor.Register[domain.Device](rt, deviceFactory); err != nil { diff --git a/examples/shadow/shadow_test.go b/examples/shadow/shadow_test.go index fc1ea28..651a180 100644 --- a/examples/shadow/shadow_test.go +++ b/examples/shadow/shadow_test.go @@ -23,6 +23,7 @@ func TestDeviceShadowTracksReportsAndWorkshopPresence(t *testing.T) { backend := store.NewMemory() rt, err := gor.New( gor.WithStore(backend), + gor.WithReminderStore(backend), gor.WithClock(sourceClock), gor.WithIdleTimeout(0), gor.WithEvictionInterval(0), @@ -32,10 +33,11 @@ func TestDeviceShadowTracksReportsAndWorkshopPresence(t *testing.T) { t.Fatal(err) } if err := shadow.Register(rt); err != nil { - rt.Close() + _ = crashRuntime(rt) t.Fatal(err) } - defer rt.Close() + startRuntime(t, rt) + defer shutdownRuntime(rt) device := gor.Ref[domain.Device](rt, "device-1") workshop := gor.Ref[domain.Workshop](rt, "assembly") @@ -71,10 +73,7 @@ func TestDeviceShadowTracksReportsAndWorkshopPresence(t *testing.T) { if err := device.Report(ctx, "assembly", "temperature=21"); err != nil { t.Fatal(err) } - reminders, err := backend.ListDue(ctx, start.Add(59*time.Second)) - if err != nil { - t.Fatal(err) - } + reminders := listDueReminders(t, backend, start.Add(59*time.Second)) if len(reminders) != 1 || !reminders[0].DueAt.Equal(start.Add(59*time.Second)) { t.Fatalf("offline schedule after second report = %#v, want one schedule due at 59s", reminders) } @@ -93,10 +92,7 @@ func TestDeviceShadowTracksReportsAndWorkshopPresence(t *testing.T) { if got, err := workshop.OnlineCount(ctx); err != nil || got != 0 { t.Fatalf("online count after timeout = (%d, %v), want (0, nil)", got, err) } - reminders, err = backend.ListDue(ctx, sourceClock.Now().Add(domain.OfflineAfter)) - if err != nil { - t.Fatal(err) - } + reminders = listDueReminders(t, backend, sourceClock.Now().Add(domain.OfflineAfter)) if len(reminders) != 0 { t.Fatalf("reminders after timeout = %#v, want none", reminders) } @@ -114,8 +110,10 @@ func TestDeviceIdleEvictionRunsLifecycleAndReloadsState(t *testing.T) { synctest.Test(t, func(t *testing.T) { sourceClock := clock.NewFake(time.Unix(0, 0).UTC()) events := make(chan domain.LifecycleEvent, 8) + backend := store.NewMemory() rt, err := gor.New( - gor.WithStore(store.NewMemory()), + gor.WithStore(backend), + gor.WithReminderStore(backend), gor.WithClock(sourceClock), gor.WithIdleTimeout(2*time.Second), gor.WithEvictionInterval(time.Second), @@ -125,10 +123,11 @@ func TestDeviceIdleEvictionRunsLifecycleAndReloadsState(t *testing.T) { t.Fatal(err) } if err := shadow.RegisterWithLifecycle(rt, events); err != nil { - rt.Close() + _ = crashRuntime(rt) t.Fatal(err) } - defer rt.Close() + startRuntime(t, rt) + defer shutdownRuntime(rt) device := gor.Ref[domain.Device](rt, "device-1") if err := device.Report(context.Background(), "assembly", "temperature=20"); err != nil { @@ -137,11 +136,11 @@ func TestDeviceIdleEvictionRunsLifecycleAndReloadsState(t *testing.T) { if err := device.Configure(context.Background(), "sample-rate=10s"); err != nil { t.Fatal(err) } - expectLifecycleEvent(t, events, gor.GrainId{GrainType: gor.TypeName[domain.Device](), GrainKey: "device-1"}, domain.LifecycleActivated) + expectLifecycleEvent(t, events, gor.GrainId{GrainType: gor.GrainType("domain.Device"), GrainKey: "device-1"}, domain.LifecycleActivated) sourceClock.Advance(3 * time.Second) synctest.Wait() - expectLifecycleEvent(t, events, gor.GrainId{GrainType: gor.TypeName[domain.Device](), GrainKey: "device-1"}, domain.LifecycleDeactivated) + expectLifecycleEvent(t, events, gor.GrainId{GrainType: gor.GrainType("domain.Device"), GrainKey: "device-1"}, domain.LifecycleDeactivated) value, err := device.Shadow(context.Background()) if err != nil { @@ -150,7 +149,7 @@ func TestDeviceIdleEvictionRunsLifecycleAndReloadsState(t *testing.T) { if value.Configuration != "sample-rate=10s" { t.Fatalf("shadow after reactivation = %#v, want configuration restored from store", value) } - expectLifecycleEvent(t, events, gor.GrainId{GrainType: gor.TypeName[domain.Device](), GrainKey: "device-1"}, domain.LifecycleActivated) + expectLifecycleEvent(t, events, gor.GrainId{GrainType: gor.GrainType("domain.Device"), GrainKey: "device-1"}, domain.LifecycleActivated) }) } @@ -173,7 +172,8 @@ func expectLifecycleEvent(t *testing.T, events <-chan domain.LifecycleEvent, id func TestScheduledFailureReachesOnError(t *testing.T) { synctest.Test(t, func(t *testing.T) { backend := &failingWorkshopStore{Memory: store.NewMemory()} - sourceClock := clock.NewFake(time.Unix(0, 0).UTC()) + start := time.Unix(0, 0).UTC() + sourceClock := clock.NewFake(start) errorsSeen := make(chan gor.BackgroundError, 1) rt, err := gor.New( gor.WithStore(backend), @@ -190,10 +190,11 @@ func TestScheduledFailureReachesOnError(t *testing.T) { t.Fatal(err) } if err := shadow.Register(rt); err != nil { - rt.Close() + _ = crashRuntime(rt) t.Fatal(err) } - defer rt.Close() + startRuntime(t, rt) + defer shutdownRuntime(rt) if err := gor.Ref[domain.Device](rt, "device-1").Report(context.Background(), "assembly", "temperature=20"); err != nil { t.Fatal(err) @@ -204,9 +205,14 @@ func TestScheduledFailureReachesOnError(t *testing.T) { select { case got := <-errorsSeen: - wantID := gor.GrainId{GrainType: gor.TypeName[domain.Device](), GrainKey: "device-1"} + wantID := gor.GrainId{GrainType: gor.GrainType("domain.Device"), GrainKey: "device-1"} source, ok := got.Source.(gor.ReminderInvocation) - if !ok || got.GrainId != wantID || source.Method != "MarkOffline" || !errors.Is(got.Err, errWorkshopWrite) { + wantTick := gor.TickStatus{ + ReminderName: "offline", + FirstTickTime: start.Add(domain.OfflineAfter), + CurrentTickTime: start.Add(domain.OfflineAfter), + } + if !ok || got.GrainId != wantID || source.Name != "offline" || source.Method != "MarkOffline" || source.TickStatus != wantTick || !errors.Is(got.Err, errWorkshopWrite) { t.Fatalf("OnError event = %#v, want %v.MarkOffline with %v", got, wantID, errWorkshopWrite) } default: @@ -223,7 +229,7 @@ type failingWorkshopStore struct { } func (s *failingWorkshopStore) Write(ctx context.Context, id store.GrainId, data []byte, expect store.ETag) (store.ETag, error) { - if s.failWorkshopWrites.Load() && id.GrainType == gor.TypeName[domain.Workshop]() { + if s.failWorkshopWrites.Load() && id.GrainType == "domain.Workshop" { return 0, errWorkshopWrite } return s.Memory.Write(ctx, id, data, expect) diff --git a/forward.go b/forward.go index f36bcd1..878c4e7 100644 --- a/forward.go +++ b/forward.go @@ -7,22 +7,22 @@ import ( "fmt" "github.com/suraciii/gor/cluster" - runtimepkg "github.com/suraciii/gor/runtime" + runtimepkg "github.com/suraciii/gor/internal/runtime" ) type callRequest struct { - Kind string `json:"kind"` - GrainType string `json:"type"` - GrainKey string `json:"key"` - Method string `json:"method"` - Args json.RawMessage `json:"args"` - Occupied []occupiedIdentity `json:"occupied,omitempty"` - RequestContext json.RawMessage `json:"request_context,omitempty"` + Kind string `json:"kind"` + GrainType string `json:"type"` + GrainKey string `json:"key"` + Method string `json:"method"` + Args json.RawMessage `json:"args"` + Occupied []occupiedIdgrain `json:"occupied,omitempty"` + RequestContext json.RawMessage `json:"request_context,omitempty"` } -// occupiedIdentity is the wire form of one entity on a forwarded call's +// occupiedIdgrain is the wire form of one Grain on a forwarded Call's // occupied chain, shaped like the request's own type/key fields. -type occupiedIdentity struct { +type occupiedIdgrain struct { GrainType string `json:"type"` GrainKey string `json:"key"` } @@ -91,7 +91,7 @@ func (rt *Runtime) forward(ctx context.Context, owner string, id GrainId, method } payload, err := json.Marshal(callRequest{ Kind: requestKindInvoke, - GrainType: id.GrainType, + GrainType: string(id.GrainType), GrainKey: id.GrainKey, Method: method, Args: encodedArgs, @@ -161,7 +161,7 @@ func (rt *Runtime) handleInvoke(ctx context.Context, request callRequest) ([]byt } ctx = withRequestContextSnapshot(ctx, snapshot) - registration, ok := rt.typeRegistration(request.GrainType) + registration, ok := rt.typeRegistrationByGrainType(GrainType(request.GrainType)) if !ok { return errorResponse(fmt.Errorf("%w: %s", ErrTypeNotInstalled, request.GrainType)) } @@ -187,7 +187,7 @@ func (rt *Runtime) handleInvoke(ctx context.Context, request callRequest) ([]byt // after a stop transition is a runtime outcome rather than caller // cancellation. select { - case <-rt.done: + case <-rt.stopping: if errors.Is(invokeErr, context.Canceled) { invokeErr = stopRejection(rt.stopCodeSnapshot()) } @@ -206,7 +206,7 @@ func (rt *Runtime) handleProbe() ([]byte, error) { if rt.clusterNode == nil { return errorResponse(withCode(ErrInvalidRequest, errors.New("cluster node is not configured"))) } - // A probe is not an entity call and does not register against the admission + // A probe is not a Grain Call and does not register against the admission // count, but it reads the same root state and refuses to reply once the // runtime has left running. rt.lifecycleMu.Lock() @@ -235,18 +235,18 @@ func encodeCallResponse(response callResponse) ([]byte, error) { return encoded, nil } -func occupiedToWire(chain []runtimepkg.GrainId) []occupiedIdentity { +func occupiedToWire(chain []runtimepkg.GrainId) []occupiedIdgrain { if len(chain) == 0 { return nil } - wire := make([]occupiedIdentity, 0, len(chain)) + wire := make([]occupiedIdgrain, 0, len(chain)) for _, id := range chain { - wire = append(wire, occupiedIdentity{GrainType: id.GrainType, GrainKey: id.GrainKey}) + wire = append(wire, occupiedIdgrain{GrainType: id.GrainType, GrainKey: id.GrainKey}) } return wire } -func occupiedFromWire(wire []occupiedIdentity) []runtimepkg.GrainId { +func occupiedFromWire(wire []occupiedIdgrain) []runtimepkg.GrainId { if len(wire) == 0 { return nil } diff --git a/forward_teardown_test.go b/forward_teardown_test.go index 506df3c..2816899 100644 --- a/forward_teardown_test.go +++ b/forward_teardown_test.go @@ -14,60 +14,60 @@ import ( "github.com/suraciii/gor/store" ) -// blockingEntity is a scenario entity whose only method blocks until released. +// blockingGrain is a scenario grain whose only method blocks until released. // It records every entry so a test can tell whether a method body ran. Its // request and reply are empty, so it carries no state and no observable result // beyond the side effect of having run. -type blockingEntity interface { +type blockingGrain interface { Hold(context.Context) error } type blockingHoldRequest struct{} type blockingHoldReply struct{} -type blockingEntityProxy struct { +type blockingGrainProxy struct { invoker Invoker id GrainId } -func (p *blockingEntityProxy) Hold(ctx context.Context) error { +func (p *blockingGrainProxy) Hold(ctx context.Context) error { return p.invoker.Invoke(ctx, p.id, "Hold", &blockingHoldRequest{}, &blockingHoldReply{}) } -type blockingEntityImpl struct { +type blockingGrainImpl struct { entries chan struct{} release chan struct{} } -func (e *blockingEntityImpl) Hold(context.Context) error { +func (e *blockingGrainImpl) Hold(context.Context) error { e.entries <- struct{}{} <-e.release return nil } -func dispatchBlockingEntity(ctx context.Context, instance blockingEntity, method string, _ any, _ any) error { +func dispatchBlockingGrain(ctx context.Context, instance blockingGrain, method string, _ any, _ any) error { if method != "Hold" { return fmt.Errorf("unknown method %q", method) } return instance.Hold(ctx) } -func newBlockingEntityCall(method string) (args any, reply any) { +func newBlockingGrainCall(method string) (args any, reply any) { if method != "Hold" { return nil, nil } return &blockingHoldRequest{}, &blockingHoldReply{} } -func installBlockingEntity(t *testing.T, rt *Runtime, entries, release chan struct{}) { +func installBlockingGrain(t *testing.T, rt *Runtime, entries, release chan struct{}) { t.Helper() - if err := InstallType[blockingEntity](rt, dispatchBlockingEntity, func(invoker Invoker, id GrainId) blockingEntity { - return &blockingEntityProxy{invoker: invoker, id: id} - }, newBlockingEntityCall, nil); err != nil { + if err := InstallType[blockingGrain](rt, GeneratedCodeVersion, "gor.blockingGrain", dispatchBlockingGrain, func(invoker Invoker, id GrainId) blockingGrain { + return &blockingGrainProxy{invoker: invoker, id: id} + }, newBlockingGrainCall, noReminderCall); err != nil { t.Fatal(err) } - if err := Register[blockingEntity](rt, func(b *Binder) blockingEntity { - return &blockingEntityImpl{entries: entries, release: release} + if err := Register[blockingGrain](rt, func(b *GrainContext) blockingGrain { + return &blockingGrainImpl{entries: entries, release: release} }); err != nil { t.Fatal(err) } @@ -96,11 +96,13 @@ func TestScenario_ForwardedCallSurvivesOwnerClose(t *testing.T) { // failure. Release the blocked method first so both runtimes can // drain, then stop them. releaseOnce.Do(func() { close(release) }) - source.Close() - target.Close() + closeRuntime(source) + closeRuntime(target) }) - installBlockingEntity(t, source, entries, release) - installBlockingEntity(t, target, entries, release) + installBlockingGrain(t, source, entries, release) + installBlockingGrain(t, target, entries, release) + mustStart(t, source) + mustStart(t, target) synctest.Wait() fakeClock.Advance(time.Second) synctest.Wait() @@ -109,8 +111,8 @@ func TestScenario_ForwardedCallSurvivesOwnerClose(t *testing.T) { var owned cluster.View var id GrainId for index := 0; index < 4096; index++ { - candidate := GrainId{GrainType: TypeName[blockingEntity](), GrainKey: strconv.Itoa(index)} - owner, ok := cluster.Owner(*targetView, store.GrainId(candidate)) + candidate := GrainId{GrainType: GrainType("gor.blockingGrain"), GrainKey: strconv.Itoa(index)} + owner, ok := cluster.Owner(*targetView, toStoreGrainID(candidate)) if ok && owner == "node-b" { id = candidate owned = *targetView @@ -121,7 +123,7 @@ func TestScenario_ForwardedCallSurvivesOwnerClose(t *testing.T) { t.Fatal("no identity owned by the target node") } // Sanity: the source routes this identity to the target. - if owner, _ := cluster.Owner(owned, store.GrainId(id)); owner != "node-b" { + if owner, _ := cluster.Owner(owned, toStoreGrainID(id)); owner != "node-b" { t.Fatalf("source routes %v to %q, want node-b", id, owner) } @@ -136,7 +138,7 @@ func TestScenario_ForwardedCallSurvivesOwnerClose(t *testing.T) { closeDone := make(chan struct{}) go func() { - target.Close() + closeRuntime(target) close(closeDone) }() synctest.Wait() diff --git a/forward_test.go b/forward_test.go index 5e4b70f..b0ee5cb 100644 --- a/forward_test.go +++ b/forward_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "strconv" "strings" "sync" "sync/atomic" @@ -20,8 +21,9 @@ import ( func TestRuntime_HandleInvokesMethodAndEncodesReply(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) registerAccount(t, rt) + mustStart(t, rt) payload, err := rt.handle(context.Background(), []byte(`{"kind":"invoke","type":"gor.Account","key":"alice","method":"Deposit","args":{"A0":4}}`)) if err != nil { @@ -45,18 +47,19 @@ func TestRuntime_HandleInvokesMethodAndEncodesReply(t *testing.T) { func TestRuntime_HandleReturnsMethodErrorInResponse(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installAccountWithDispatch(t, rt, func(ctx context.Context, instance Account, method string, args any, reply any) error { if method == "Fail" { return errors.New("method failed") } return dispatchAccount(ctx, instance, method, args, reply) }) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) payload, err := rt.handle(context.Background(), []byte(`{"kind":"invoke","type":"gor.Account","key":"alice","method":"Fail","args":{}}`)) if err != nil { @@ -73,8 +76,9 @@ func TestRuntime_HandleReturnsMethodErrorInResponse(t *testing.T) { func TestRuntime_HandlePrioritizesBusinessErrorOverReplyEncoding(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installEnvelopeAccount(t, rt) + mustStart(t, rt) payload, err := rt.handle(context.Background(), []byte(`{"kind":"invoke","type":"gor.envelopeAccount","key":"alice","method":"Fail","args":{}}`)) if err != nil { @@ -102,8 +106,9 @@ func TestRuntime_HandlePrioritizesBusinessErrorOverReplyEncoding(t *testing.T) { func TestRuntime_HandleRejectsUnknownMethod(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) registerAccount(t, rt) + mustStart(t, rt) payload, err := rt.handle(context.Background(), []byte(`{"kind":"invoke","type":"gor.Account","key":"alice","method":"Missing","args":{}}`)) if err != nil { @@ -120,7 +125,8 @@ func TestRuntime_HandleRejectsUnknownMethod(t *testing.T) { func TestRuntime_HandleRejectsUnregisteredType(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) + mustStart(t, rt) payload, err := rt.handle(context.Background(), []byte(`{"kind":"invoke","type":"missing.Account","key":"alice","method":"Balance","args":{}}`)) if err != nil { @@ -137,7 +143,8 @@ func TestRuntime_HandleRejectsUnregisteredType(t *testing.T) { func TestRuntime_HandleRejectsBadJSON(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) + mustStart(t, rt) payload, err := rt.handle(context.Background(), []byte(`{"kind":"invoke","type":"gor.Account","key":"alice","method":"Deposit","args":{"A0":`)) if err != nil { @@ -154,8 +161,9 @@ func TestRuntime_HandleRejectsBadJSON(t *testing.T) { func TestRuntime_HandleRejectsBadArguments(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) registerAccount(t, rt) + mustStart(t, rt) payload, err := rt.handle(context.Background(), []byte(`{"kind":"invoke","type":"gor.Account","key":"alice","method":"Deposit","args":{"A0":"wrong"}}`)) if err != nil { @@ -172,7 +180,7 @@ func TestRuntime_HandleRejectsBadArguments(t *testing.T) { func TestRuntime_HandleRejectsClosedRuntime(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - rt.Close() + closeRuntime(rt) payload, err := rt.handle(context.Background(), []byte(`{"kind":"invoke","type":"gor.Account","key":"alice","method":"Balance","args":{}}`)) if err != nil { @@ -200,22 +208,23 @@ func TestRuntime_HandleRejectsWhileClosing(t *testing.T) { } return dispatchAccount(ctx, instance, method, args, reply) }) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) callDone := make(chan error, 1) go func() { - callDone <- rt.Invoke(context.Background(), GrainId{GrainType: TypeName[Account](), GrainKey: "alice"}, "Block", nil, nil) + callDone <- rt.Invoke(context.Background(), GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"}, "Block", nil, nil) }() synctest.Wait() <-started closeDone := make(chan struct{}) go func() { - rt.Close() + closeRuntime(rt) close(closeDone) }() synctest.Wait() @@ -256,10 +265,12 @@ func TestRuntime_InvokeForwardsToOwner(t *testing.T) { secondOptions = append(secondOptions, WithTransport(secondTransport)) first := mustNew(t, firstOptions...) second := mustNew(t, secondOptions...) - defer first.Close() - defer second.Close() + defer closeRuntime(first) + defer closeRuntime(second) installRoutedAccount(t, first, "node-a") installRoutedAccount(t, second, "node-b") + mustStart(t, first) + mustStart(t, second) synctest.Wait() <-firstTransport.served <-secondTransport.served @@ -378,18 +389,20 @@ func TestRuntime_ForwardedCallsUseLocalInvokeSerialization(t *testing.T) { secondOptions = append(secondOptions, WithTransport(secondTransport)) first := mustNew(t, firstOptions...) second := mustNew(t, secondOptions...) - defer first.Close() - defer second.Close() + defer closeRuntime(first) + defer closeRuntime(second) installRoutedAccount(t, first, "node-a") started := make(chan struct{}) release := make(chan struct{}) - installRoutedAccountWithFactory(t, second, func(*Binder) routedAccount { - return &routedAccountEntity{ + installRoutedAccountWithFactory(t, second, func(*GrainContext) routedAccount { + return &routedAccountGrain{ label: "node-b", blockStarted: started, blockRelease: release, } }) + mustStart(t, first) + mustStart(t, second) synctest.Wait() <-firstTransport.served <-secondTransport.served @@ -434,6 +447,120 @@ func TestRuntime_ForwardedCallsUseLocalInvokeSerialization(t *testing.T) { }) } +func TestRuntime_OwnershipChangeReroutesQueuedCall(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(1235, 0).UTC() + fakeClock := clock.NewFake(start) + members := store.NewMemory() + backend := store.NewMemory() + network := newTestTransportNetwork() + first := mustNew(t, clusterRuntimeOptions(backend, members, fakeClock, "node-a", "generation-a", network.add("node-a"))...) + defer closeRuntime(first) + + started := make(chan struct{}) + release := make(chan struct{}) + var firstFactories atomic.Int32 + installRoutedAccountWithFactory(t, first, func(*GrainContext) routedAccount { + firstFactories.Add(1) + return &routedAccountGrain{label: "node-a", blockStarted: started, blockRelease: release} + }) + mustStart(t, first) + + before := cluster.NewView([]store.Member{{ + NodeAddr: "node-a", + Generation: "generation-a", + Status: store.MemberActive, + }}) + after := cluster.NewView([]store.Member{ + { + NodeAddr: "node-a", + Generation: "generation-a", + Status: store.MemberActive, + }, + { + NodeAddr: "node-b", + Generation: "generation-b", + Status: store.MemberActive, + }, + }) + var target GrainId + for index := 0; index < 4096; index++ { + candidate := GrainId{GrainType: "gor.routedAccount", GrainKey: strconv.Itoa(index)} + beforeOwner, beforeOK := cluster.Owner(before, toStoreGrainID(candidate)) + afterOwner, afterOK := cluster.Owner(after, toStoreGrainID(candidate)) + if beforeOK && afterOK && beforeOwner == "node-a" && afterOwner == "node-b" { + target = candidate + break + } + } + if target == (GrainId{}) { + t.Fatal("no identity moved from node-a to node-b") + } + + blockDone := make(chan error, 1) + go func() { + blockDone <- first.Invoke(context.Background(), target, "Block", &routedAccountBlockRequest{}, &routedAccountBlockReply{}) + }() + synctest.Wait() + <-started + + whoDone := make(chan struct { + reply routedAccountWhoReply + err error + }, 1) + go func() { + var reply routedAccountWhoReply + err := first.Invoke(context.Background(), target, "Who", &routedAccountWhoRequest{}, &reply) + whoDone <- struct { + reply routedAccountWhoReply + err error + }{reply: reply, err: err} + }() + synctest.Wait() + select { + case <-whoDone: + t.Fatal("second Call did not wait in the old owner lane") + default: + } + + second := mustNew(t, clusterRuntimeOptions(backend, members, fakeClock, "node-b", "generation-b", network.add("node-b"))...) + defer closeRuntime(second) + var secondFactories atomic.Int32 + installRoutedAccountWithFactory(t, second, func(*GrainContext) routedAccount { + secondFactories.Add(1) + return &routedAccountGrain{label: "node-b"} + }) + mustStart(t, second) + synctest.Wait() + + first.clusterView.Store(&after) + second.clusterView.Store(&after) + first.deactivateMovedActivations(after) + synctest.Wait() + select { + case <-whoDone: + t.Fatal("queued Call completed before the old method ended") + default: + } + + close(release) + synctest.Wait() + if err := <-blockDone; err != nil { + t.Fatalf("old owner Block error = %v", err) + } + result := <-whoDone + if result.err != nil || result.reply.R0 != "node-b" { + t.Fatalf("rerouted Call = (%q, %v), want (node-b, nil)", result.reply.R0, result.err) + } + if got := firstFactories.Load(); got != 1 { + t.Fatalf("node-a factory calls = %d, want 1", got) + } + if got := secondFactories.Load(); got != 1 { + t.Fatalf("node-b factory calls = %d, want 1", got) + } + }) +} + func TestRuntime_HandleDoesNotRouteAgain(t *testing.T) { synctest.Test(t, func(t *testing.T) { start := time.Unix(1250, 0).UTC() @@ -449,10 +576,12 @@ func TestRuntime_HandleDoesNotRouteAgain(t *testing.T) { secondOptions = append(secondOptions, WithTransport(secondTransport)) first := mustNew(t, firstOptions...) second := mustNew(t, secondOptions...) - defer first.Close() - defer second.Close() + defer closeRuntime(first) + defer closeRuntime(second) installRoutedAccount(t, first, "node-a") installRoutedAccount(t, second, "node-b") + mustStart(t, first) + mustStart(t, second) synctest.Wait() <-firstTransport.served <-secondTransport.served @@ -497,15 +626,15 @@ func TestRuntime_ForwardCancellationDoesNotCancelRemote(t *testing.T) { secondOptions = append(secondOptions, WithTransport(secondTransport)) first := mustNew(t, firstOptions...) second := mustNew(t, secondOptions...) - defer first.Close() - defer second.Close() + defer closeRuntime(first) + defer closeRuntime(second) installRoutedAccount(t, first, "node-a") started := make(chan struct{}) release := make(chan struct{}) observedContext := make(chan context.Context) finished := make(chan struct{}) - installRoutedAccountWithFactory(t, second, func(*Binder) routedAccount { - return &routedAccountEntity{ + installRoutedAccountWithFactory(t, second, func(*GrainContext) routedAccount { + return &routedAccountGrain{ label: "node-b", blockStarted: started, blockRelease: release, @@ -513,6 +642,8 @@ func TestRuntime_ForwardCancellationDoesNotCancelRemote(t *testing.T) { blockFinished: finished, } }) + mustStart(t, first) + mustStart(t, second) synctest.Wait() <-firstTransport.served <-secondTransport.served @@ -578,8 +709,9 @@ func TestRuntime_InvokeDoesNotSendWithoutOwner(t *testing.T) { options := clusterRuntimeOptions(store.NewMemory(), members, fakeClock, "node-a", "generation-a") options = append(options, WithTransport(fakeTransport)) rt := mustNew(t, options...) - defer rt.Close() + defer closeRuntime(rt) installRoutedAccount(t, rt, "node-a") + mustStart(t, rt) synctest.Wait() <-fakeTransport.served @@ -592,7 +724,7 @@ func TestRuntime_InvokeDoesNotSendWithoutOwner(t *testing.T) { synctest.Wait() var reply routedAccountWhoReply - err := rt.Invoke(context.Background(), GrainId{GrainType: TypeName[routedAccount](), GrainKey: "alice"}, "Who", &routedAccountWhoRequest{}, &reply) + err := rt.Invoke(context.Background(), GrainId{GrainType: GrainType("gor.routedAccount"), GrainKey: "alice"}, "Who", &routedAccountWhoRequest{}, &reply) if !errors.Is(err, ErrNodeDead) { t.Fatalf("invocation on a dead node error = %v, want ErrNodeDead", err) } @@ -612,15 +744,16 @@ func TestRuntime_StartsAndClosesConfiguredTransport(t *testing.T) { options := clusterRuntimeOptions(store.NewMemory(), members, fakeClock, "node-a", "generation-a") options = append(options, WithTransport(fakeTransport)) rt := mustNew(t, options...) + mustStart(t, rt) <-fakeTransport.served closeDone := make(chan struct{}, 2) go func() { - rt.Close() + closeRuntime(rt) closeDone <- struct{}{} }() go func() { - rt.Close() + closeRuntime(rt) closeDone <- struct{}{} }() synctest.Wait() @@ -672,15 +805,15 @@ type envelopeAccountReply struct { Callback func() } -type envelopeAccountEntity struct{} +type envelopeAccountGrain struct{} const envelopeFailureCode Code = "test.envelope_failure" -func (*envelopeAccountEntity) Fail(context.Context) error { +func (*envelopeAccountGrain) Fail(context.Context) error { return fmt.Errorf("business failure: %w", envelopeFailureCode) } -func (*envelopeAccountEntity) Succeed(context.Context) error { +func (*envelopeAccountGrain) Succeed(context.Context) error { return nil } @@ -708,13 +841,13 @@ func newEnvelopeAccountCall(method string) (args any, reply any) { func installEnvelopeAccount(t *testing.T, rt *Runtime) { t.Helper() - if err := InstallType[envelopeAccount](rt, dispatchEnvelopeAccount, func(invoker Invoker, id GrainId) envelopeAccount { + if err := InstallType[envelopeAccount](rt, GeneratedCodeVersion, "gor.envelopeAccount", dispatchEnvelopeAccount, func(invoker Invoker, id GrainId) envelopeAccount { return &envelopeAccountProxy{invoker: invoker, id: id} - }, newEnvelopeAccountCall, nil); err != nil { + }, newEnvelopeAccountCall, noReminderCall); err != nil { t.Fatal(err) } - if err := Register[envelopeAccount](rt, func(*Binder) envelopeAccount { - return &envelopeAccountEntity{} + if err := Register[envelopeAccount](rt, func(*GrainContext) envelopeAccount { + return &envelopeAccountGrain{} }); err != nil { t.Fatal(err) } @@ -735,7 +868,7 @@ func (p *envelopeAccountProxy) Succeed(ctx context.Context) error { return p.invoker.Invoke(ctx, p.id, "Succeed", &envelopeAccountRequest{}, &reply) } -type routedAccountEntity struct { +type routedAccountGrain struct { label string blockStarted chan struct{} blockRelease chan struct{} @@ -786,15 +919,15 @@ type unmarshalableArgs struct { Callback func() } -func (a *routedAccountEntity) Who(context.Context) (string, error) { +func (a *routedAccountGrain) Who(context.Context) (string, error) { return a.label, nil } -func (a *routedAccountEntity) Echo(_ context.Context, value string) (string, error) { +func (a *routedAccountGrain) Echo(_ context.Context, value string) (string, error) { return a.label + ":" + value, nil } -func (a *routedAccountEntity) Block(ctx context.Context) error { +func (a *routedAccountGrain) Block(ctx context.Context) error { if a.blockStarted != nil { close(a.blockStarted) } @@ -808,11 +941,11 @@ func (a *routedAccountEntity) Block(ctx context.Context) error { return nil } -func (*routedAccountEntity) Fail(context.Context) error { +func (*routedAccountGrain) Fail(context.Context) error { return fmt.Errorf("remote failure: %w", routedFailureCode) } -func (*routedAccountEntity) Opaque(context.Context) error { +func (*routedAccountGrain) Opaque(context.Context) error { return remoteFailure } @@ -859,16 +992,16 @@ func newRoutedAccountCall(method string) (args any, reply any) { } func installRoutedAccount(t *testing.T, rt *Runtime, label string) { - installRoutedAccountWithFactory(t, rt, func(*Binder) routedAccount { - return &routedAccountEntity{label: label} + installRoutedAccountWithFactory(t, rt, func(*GrainContext) routedAccount { + return &routedAccountGrain{label: label} }) } -func installRoutedAccountWithFactory(t *testing.T, rt *Runtime, factory func(*Binder) routedAccount) { +func installRoutedAccountWithFactory(t *testing.T, rt *Runtime, factory func(*GrainContext) routedAccount) { t.Helper() - if err := InstallType[routedAccount](rt, dispatchRoutedAccount, func(invoker Invoker, id GrainId) routedAccount { + if err := InstallType[routedAccount](rt, GeneratedCodeVersion, "gor.routedAccount", dispatchRoutedAccount, func(invoker Invoker, id GrainId) routedAccount { return &routedAccountProxy{invoker: invoker, id: id} - }, newRoutedAccountCall, nil); err != nil { + }, newRoutedAccountCall, noReminderCall); err != nil { t.Fatal(err) } if err := Register[routedAccount](rt, factory); err != nil { @@ -912,8 +1045,8 @@ func findForwardTarget(t *testing.T, rt *Runtime, owner string) GrainId { t.Helper() view := rt.clusterView.Load() for index := 0; index < 4096; index++ { - candidate := GrainId{GrainType: TypeName[routedAccount](), GrainKey: fmt.Sprintf("forward-%d", index)} - candidateOwner, ok := cluster.Owner(*view, store.GrainId(candidate)) + candidate := GrainId{GrainType: GrainType("gor.routedAccount"), GrainKey: fmt.Sprintf("forward-%d", index)} + candidateOwner, ok := cluster.Owner(*view, toStoreGrainID(candidate)) if ok && candidateOwner == owner { return candidate } @@ -1107,8 +1240,8 @@ func (r *recordingTransport) Kill() error { } // TestRuntime_StopModeRouting pins which transport stop method each root stop -// path selects: a graceful stop calls Close and never Kill; a Kill escalation -// reaches the transport as Kill even when a Close already completed. +// path selects. A graceful Shutdown calls Close. A Shutdown whose context is +// already canceled uses the abrupt path and calls Kill. func TestRuntime_StopModeRouting(t *testing.T) { synctest.Test(t, func(t *testing.T) { start := time.Unix(1750, 0).UTC() @@ -1123,12 +1256,13 @@ func TestRuntime_StopModeRouting(t *testing.T) { } options := clusterRuntimeOptions(store.NewMemory(), members, fakeClock, "node-a", "generation-a", recorded) rt := mustNew(t, options...) + mustStart(t, rt) synctest.Wait() <-base.served closeDone := make(chan struct{}) go func() { - rt.Close() + closeRuntime(rt) close(closeDone) }() synctest.Wait() @@ -1148,23 +1282,29 @@ func TestRuntime_StopModeRouting(t *testing.T) { t.Fatal("runtime Close did not return") } - // A Kill after a completed graceful stop still reaches the transport - // as Kill: the transport must not treat the later Kill as a no-op. - killDone := make(chan struct{}) - go func() { - rt.Kill() - close(killDone) - }() + abruptBase := network.add("node-b") + abruptTransport := &recordingTransport{ + testTransport: abruptBase, + closeCalls: make(chan struct{}, 4), + killCalls: make(chan struct{}, 4), + } + abruptOptions := clusterRuntimeOptions(store.NewMemory(), members, fakeClock, "node-b", "generation-b", abruptTransport) + abruptRuntime := mustNew(t, abruptOptions...) + mustStart(t, abruptRuntime) + synctest.Wait() + <-abruptBase.served + + killRuntime(abruptRuntime) synctest.Wait() select { - case <-recorded.killCalls: + case <-abruptTransport.killCalls: default: - t.Fatal("Kill after graceful stop did not call Transport.Kill") + t.Fatal("abrupt Shutdown did not call Transport.Kill") } select { - case <-killDone: + case <-abruptTransport.closeCalls: + t.Fatal("abrupt Shutdown also started Transport.Close") default: - t.Fatal("runtime Kill did not return") } }) } diff --git a/generated_code_version_test.go b/generated_code_version_test.go new file mode 100644 index 0000000..d49537a --- /dev/null +++ b/generated_code_version_test.go @@ -0,0 +1,38 @@ +package gor + +import ( + "errors" + "strings" + "testing" +) + +func TestInstallType_RejectsGeneratedCodeVersionBeforeMutation(t *testing.T) { + rt := mustNew(t) + err := InstallType[Account]( + rt, + GeneratedCodeVersion+1, + "gor.Account", + dispatchAccount, + accountProxyFactory, + newAccountCall, + noReminderCall, + ) + if !errors.Is(err, ErrInvalidSetup) { + t.Fatalf("version mismatch = %v, want ErrInvalidSetup", err) + } + if !strings.Contains(err.Error(), "run go generate") { + t.Fatalf("version mismatch = %q, want regeneration action", err) + } + + if err := InstallType[Account]( + rt, + GeneratedCodeVersion, + "gor.Account", + dispatchAccount, + accountProxyFactory, + newAccountCall, + noReminderCall, + ); err != nil { + t.Fatalf("install after rejected version = %v", err) + } +} diff --git a/go.mod b/go.mod index ca6c16c..e342948 100644 --- a/go.mod +++ b/go.mod @@ -4,20 +4,31 @@ go 1.25.0 require ( github.com/anishathalye/porcupine v1.3.0 + golang.org/x/mod v0.38.0 + golang.org/x/sys v0.47.0 golang.org/x/tools v0.48.0 modernc.org/sqlite v1.55.0 ) require ( + github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/mod v0.38.0 // indirect + golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect + golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 // indirect + golang.org/x/vuln v1.6.0 // indirect + honnef.co/go/tools v0.7.0 // indirect modernc.org/libc v1.74.1 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect ) + +tool ( + github.com/suraciii/gor/cmd/gorgen + golang.org/x/vuln/cmd/govulncheck + honnef.co/go/tools/cmd/staticcheck +) diff --git a/go.sum b/go.sum index 9043241..c99f9c7 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,17 @@ +github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs= +github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/anishathalye/porcupine v1.3.0 h1:yo51Niv8Tg0tAAn5XOG2UVvJXUregK4WFuLrBRoowP8= github.com/anishathalye/porcupine v1.3.0/go.mod h1:WM0SsFjWNl2Y4BqHr/E/ll2yY1GY1jqn+W7Z/84Zoog= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU= +github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -16,6 +22,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 h1:1P7xPZEwZMoBoz0Yze5Nx2/4pxj6nw9ZqHWXqP0iRgQ= +golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= @@ -23,8 +31,18 @@ golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 h1:RJhm5l6Fo4rmEIcndxDllNhhf/fAx8qIm4t6A7vpm2A= +golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959/go.mod h1:LV7u5Oco+Z/g6XI7PqN+EUUUGGkEcmB1uj2ceI0fOVg= golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= +golang.org/x/vuln v1.6.0 h1:FeMO9Rm/HwyduOztbvKcOw+zvDEPr4I4aQNSfevFcKY= +golang.org/x/vuln v1.6.0/go.mod h1:bWlG2493/sjR7ksvicBgMrznH3eYQEyK8ifUYBrqUbg= +honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= +honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= diff --git a/gor.go b/gor.go index b39a4d9..4761eca 100644 --- a/gor.go +++ b/gor.go @@ -1,44 +1,64 @@ -// Package gor provides the application-facing API for defining, registering, -// invoking, and persisting virtual actors. +// Package gor provides the Application API for defining, registering, +// calling, and persisting Grains. package gor import ( "context" "errors" "fmt" - "strings" + "reflect" + "sort" "sync" "sync/atomic" "time" "github.com/suraciii/gor/clock" "github.com/suraciii/gor/cluster" - runtimepkg "github.com/suraciii/gor/runtime" + "github.com/suraciii/gor/internal/generatedcode" + runtimepkg "github.com/suraciii/gor/internal/runtime" + "github.com/suraciii/gor/internal/timer" "github.com/suraciii/gor/store" - "github.com/suraciii/gor/timer" "github.com/suraciii/gor/transport" ) -// GrainId identifies an entity by its registered type name and key. -type GrainId = runtimepkg.GrainId -type Activation = runtimepkg.Activation +// GrainType is the stable Application name of one Grain type. +type GrainType string + +// GeneratedCodeVersion is the generated binding version accepted by Runtime. +// Generated files pass it to InstallType. Application code does not use it. +const GeneratedCodeVersion = generatedcode.Version + +// GrainId identifies a Grain by its stable type and key. +type GrainId struct { + GrainType GrainType + GrainKey string +} + +// Activation describes one active Grain instance. +type Activation struct { + GrainId GrainId + Queued int +} // DeactivationReason describes why an activation left the active state. The // reason is fixed at the first transition out of active and is never rewritten // by later events. -type DeactivationReason = runtimepkg.DeactivationReason +type DeactivationReason uint8 const ( // Idle reports that the activation was evicted for idleness. - Idle DeactivationReason = runtimepkg.Idle - // OwnershipLost reports that this node no longer owns the identity, or + Idle DeactivationReason = iota + 1 + // ApplicationRequested reports that the Grain requested Deactivate on + // Idle during a method. + ApplicationRequested + // OwnershipLost reports that this Silo no longer owns the GrainId, or // the view has no active owner. - OwnershipLost DeactivationReason = runtimepkg.OwnershipLost + OwnershipLost // RuntimeClosed reports that the root runtime began a normal shutdown. - RuntimeClosed DeactivationReason = runtimepkg.RuntimeClosed - // Faulted reports that the instance is no longer trusted: a method - // panicked, or the entity requested discarding the current instance. - Faulted DeactivationReason = runtimepkg.Faulted + RuntimeClosed + // Faulted reports that the Activation is no longer trusted. A method + // panicked, or the Grain Runtime discarded it after a State failure. + Faulted ) // CallObservation describes one invocation observed by an OnCall callback. @@ -51,44 +71,81 @@ const ( // observation. A remote coded error is reconstructed so errors.Is matches its // Code; an opaque remote error retains only its diagnostic text. type CallObservation struct { - GrainType string + GrainType GrainType Method string Duration time.Duration Err error } -// Scope is the generated-reference scope accepted by Ref. Runtime and Binder -// implement it; application code should pass those values rather than -// implement Scope. +// Scope is the generated-reference scope accepted by Ref. Runtime and +// GrainContext implement it. Application code should pass those values rather +// than implement Scope. type Scope interface { scopeRuntime() *Runtime } -// BackgroundError reports a failure of an application callback that has no -// caller waiting for its result: a claimed Reminder invocation, or a normal -// deactivation hook. GrainId is the affected Grain, Err is the callback's -// error, and Source identifies which kind of callback failed. +// BackgroundError reports a failure that has no caller waiting for its result. +// GrainId is the affected Grain when one is known. Err contains the original +// callback or Store error. Source identifies the background work. type BackgroundError struct { GrainId GrainId Err error Source ErrorSource } -// ErrorSource identifies the kind of callback that produced a BackgroundError. +// ErrorSource identifies the background work that produced a BackgroundError. // Its unexported method seals the set: only the gor package can add sources, // so application code can branch on the concrete type without a fallback. type ErrorSource interface { errorSource() } -// ReminderInvocation is the source of a failure from a claimed Reminder. -// Method is the Grain method that was invoked. +// ReminderInvocation is the source of a failure from a claimed Reminder Call. +// It identifies the Reminder and the tick delivered to the Grain. type ReminderInvocation struct { - Method string + Name string + Method string + TickStatus TickStatus } func (ReminderInvocation) errorSource() {} +// ReminderScan is the source of a failed Reminder Store scan. +type ReminderScan struct{} + +func (ReminderScan) errorSource() {} + +// ReminderClaim is the source of a Reminder Store Claim error. A lost Claim +// CAS is normal contention and is not a BackgroundError. +type ReminderClaim struct { + Name string +} + +func (ReminderClaim) errorSource() {} + +// ReminderDispatch is the source of a stored GrainType or Reminder method +// that the Runtime cannot resolve before Claim. +type ReminderDispatch struct { + Name string + Method string +} + +func (ReminderDispatch) errorSource() {} + +// ReminderTerminal is the source of a Store error while the Runtime gives an +// Invalid Reminder a Terminal Result. +type ReminderTerminal struct { + Name string + Method string +} + +func (ReminderTerminal) errorSource() {} + +// GrainTimerInvocation is the source of a Grain Timer callback failure. +type GrainTimerInvocation struct{} + +func (GrainTimerInvocation) errorSource() {} + // Deactivation is the source of a failure from a normal deactivation hook. // Reason is the reason of that deactivation. type Deactivation struct { @@ -97,29 +154,31 @@ type Deactivation struct { func (Deactivation) errorSource() {} -// Activatable is implemented by an entity that needs a hook after its state is +// Activatable is implemented by a Grain that needs a hook after its State is // loaded and before its first method call. An OnActivate error prevents that // activation from being established and is returned by the triggering call. type Activatable interface { OnActivate(context.Context) error } -// Deactivatable is implemented by an entity that needs a hook when its +// Deactivatable is implemented by a Grain that needs a hook when its // activation leaves the active state. reason is fixed when the deactivation // begins and is never rewritten. The hook receives a fresh context with no // deadline that is never canceled, independent of any caller's context; a // normal shutdown waits for the hook, so it must finish promptly. The hook // cannot prevent deactivation. Its error is reported through OnError when -// configured; Kill skips this hook. +// configured; an abrupt shutdown skips this hook. type Deactivatable interface { OnDeactivate(context.Context, DeactivationReason) error } -// Runtime coordinates entity registration, activation, invocation, state, and -// reminders. Invocations for the same identity are serialized, and a runtime -// configured for a cluster can route an invocation to its current owner. -// Create a Runtime with New and stop it with Close or Kill. +// Runtime coordinates Grain registration, Activation, Calls, State, Grain +// Timers, and Reminders. Calls for the same GrainId are serialized. A Runtime +// configured for a Cluster can route a Call to its current owner. +// Create a Runtime with New, complete setup, call Start, and stop it with +// Shutdown. type Runtime struct { + config Config engine *runtimepkg.Runtime store store.Store reminderStore store.ReminderStore @@ -134,7 +193,14 @@ type Runtime struct { clusterNode *cluster.Node clusterView atomic.Pointer[cluster.View] clusterDone chan struct{} + stopping chan struct{} done chan struct{} + abrupt chan struct{} + stoppingOnce sync.Once + doneOnce sync.Once + abruptOnce sync.Once + startDone chan struct{} + startCancel context.CancelFunc lifecycleMu sync.Mutex state rootState @@ -142,9 +208,9 @@ type Runtime struct { inflight int drained chan struct{} - nodeAddr string - typesMu sync.Mutex - types map[string]typeRegistration + nodeAddr string + types map[reflect.Type]*typeRegistration + grainTypes map[GrainType]*typeRegistration } // Config contains the settings assembled by New from Option values. Use the @@ -152,10 +218,16 @@ type Runtime struct { // inspect or modify Config directly. Omitted fields receive the defaults // described by their corresponding option. type Config struct { - runtimepkg.Config + Clock clock.Clock + MailboxCapacity int + MaxActivations int + IdleTimeout time.Duration + EvictionInterval time.Duration Store store.Store ReminderStore store.ReminderStore ReminderInterval time.Duration + ReminderPageSize int + ReminderWorkers int Transport transport.Transport OnError func(BackgroundError) OnCall func(CallObservation) @@ -172,9 +244,9 @@ type Config struct { MaxTableLatency time.Duration } -// Invoker is the generated proxy call boundary. Generated code receives an -// Invoker from the runtime; application code normally consumes generated -// proxies instead of implementing Invoker. +// Invoker is the generated Grain Reference Call boundary. Generated code +// receives an Invoker from the Runtime. Application code uses generated Grain +// References instead of implementing Invoker. type Invoker interface { // Invoke invokes method for id, using the generated request in args and // writing the generated response to reply. For a local call, ctx bounds @@ -188,112 +260,314 @@ type Invoker interface { var _ Invoker = (*Runtime)(nil) type typeRegistration struct { + localType reflect.Type + grainType GrainType dispatch runtimepkg.Dispatch newProxy func(Invoker, GrainId) any newCall func(string) (any, any) newReminderCall func(string, TickStatus) (any, any) + factory func(*GrainContext) any } // Option configures a Runtime created by New. New applies options in argument -// order, then derives a schedule store when none was supplied. +// order. Start validates the complete result. type Option func(*Config) func (rt *Runtime) scopeRuntime() *Runtime { return rt } -func (b *Binder) scopeRuntime() *Runtime { - return b.runtime +func (g *GrainContext) scopeRuntime() *Runtime { + return g.runtime } -// New creates and starts a Runtime. -// -// By default, New uses clock.Real{}, store.NewMemory for entity state and -// reminders, a mailbox capacity of 16, a one-minute idle timeout, one-second -// eviction and Reminder intervals, and one-second heartbeat and view -// intervals. A MemberStore and Transport must be configured together. In -// clustered mode, ProbeInterval, ProbeTimeout, ProbeFailures, VoteTTL, -// MaxTickGap, and MaxTableLatency default to one second, 500 ms, three, six -// seconds, two seconds, and 500 ms; a negative value returns an error matching -// cluster.ErrInvalidConfig. +// New constructs a Runtime. It does not start background work or access a +// Store. Install generated bindings and register Grain factories before Start. // -// New returns an error if cluster initialization fails. The returned Runtime -// is ready for entity installation and registration. +// The State Store and Reminder Store are required and have no default. The +// memory Store is available only when the Application selects it explicitly. func New(options ...Option) (*Runtime, error) { config := Config{ - Config: runtimepkg.Config{ - Clock: clock.Real{}, - MailboxCapacity: 16, - IdleTimeout: time.Minute, - EvictionInterval: time.Second, - }, - Store: store.NewMemory(), + Clock: clock.Real{}, + MailboxCapacity: 16, + MaxActivations: 10000, + IdleTimeout: time.Minute, + EvictionInterval: time.Second, ReminderInterval: time.Second, + ReminderPageSize: 256, + ReminderWorkers: 16, HeartbeatInterval: time.Second, ViewInterval: time.Second, } for _, option := range options { option(&config) } - if (config.MemberStore == nil) != (config.Transport == nil) { - return nil, errors.New("member store and transport must be configured together") + rt := &Runtime{ + config: config, + store: config.Store, + reminderStore: config.ReminderStore, + clock: config.Clock, + onError: config.OnError, + onCall: config.OnCall, + transport: config.Transport, + stopping: make(chan struct{}), + done: make(chan struct{}), + abrupt: make(chan struct{}), + state: rootConstructed, + stopCode: ErrRuntimeNotStarted, + types: make(map[reflect.Type]*typeRegistration), } - if config.ReminderStore == nil { - if reminders, ok := config.Store.(store.ReminderStore); ok { - config.ReminderStore = reminders + return rt, nil +} + +// Start validates and freezes setup, checks the Stores, and starts the +// Runtime. A failed Start cannot be retried on the same Runtime. +func (rt *Runtime) Start(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + startupCtx, cancel := context.WithCancel(ctx) + rt.lifecycleMu.Lock() + if rt.state != rootConstructed { + rt.lifecycleMu.Unlock() + cancel() + return withCode(ErrSetupFrozen, errors.New("runtime setup is frozen")) + } + rt.state = rootStarting + rt.startDone = make(chan struct{}) + rt.startCancel = cancel + startDone := rt.startDone + rt.lifecycleMu.Unlock() + defer func() { + cancel() + rt.lifecycleMu.Lock() + rt.startCancel = nil + close(startDone) + rt.lifecycleMu.Unlock() + }() + + registrations, grainTypes, err := rt.validateSetup() + if err != nil { + rt.failStart() + return err + } + if err := rt.store.Check(startupCtx); err != nil { + rt.failStart() + return setupCheckError("State Store", err) + } + if err := rt.reminderStore.Check(startupCtx); err != nil { + rt.failStart() + return setupCheckError("Reminder Store", err) + } + if err := startupCtx.Err(); err != nil { + rt.failStart() + return err + } + + engine := runtimepkg.New(runtimepkg.Config{ + Clock: rt.config.Clock, + MailboxCapacity: rt.config.MailboxCapacity, + MaxActivations: rt.config.MaxActivations, + IdleTimeout: rt.config.IdleTimeout, + EvictionInterval: rt.config.EvictionInterval, + }) + for _, registration := range registrations { + if err := engine.Register(string(registration.grainType), rt.runtimeRegistration(registration)); err != nil { + engine.BeginKill() + <-engine.Done() + rt.failStart() + return withCode(ErrInvalidSetup, fmt.Errorf("register GrainType %q: %w", registration.grainType, err)) } } + var ( clusterNode *cluster.Node initialView cluster.View ) - if config.MemberStore != nil { - var err error - clusterNode, err = cluster.New(cluster.Config{ - Table: config.MemberStore, - Clock: config.Clock, - Prober: transportProber{transport: config.Transport}, - NodeAddr: config.NodeAddr, - Generation: config.Generation, - HeartbeatInterval: config.HeartbeatInterval, - ViewInterval: config.ViewInterval, - ProbeInterval: config.ProbeInterval, - ProbeTimeout: config.ProbeTimeout, - ProbeFailures: config.ProbeFailures, - VoteTTL: config.VoteTTL, - MaxTickGap: config.MaxTickGap, - MaxTableLatency: config.MaxTableLatency, + if !isNil(rt.config.MemberStore) { + clusterNode, err = cluster.NewContext(startupCtx, cluster.Config{ + Table: rt.config.MemberStore, + Clock: rt.config.Clock, + Prober: transportProber{transport: rt.config.Transport}, + NodeAddr: rt.config.NodeAddr, + Generation: rt.config.Generation, + HeartbeatInterval: rt.config.HeartbeatInterval, + ViewInterval: rt.config.ViewInterval, + ProbeInterval: rt.config.ProbeInterval, + ProbeTimeout: rt.config.ProbeTimeout, + ProbeFailures: rt.config.ProbeFailures, + VoteTTL: rt.config.VoteTTL, + MaxTickGap: rt.config.MaxTickGap, + MaxTableLatency: rt.config.MaxTableLatency, }) if err != nil { - return nil, fmt.Errorf("start cluster node: %w", err) + engine.BeginKill() + <-engine.Done() + rt.failStart() + return setupCheckError("cluster", err) } initialView = <-clusterNode.ViewChanges() } - rt := &Runtime{ - engine: runtimepkg.New(config.Config), - store: config.Store, - reminderStore: config.ReminderStore, - clock: config.Clock, - onError: config.OnError, - onCall: config.OnCall, - transport: config.Transport, - done: make(chan struct{}), - types: make(map[string]typeRegistration), + if err := startupCtx.Err(); err != nil { + if clusterNode != nil { + clusterNode.CancelStart() + } + engine.BeginKill() + <-engine.Done() + rt.failStart() + return err } + + rt.lifecycleMu.Lock() + rt.engine = engine + rt.clusterNode = clusterNode + rt.grainTypes = grainTypes + rt.nodeAddr = rt.config.NodeAddr if clusterNode != nil { - rt.clusterNode = clusterNode rt.clusterDone = make(chan struct{}) - rt.nodeAddr = config.NodeAddr rt.clusterView.Store(&initialView) + } + rt.state = rootRunning + rt.stopCode = "" + if clusterNode != nil { go rt.watchCluster() } - if config.ReminderStore != nil && config.ReminderInterval > 0 { - rt.poller = timer.New(config.ReminderStore, config.Clock, config.ReminderInterval, reminderInvoker{runtime: rt}, rt.newReminderCall) + if rt.config.ReminderInterval > 0 { + rt.poller = timer.New(timer.Config{ + Table: rt.reminderStore, + Clock: rt.clock, + Interval: rt.config.ReminderInterval, + PageSize: rt.config.ReminderPageSize, + Workers: rt.config.ReminderWorkers, + Invoker: reminderInvoker{runtime: rt}, + NewCall: rt.resolveReminderCall, + OnFailure: rt.reportReminderFailure, + }) } - if rt.clusterNode != nil && rt.transport != nil { + if clusterNode != nil { rt.startTransport() } - return rt, nil + rt.lifecycleMu.Unlock() + return nil +} + +func (rt *Runtime) validateSetup() ([]*typeRegistration, map[GrainType]*typeRegistration, error) { + config := rt.config + switch { + case isNil(config.Clock): + return nil, nil, invalidSetup("Clock is required") + case isNil(config.Store): + return nil, nil, invalidSetup("State Store is required") + case isNil(config.ReminderStore): + return nil, nil, invalidSetup("Reminder Store is required") + case config.MailboxCapacity <= 0: + return nil, nil, invalidSetup("mailbox capacity must be positive") + case config.MaxActivations <= 0: + return nil, nil, invalidSetup("activation limit must be positive") + case config.IdleTimeout < 0: + return nil, nil, invalidSetup("idle timeout must not be negative") + case config.EvictionInterval < 0: + return nil, nil, invalidSetup("eviction interval must not be negative") + case config.ReminderInterval < 0: + return nil, nil, invalidSetup("Reminder interval must not be negative") + case config.ReminderPageSize <= 0: + return nil, nil, invalidSetup("Reminder page size must be positive") + case config.ReminderWorkers <= 0: + return nil, nil, invalidSetup("Reminder worker limit must be positive") + case isNil(config.MemberStore) != isNil(config.Transport): + return nil, nil, invalidSetup("Member Store and Transport must be configured together") + } + if !isNil(config.MemberStore) { + if config.HeartbeatInterval <= 0 || config.ViewInterval <= 0 { + return nil, nil, invalidSetup("cluster heartbeat and view intervals must be positive") + } + if config.ProbeInterval < 0 || config.ProbeTimeout < 0 || config.ProbeFailures < 0 || config.VoteTTL < 0 || config.MaxTickGap < 0 || config.MaxTableLatency < 0 { + return nil, nil, invalidSetup("cluster limits must not be negative") + } + } + + registrations := make([]*typeRegistration, 0, len(rt.types)) + for _, registration := range rt.types { + registrations = append(registrations, registration) + } + sort.Slice(registrations, func(i, j int) bool { + return localTypeName(registrations[i].localType) < localTypeName(registrations[j].localType) + }) + grainTypes := make(map[GrainType]*typeRegistration, len(registrations)) + for _, registration := range registrations { + if !validGrainType(registration.grainType) { + return nil, nil, invalidSetup(fmt.Sprintf("GrainType %q for %s is invalid", registration.grainType, localTypeName(registration.localType))) + } + if registration.factory == nil { + return nil, nil, invalidSetup(fmt.Sprintf("Grain type %s has no factory", localTypeName(registration.localType))) + } + if registration.dispatch == nil || registration.newProxy == nil || registration.newCall == nil || registration.newReminderCall == nil { + return nil, nil, invalidSetup(fmt.Sprintf("Grain type %s has incomplete generated bindings", localTypeName(registration.localType))) + } + if previous, exists := grainTypes[registration.grainType]; exists { + return nil, nil, invalidSetup(fmt.Sprintf("GrainType %q is used by %s and %s", registration.grainType, localTypeName(previous.localType), localTypeName(registration.localType))) + } + grainTypes[registration.grainType] = registration + } + return registrations, grainTypes, nil +} + +func (rt *Runtime) failStart() { + rt.closeTransport(false) + rt.lifecycleMu.Lock() + rt.state = rootStartFailed + rt.stopCode = ErrRuntimeNotStarted + rt.signalStopping() + rt.signalDone() + rt.lifecycleMu.Unlock() +} + +func setupCheckError(name string, err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + return withCode(ErrInvalidSetup, fmt.Errorf("check %s: %w", name, err)) +} + +func invalidSetup(message string) error { + return withCode(ErrInvalidSetup, errors.New(message)) +} + +func isNil(value any) bool { + if value == nil { + return true + } + reflected := reflect.ValueOf(value) + switch reflected.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + return reflected.IsNil() + default: + return false + } +} + +func validGrainType(value GrainType) bool { + if len(value) == 0 || len(value) > 255 { + return false + } + for index := range len(value) { + character := value[index] + alphanumeric := character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' + if alphanumeric { + continue + } + if index == 0 || index == len(value)-1 || character != '.' && character != '/' && character != '_' && character != '-' { + return false + } + } + return true +} + +func localTypeName(value reflect.Type) string { + if value.PkgPath() == "" { + return value.String() + } + return value.PkgPath() + "." + value.Name() } // WithClock sets the clock used by runtime and cluster timers. If omitted, New @@ -304,16 +578,25 @@ func WithClock(value clock.Clock) Option { } } -// WithMailboxCapacity sets the number of calls that may wait in one entity's -// mailbox. If omitted, New allows 16 queued calls per entity; calls that cannot -// be queued are rejected. The value must not be negative; New panics when it -// creates a mailbox with a negative capacity. +// WithMailboxCapacity sets the number of Calls that may wait in one Grain's +// mailbox. If omitted, the Runtime allows 16 queued Calls per Grain. A Call +// that cannot enter the mailbox is rejected. Start requires a positive value. func WithMailboxCapacity(value int) Option { return func(config *Config) { config.MailboxCapacity = value } } +// WithMaxActivations sets the maximum number of starting, active, or +// deactivating Activations in one Silo. If omitted, the Runtime allows 10,000. +// Start requires a positive value. A full limit rejects a Call before its +// factory runs. +func WithMaxActivations(value int) Option { + return func(config *Config) { + config.MaxActivations = value + } +} + // WithIdleTimeout sets how long an unused activation may remain before idle // eviction. If omitted, New uses one minute. A non-positive value disables // idle eviction. @@ -331,18 +614,16 @@ func WithEvictionInterval(value time.Duration) Option { } } -// WithStore sets the store used for Grain State. If omitted, New uses an -// in-memory store; when the selected store also implements ReminderStore and -// no Reminder store is supplied, New uses it for Reminders too. +// WithStore sets the Store used for Grain State. The Application must set it. +// Start rejects a Runtime that has no State Store. func WithStore(value store.Store) Option { return func(config *Config) { config.Store = value } } -// WithReminderStore sets the store used for Reminders. If omitted, New -// derives it from Store when Store implements ReminderStore; otherwise Reminder -// operations return ErrReminderStoreUnavailable. +// WithReminderStore sets the Store used for Reminders. The Application must +// set it. Start rejects a Runtime that has no Reminder Store. func WithReminderStore(value store.ReminderStore) Option { return func(config *Config) { config.ReminderStore = value @@ -350,14 +631,31 @@ func WithReminderStore(value store.ReminderStore) Option { } // WithReminderInterval sets the interval for background Reminder polling. If -// omitted, New polls once per second. A non-positive value keeps Reminders -// persisted but disables automatic polling. +// omitted, the Runtime polls once per second after Start. Zero disables +// automatic polling. Start rejects a negative value. func WithReminderInterval(value time.Duration) Option { return func(config *Config) { config.ReminderInterval = value } } +// WithReminderPageSize sets the maximum due rows returned by one Reminder +// Store scan page. If omitted, the Runtime uses 256. Start requires a positive +// value. +func WithReminderPageSize(value int) Option { + return func(config *Config) { + config.ReminderPageSize = value + } +} + +// WithReminderWorkers sets the maximum concurrent Reminder Claim and Call +// workers. If omitted, the Runtime uses 16. Start requires a positive value. +func WithReminderWorkers(value int) Option { + return func(config *Config) { + config.ReminderWorkers = value + } +} + // WithTransport sets the transport used by a clustered runtime for serving and // forwarding calls. If omitted, no transport is started; a transport must be // configured together with a MemberStore. @@ -367,14 +665,16 @@ func WithTransport(value transport.Transport) Option { } } -// OnError sets the callback for failures of background application callbacks: -// claimed Reminder invocations and normal OnDeactivate hooks. If omitted, -// those errors are not reported. The callback may run asynchronously and -// concurrently with application code; it is not called for ordinary foreground -// Invoke errors. A Reminder invocation whose delivery is canceled because the -// poller's context was canceled during shutdown is not reported. ListDue and -// Claim failures are not reported either. Event sources are sealed: branch on -// the concrete type of Source, never on method-name strings. +// OnError sets the callback for Grain Timer failures, Reminder scan, dispatch, +// terminal, claim, and Call failures, OnDeactivate hook errors, and State +// write failures during OnDeactivate. If omitted, these errors are not +// reported. The callback may run asynchronously and concurrently with +// Application code. It is not called for ordinary foreground Call errors. +// Runtime cancellation during shutdown and a lost Reminder Claim CAS are not +// reported. +// The callback must return promptly and must not do blocking I/O. +// Event sources are sealed: branch on the concrete Source type. The Runtime +// catches a panic from this callback and does not report that panic again. func OnError(f func(BackgroundError)) Option { return func(config *Config) { config.OnError = f @@ -388,149 +688,163 @@ func OnError(f func(BackgroundError)) Option { // The callback runs synchronously on the invoking goroutine and may be called // concurrently for different calls. A forwarded call produces one observation // on the initiating Runtime; the target Runtime does not produce a second -// observation, and the duration includes forwarding. +// observation, and the duration includes forwarding. The Runtime catches a +// panic from this callback. The panic does not change the Call result. func OnCall(f func(CallObservation)) Option { return func(config *Config) { config.OnCall = f } } -// WithMemberStore sets the membership store used to enable clustering. If -// omitted, the runtime operates without cluster membership; a MemberStore must -// be configured together with a Transport. +// WithMemberStore sets the membership store for the Cluster preview. If it is +// omitted, the Runtime runs as one Silo. A MemberStore requires a Transport. func WithMemberStore(value store.MemberStore) Option { return func(config *Config) { config.MemberStore = value } } -// WithNodeAddr sets this node's address in cluster membership and ownership -// decisions. If omitted, the address is the empty string; the option has no -// effect when clustering is disabled. +// WithNodeAddr sets this Silo's address for the Cluster preview. If omitted, +// the address is empty. The option has no effect for a Single Silo. func WithNodeAddr(value string) Option { return func(config *Config) { config.NodeAddr = value } } -// WithGeneration sets this node's membership generation. If omitted, the -// generation is the empty string; the option has no effect when clustering is -// disabled. +// WithGeneration sets this Silo's membership generation. If omitted, the +// generation is empty. The option has no effect for a Single Silo. func WithGeneration(value string) Option { return func(config *Config) { config.Generation = value } } -// WithHeartbeatInterval sets the cluster heartbeat interval. If omitted, New +// WithHeartbeatInterval sets the Cluster heartbeat interval. If omitted, New // uses one second; the option has no effect when clustering is disabled. When -// clustering is enabled, the value must be positive; a non-positive value makes -// New panic while creating the cluster ticker. +// clustering is enabled, Start rejects a non-positive value. func WithHeartbeatInterval(value time.Duration) Option { return func(config *Config) { config.HeartbeatInterval = value } } -// WithViewInterval sets how often the cluster membership view is refreshed. If +// WithViewInterval sets how often the Cluster membership view is refreshed. If // omitted, New uses one second; the option has no effect when clustering is -// disabled. When clustering is enabled, the value must be positive; a -// non-positive value makes New panic while creating the cluster ticker. +// disabled. When clustering is enabled, Start rejects a non-positive value. func WithViewInterval(value time.Duration) Option { return func(config *Config) { config.ViewInterval = value } } -// WithProbeInterval sets the cluster probe interval. If omitted, New uses one -// second. A negative value makes a clustered New return an error matching -// cluster.ErrInvalidConfig. It has no effect when clustering is disabled. +// WithProbeInterval sets the Cluster probe interval. If omitted, Start uses +// one second. Start rejects a negative value when clustering is enabled. func WithProbeInterval(value time.Duration) Option { return func(config *Config) { config.ProbeInterval = value } } -// WithProbeTimeout sets the deadline for a cluster probe. If omitted, New uses -// 500 ms. A negative value makes a clustered New return an error matching -// cluster.ErrInvalidConfig. It has no effect when clustering is disabled. +// WithProbeTimeout sets the deadline for a Cluster probe. If omitted, Start +// uses 500 ms. Start rejects a negative value when clustering is enabled. func WithProbeTimeout(value time.Duration) Option { return func(config *Config) { config.ProbeTimeout = value } } -// WithProbeFailures sets the number of failed probes required before a member -// is considered for a death vote. If omitted, New uses three failures. A -// negative value makes a clustered New return an error matching -// cluster.ErrInvalidConfig. It has no effect when clustering is disabled. +// WithProbeFailures sets the number of failed probes required before a Silo is +// considered for a death vote. If omitted, Start uses three failures. Start +// rejects a negative value when clustering is enabled. func WithProbeFailures(value int) Option { return func(config *Config) { config.ProbeFailures = value } } -// WithVoteTTL sets how long a cluster suspect vote remains valid. If omitted, -// New uses six seconds. A negative value makes a clustered New return an error -// matching cluster.ErrInvalidConfig. It has no effect when clustering is -// disabled. +// WithVoteTTL sets how long a Cluster suspect vote remains valid. If omitted, +// Start uses six seconds. Start rejects a negative value when clustering is +// enabled. func WithVoteTTL(value time.Duration) Option { return func(config *Config) { config.VoteTTL = value } } -// WithMaxTickGap sets the maximum allowed gap between healthy cluster ticks. If -// omitted, New uses two seconds. A negative value makes a clustered New return -// an error matching cluster.ErrInvalidConfig. It has no effect when clustering -// is disabled. +// WithMaxTickGap sets the maximum allowed gap between healthy Cluster ticks. +// If omitted, Start uses two seconds. Start rejects a negative value when +// clustering is enabled. func WithMaxTickGap(value time.Duration) Option { return func(config *Config) { config.MaxTickGap = value } } -// WithMaxTableLatency sets the maximum acceptable membership-store latency. If -// omitted, New uses 500 ms. A negative value makes a clustered New return an -// error matching cluster.ErrInvalidConfig. It has no effect when clustering is -// disabled. +// WithMaxTableLatency sets the maximum acceptable membership Store latency. +// If omitted, Start uses 500 ms. Start rejects a negative value when +// clustering is enabled. func WithMaxTableLatency(value time.Duration) Option { return func(config *Config) { config.MaxTableLatency = value } } -// Invoke calls method for id, passing args and reply to the registered entity -// dispatch. Calls for the same identity are serialized; calls for different -// identities may run concurrently. +// Invoke calls method for id, passing args and reply to the registered Grain +// dispatch. Calls for the same GrainId are serialized. Calls for different +// GrainIds may run concurrently. // // For a local call, ctx limits waiting for activation and delivery and is -// passed to the entity method. For a remote owner, ctx limits the forwarding +// passed to the Grain method. For a remote owner, ctx limits the forwarding // operation at the initiating Runtime; canceling it does not cancel the -// already forwarded entity call, which may continue on the remote Runtime. +// already forwarded Grain Call, which may continue on the remote Runtime. // An identity with no current owner returns an error matching ErrNoOwner // without being forwarded. A forwarded error with a Code is reconstructed so // errors.Is can match that Code; errors from an opaque error retain only text. // Caller cancellation and deadline errors are returned unchanged. // Once the Runtime has begun stopping, new calls are rejected at the root // admission gate with a stable stop error before ownership is decided or a -// call is forwarded: gor.runtime_closed after Close or Kill, and gor.node_dead -// once the cluster has declared this node dead. +// Call is forwarded: gor.runtime_closed after Shutdown, and gor.node_dead +// once the Cluster has declared this Silo dead. func (rt *Runtime) Invoke(ctx context.Context, id GrainId, method string, args any, reply any) error { if rt.onCall == nil { return publicError(rt.invoke(ctx, id, method, args, reply)) } - started := rt.clock.Now() + var started time.Time + if !isNil(rt.clock) { + started = rt.clock.Now() + } err := publicError(rt.invoke(ctx, id, method, args, reply)) - rt.onCall(CallObservation{ + var duration time.Duration + if !isNil(rt.clock) { + duration = rt.clock.Now().Sub(started) + } + rt.reportCall(CallObservation{ GrainType: id.GrainType, Method: method, - Duration: rt.clock.Now().Sub(started), + Duration: duration, Err: err, }) return err } +func (rt *Runtime) reportCall(observation CallObservation) { + defer func() { + recover() + }() + rt.onCall(observation) +} + +func (rt *Runtime) reportBackgroundError(event BackgroundError) { + if rt.onError == nil { + return + } + defer func() { + recover() + }() + rt.onError(event) +} + func (rt *Runtime) invoke(ctx context.Context, id GrainId, method string, args any, reply any) error { release, err := rt.admit() if err != nil { @@ -540,25 +854,39 @@ func (rt *Runtime) invoke(ctx context.Context, id GrainId, method string, args a if rt.clusterNode == nil { return rt.invokeLocal(ctx, id, method, args, reply) } - view := rt.clusterView.Load() - owner, ok := cluster.Owner(*view, store.GrainId(id)) - if !ok { - return fmt.Errorf("%w: identity currently has no active owner", ErrNoOwner) - } - if owner != rt.nodeAddr { - return rt.forward(ctx, owner, id, method, args, reply) + for { + if err := ctx.Err(); err != nil { + return err + } + view := rt.clusterView.Load() + owner, ok := cluster.Owner(*view, toStoreGrainID(id)) + if !ok { + return fmt.Errorf("%w: identity currently has no active owner", ErrNoOwner) + } + if owner != rt.nodeAddr { + return rt.forward(ctx, owner, id, method, args, reply) + } + outcome := rt.engine.InvokeOutcome(ctx, toRuntimeGrainID(id), method, args, reply) + if !outcome.OwnershipLost { + return outcome.Err + } } - return rt.invokeLocal(ctx, id, method, args, reply) } func (rt *Runtime) invokeLocal(ctx context.Context, id GrainId, method string, args any, reply any) error { - return rt.engine.Invoke(ctx, id, method, args, reply) + return rt.engine.Invoke(ctx, toRuntimeGrainID(id), method, args, reply) } // Owns reports whether this runtime currently owns id. It is an integration // seam for scheduling and cluster plumbing; application code should normally // invoke a reference or Runtime.Invoke and let routing choose the owner. func (rt *Runtime) Owns(id store.GrainId) bool { + rt.lifecycleMu.Lock() + running := rt.state == rootRunning + rt.lifecycleMu.Unlock() + if !running { + return false + } if rt.clusterNode == nil { return true } @@ -568,28 +896,45 @@ func (rt *Runtime) Owns(id store.GrainId) bool { } type boundInstance struct { - entity any - binder *Binder -} - -// InstallType installs the dispatch, proxy, normal-call, and Reminder-call -// factories required for T in rt. Generated Install code calls it; application -// code should use the generated installer rather than hand-writing this seam. -// It returns an error when T is already installed in rt. -func InstallType[T any](rt *Runtime, dispatch func(context.Context, T, string, any, any) error, newProxy func(Invoker, GrainId) T, newCall func(string) (any, any), newReminderCall func(string, TickStatus) (any, any)) error { - name := TypeName[T]() - rt.typesMu.Lock() - defer rt.typesMu.Unlock() - if _, exists := rt.types[name]; exists { - return fmt.Errorf("entity type %q is already installed", name) - } - rt.types[name] = typeRegistration{ - dispatch: func(ctx context.Context, instance any, method string, args any, reply any) error { + value any + context *GrainContext +} + +// InstallType installs dispatch and Grain Reference factories for T in rt. +// Generated Install code calls it. Application code must use the generated +// installer instead of writing this boundary. +// It returns an error when the generated code version does not match the +// Runtime or when T is already installed in rt. +func InstallType[T any](rt *Runtime, generatedVersion int, grainType GrainType, dispatch func(context.Context, T, string, any, any) error, newProxy func(Invoker, GrainId) T, newCall func(string) (any, any), newReminderCall func(string, TickStatus) (any, any)) error { + if generatedVersion != GeneratedCodeVersion { + return withCode(ErrInvalidSetup, fmt.Errorf("generated code version %d does not match Runtime version %d; run go generate", generatedVersion, GeneratedCodeVersion)) + } + localType := reflect.TypeFor[T]() + rt.lifecycleMu.Lock() + defer rt.lifecycleMu.Unlock() + if rt.state != rootConstructed { + return withCode(ErrSetupFrozen, errors.New("runtime setup is frozen")) + } + if _, exists := rt.types[localType]; exists { + return withCode(ErrInvalidSetup, fmt.Errorf("grain type %s is already installed", localTypeName(localType))) + } + var runtimeDispatch runtimepkg.Dispatch + if dispatch != nil { + runtimeDispatch = func(ctx context.Context, instance any, method string, args any, reply any) error { return dispatch(ctx, instance.(T), method, args, reply) - }, - newProxy: func(invoker Invoker, id GrainId) any { + } + } + var proxyFactory func(Invoker, GrainId) any + if newProxy != nil { + proxyFactory = func(invoker Invoker, id GrainId) any { return newProxy(invoker, id) - }, + } + } + rt.types[localType] = &typeRegistration{ + localType: localType, + grainType: grainType, + dispatch: runtimeDispatch, + newProxy: proxyFactory, newCall: newCall, newReminderCall: newReminderCall, } @@ -601,116 +946,227 @@ func InstallType[T any](rt *Runtime, dispatch func(context.Context, T, string, a // ErrTypeNotInstalled. Register rejects a second registration of the same // type in one runtime. // -// factory is called to create each activation and receives a Binder for that -// activation's identity. Register itself does not create an activation. -func Register[T any](rt *Runtime, factory func(*Binder) T) error { - name := TypeName[T]() - registration, ok := rt.typeRegistration(name) +// factory creates each Activation and receives its GrainContext. Register does +// not create an Activation. +func Register[T any](rt *Runtime, factory func(*GrainContext) T) error { + localType := reflect.TypeFor[T]() + rt.lifecycleMu.Lock() + defer rt.lifecycleMu.Unlock() + if rt.state != rootConstructed { + return withCode(ErrSetupFrozen, errors.New("runtime setup is frozen")) + } + registration, ok := rt.types[localType] if !ok { - return fmt.Errorf("%w: %s", ErrTypeNotInstalled, name) + return fmt.Errorf("%w: %s", ErrTypeNotInstalled, localTypeName(localType)) } - return rt.engine.Register(name, runtimepkg.Registration{ + if registration.factory != nil { + return withCode(ErrInvalidSetup, fmt.Errorf("grain type %s already has a factory", localTypeName(localType))) + } + if factory == nil { + return withCode(ErrInvalidSetup, fmt.Errorf("grain type %s has a nil factory", localTypeName(localType))) + } + registration.factory = func(grain *GrainContext) any { + return factory(grain) + } + return nil +} + +func (rt *Runtime) runtimeRegistration(registration *typeRegistration) runtimepkg.Registration { + return runtimepkg.Registration{ Factory: func(ctx context.Context, id runtimepkg.GrainId) (any, error) { - binder := newBinder(rt, id) - entity := factory(binder) - if err := binder.load(ctx); err != nil { + grain := newGrainContext(rt, fromRuntimeGrainID(id)) + grain.deactivateOnIdle = runtimepkg.DeactivateOnIdleFrom(ctx) + registerTimer := runtimepkg.GrainTimerRegistrarFrom(ctx) + grain.registerGrainTimer = func(callback func(context.Context) error, options GrainTimerOptions) (GrainTimer, error) { + return registerTimer(callback, runtimepkg.GrainTimerOptions{ + DueTime: options.DueTime, + Period: options.Period, + KeepAlive: options.KeepAlive, + }) + } + value := registration.factory(grain) + grain.freezeStateDeclarations() + if err := grain.load(ctx); err != nil { return nil, err } - if activatable, ok := any(entity).(Activatable); ok { - if err := activatable.OnActivate(ctx); err != nil { - return nil, err - } + var activationErr error + if activatable, ok := any(value).(Activatable); ok { + activationErr = activatable.OnActivate(ctx) + } + if result, discarded := grain.resultWithDiscard(activationErr); discarded || result != nil { + return nil, result } - return boundInstance{entity: entity, binder: binder}, nil + return boundInstance{value: value, context: grain}, nil + }, + ActivationContext: func(lifecycleCtx context.Context, callCtx context.Context) context.Context { + return withRequestContextSnapshot(lifecycleCtx, requestContextSnapshotFrom(callCtx)) }, Dispatch: func(ctx context.Context, instance any, method string, args any, reply any) error { bound := instance.(boundInstance) - err := registration.dispatch(ctx, bound.entity, method, args, reply) - if discard := bound.binder.discardError(); discard != nil { - return runtimepkg.Discard{Err: errors.Join(err, discard)} + err := registration.dispatch(ctx, bound.value, method, args, reply) + if result, discarded := bound.context.resultWithDiscard(err); discarded { + return runtimepkg.Discard{Err: result} } return err }, OnDeactivate: func(ctx context.Context, id runtimepkg.GrainId, reason runtimepkg.DeactivationReason, instance any) { bound := instance.(boundInstance) - deactivatable, ok := bound.entity.(Deactivatable) + deactivatable, ok := bound.value.(Deactivatable) if !ok { return } - err := deactivatable.OnDeactivate(ctx, reason) - if err != nil && rt.onError != nil { - rt.onError(BackgroundError{ - GrainId: GrainId(id), + stateFailureSnapshot := bound.context.stateFailureSnapshot() + publicReason := DeactivationReason(reason) + err := callOnDeactivate(deactivatable, ctx, publicReason) + if stateErr := bound.context.stateFailureAfter(stateFailureSnapshot); stateErr != nil { + if err == nil { + err = stateErr + } else if !errors.Is(err, stateErr) { + err = errors.Join(err, stateErr) + } + } + if err != nil { + rt.reportBackgroundError(BackgroundError{ + GrainId: fromRuntimeGrainID(id), Err: err, - Source: Deactivation{Reason: reason}, + Source: Deactivation{Reason: publicReason}, }) } }, - }) + OnGrainTimerError: func(id runtimepkg.GrainId, err error) { + rt.reportBackgroundError(BackgroundError{ + GrainId: fromRuntimeGrainID(id), + Err: publicError(err), + Source: GrainTimerInvocation{}, + }) + }, + } } -// Now returns the current time from the clock configured for the entity bound -// to b. -func Now(b *Binder) time.Time { +func callOnDeactivate(grain Deactivatable, ctx context.Context, reason DeactivationReason) (err error) { + defer func() { + if value := recover(); value != nil { + err = withCode(ErrPanic, fmt.Errorf("OnDeactivate panicked: %v", value)) + } + }() + return grain.OnDeactivate(ctx, reason) +} + +// Now returns the current time from the Clock for the Grain bound to b. +func Now(b *GrainContext) time.Time { return b.runtime.clock.Now() } -// Ref returns a typed reference to entity T identified by key. The type must +// Ref returns a typed Grain Reference to T with the specified GrainKey. T must // already be installed in the runtime represented by scope; otherwise Ref // panics with an ErrTypeNotInstalled message. Creating a reference does not -// activate the entity; activation begins when a method is invoked on it. +// start the Grain. Activation starts when a Call is made through the reference. func Ref[T any](scope Scope, key string) T { rt := scope.scopeRuntime() - name := TypeName[T]() - registration, ok := rt.typeRegistration(name) + localType := reflect.TypeFor[T]() + registration, ok := rt.typeRegistrationByLocalType(localType) + if !ok { + panic(fmt.Sprintf("%v: %s", ErrTypeNotInstalled, localTypeName(localType))) + } + return registration.newProxy(rt, GrainId{GrainType: registration.grainType, GrainKey: key}).(T) +} + +// GrainTypeOf returns the installed stable GrainType for T. It panics when T +// is not installed in the supplied scope. +func GrainTypeOf[T any](scope Scope) GrainType { + rt := scope.scopeRuntime() + localType := reflect.TypeFor[T]() + registration, ok := rt.typeRegistrationByLocalType(localType) if !ok { - panic(fmt.Sprintf("%v: %s", ErrTypeNotInstalled, name)) + panic(fmt.Sprintf("%v: %s", ErrTypeNotInstalled, localTypeName(localType))) } - return registration.newProxy(rt, GrainId{GrainType: name, GrainKey: key}).(T) + return registration.grainType +} + +func (rt *Runtime) typeRegistrationByLocalType(localType reflect.Type) (*typeRegistration, bool) { + rt.lifecycleMu.Lock() + defer rt.lifecycleMu.Unlock() + registration, ok := rt.types[localType] + return registration, ok } -func (rt *Runtime) typeRegistration(name string) (typeRegistration, bool) { - rt.typesMu.Lock() - defer rt.typesMu.Unlock() - registration, ok := rt.types[name] +func (rt *Runtime) typeRegistrationByGrainType(grainType GrainType) (*typeRegistration, bool) { + rt.lifecycleMu.Lock() + defer rt.lifecycleMu.Unlock() + registration, ok := rt.grainTypes[grainType] return registration, ok } -func (rt *Runtime) newReminderCall(id store.GrainId, method string, firstTickTime time.Time, period time.Duration, currentTickTime time.Time) (any, any) { - registration, ok := rt.typeRegistration(id.GrainType) +func (rt *Runtime) resolveReminderCall(id store.GrainId, method string) (timer.ReminderCallBuilder, error) { + registration, ok := rt.typeRegistrationByGrainType(GrainType(id.GrainType)) if !ok || registration.newReminderCall == nil { - return nil, nil + return nil, withCode(ErrTypeNotInstalled, fmt.Errorf("GrainType %q is not installed", id.GrainType)) } - return registration.newReminderCall(method, TickStatus{ - FirstTickTime: firstTickTime, - Period: period, - CurrentTickTime: currentTickTime, - }) + args, reply := registration.newReminderCall(method, TickStatus{}) + if args == nil || reply == nil { + return nil, withCode(ErrUnknownMethod, fmt.Errorf("unknown Reminder method %q for GrainType %q", method, id.GrainType)) + } + return func(reminderName string, firstTickTime time.Time, period time.Duration, currentTickTime time.Time) (any, any) { + return registration.newReminderCall(method, TickStatus{ + ReminderName: reminderName, + FirstTickTime: firstTickTime, + Period: period, + CurrentTickTime: currentTickTime, + }) + }, nil } -// Close begins an orderly shutdown. It stops admitting new entity calls, -// lets calls already admitted finish, rejects queued calls without entering -// their method bodies, and then waits for in-flight methods, normal -// deactivation callbacks, and the runtime's own infrastructure goroutines -// before closing configured cluster and transport resources. -// -// Scheduled delivery, direct Invoke, and inbound forwarded invokes all pass -// through the same admission gate, so all three are rejected once Close has -// begun. Repeated Close or Kill calls are safe and do not start another -// shutdown; a Kill during Close escalates to immediate shutdown semantics. -func (rt *Runtime) Close() { - rt.beginClose() - rt.closeGracefully() -} +// Shutdown stops Call admission and waits for Runtime infrastructure. When +// ctx ends first, Shutdown escalates to an abrupt stop and returns ctx.Err(). +func (rt *Runtime) Shutdown(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + for { + rt.lifecycleMu.Lock() + if rt.state != rootStarting { + rt.lifecycleMu.Unlock() + break + } + cancel := rt.startCancel + startDone := rt.startDone + rt.lifecycleMu.Unlock() + if cancel != nil { + cancel() + } + select { + case <-startDone: + continue + case <-ctx.Done(): + return ctx.Err() + } + } -// Kill begins an immediate shutdown. It stops admitting new entity calls, -// cancels the contexts of calls already running, rejects queued work, and -// skips deactivation callbacks. Unlike Close, Kill does not wait for user -// methods to return: Go cannot forcibly stop code that ignores cancellation. -// It is safe to call repeatedly; a Kill during Close escalates immediately. -func (rt *Runtime) Kill() { - rt.beginKill() - rt.closeImmediately() + if rt.beginClose() { + if err := ctx.Err(); err != nil { + if rt.beginKill() { + go rt.closeImmediately() + } + <-rt.done + return err + } + go rt.closeGracefully() + } + select { + case <-rt.done: + return nil + default: + } + select { + case <-rt.done: + return nil + case <-ctx.Done(): + if rt.beginKill() { + go rt.closeImmediately() + } + <-rt.done + return ctx.Err() + } } func (rt *Runtime) closeGracefully() { @@ -721,9 +1177,11 @@ func (rt *Runtime) closeGracefully() { rt.clusterNode.Close() <-rt.clusterDone } - rt.engine.BeginClose() - <-rt.engine.Done() - rt.waitDrained() + if rt.engine != nil { + rt.engine.BeginClose() + <-rt.engine.Done() + } + rt.waitDrainedOrAbrupt() rt.closeTransport(true) rt.finishStop() } @@ -736,21 +1194,33 @@ func (rt *Runtime) closeImmediately() { rt.clusterNode.Kill() <-rt.clusterDone } - rt.engine.BeginKill() - <-rt.engine.Done() + if rt.engine != nil { + rt.engine.BeginKill() + <-rt.engine.Done() + } rt.closeTransport(false) rt.finishStop() } -// Activations returns a sorted snapshot of this runtime's active entities. +// Activations returns a sorted snapshot of this Runtime's active Grains. func (rt *Runtime) Activations() []Activation { - return rt.engine.Activations() + if rt.engine == nil { + return nil + } + internal := rt.engine.Activations() + activations := make([]Activation, len(internal)) + for index, activation := range internal { + activations[index] = Activation{GrainId: fromRuntimeGrainID(activation.GrainId), Queued: activation.Queued} + } + return activations +} + +// Stopping returns a channel that closes when Call admission ends. +func (rt *Runtime) Stopping() <-chan struct{} { + return rt.stopping } -// Done returns a channel that is closed when shutdown begins and the runtime -// stops accepting forwarded requests. It may close before Close or Kill has -// finished waiting for invocations, deactivation callbacks, or resources. -// It also closes when a clustered runtime's node is declared dead. +// Done returns a channel that closes after Runtime infrastructure ends. func (rt *Runtime) Done() <-chan struct{} { return rt.done } @@ -762,10 +1232,13 @@ func (rt *Runtime) Done() <-chan struct{} { type rootState uint8 const ( - rootRunning rootState = iota + rootConstructed rootState = iota + rootStarting + rootRunning rootClosing rootKilling rootDead + rootStartFailed rootStopped ) @@ -775,11 +1248,25 @@ const ( func (rt *Runtime) beginClose() bool { rt.lifecycleMu.Lock() defer rt.lifecycleMu.Unlock() - if rt.state != rootRunning { + switch rt.state { + case rootConstructed: + rt.state = rootClosing + rt.stopCode = ErrRuntimeClosed + rt.drained = make(chan struct{}) + close(rt.drained) + rt.signalStopping() + return true + case rootRunning: + rt.leaveRunning(rootClosing, ErrRuntimeClosed) + return true + case rootStartFailed: + rt.state = rootStopped + rt.signalStopping() + rt.signalDone() + return false + default: return false } - rt.leaveRunning(rootClosing, ErrRuntimeClosed) - return true } // beginKill transitions running or closing to killing. From running it is the @@ -790,9 +1277,11 @@ func (rt *Runtime) beginKill() bool { switch rt.state { case rootRunning: rt.leaveRunning(rootKilling, ErrRuntimeClosed) + rt.signalAbrupt() return true case rootClosing: rt.state = rootKilling + rt.signalAbrupt() return true default: return false @@ -809,6 +1298,7 @@ func (rt *Runtime) becomeDead() bool { return false } rt.leaveRunning(rootDead, ErrNodeDead) + rt.signalAbrupt() return true } @@ -820,6 +1310,7 @@ func (rt *Runtime) finishStop() { switch rt.state { case rootClosing, rootKilling, rootDead: rt.state = rootStopped + rt.signalDone() } } @@ -834,11 +1325,11 @@ func (rt *Runtime) leaveRunning(next rootState, code Code) { if rt.inflight == 0 { close(rt.drained) } - close(rt.done) + rt.signalStopping() } -// admit registers one entity call against the root lifecycle. It returns a -// release function when the call may proceed, or a stop error otherwise. It is +// admit registers one Grain Call against the root lifecycle. It returns a +// release function when the Call may proceed, or a stop error otherwise. It is // the single admission gate shared by public Invoke, the inbound invoke // handler, and scheduled delivery; the ownership decision and forwarding happen // only after a successful admit. @@ -849,8 +1340,12 @@ func (rt *Runtime) admit() (func(), error) { rt.lifecycleMu.Unlock() return rt.release, nil } + state := rt.state code := rt.stopCode rt.lifecycleMu.Unlock() + if state == rootConstructed || state == rootStarting || state == rootStartFailed { + return nil, withCode(ErrRuntimeNotStarted, errors.New("runtime is not started")) + } return nil, stopRejection(code) } @@ -864,17 +1359,32 @@ func (rt *Runtime) release() { rt.lifecycleMu.Unlock() } -// waitDrained blocks until every call admitted before the stop transition has -// released. Calls admitted after the transition are impossible. -func (rt *Runtime) waitDrained() { +// waitDrainedOrAbrupt waits for admitted Calls during graceful shutdown. An +// abrupt escalation stops this wait because user code can ignore cancellation. +func (rt *Runtime) waitDrainedOrAbrupt() { rt.lifecycleMu.Lock() drained := rt.drained rt.lifecycleMu.Unlock() if drained != nil { - <-drained + select { + case <-drained: + case <-rt.abrupt: + } } } +func (rt *Runtime) signalStopping() { + rt.stoppingOnce.Do(func() { close(rt.stopping) }) +} + +func (rt *Runtime) signalDone() { + rt.doneOnce.Do(func() { close(rt.done) }) +} + +func (rt *Runtime) signalAbrupt() { + rt.abruptOnce.Do(func() { close(rt.abrupt) }) +} + func (rt *Runtime) stopCodeSnapshot() Code { rt.lifecycleMu.Lock() defer rt.lifecycleMu.Unlock() @@ -883,7 +1393,10 @@ func (rt *Runtime) stopCodeSnapshot() Code { func stopRejection(code Code) error { if code == ErrNodeDead { - return withCode(ErrNodeDead, errors.New("node declared dead")) + return withCode(ErrNodeDead, errors.New("silo declared dead")) + } + if code == ErrRuntimeNotStarted { + return withCode(ErrRuntimeNotStarted, errors.New("runtime is not started")) } return withCode(ErrRuntimeClosed, errors.New("runtime is not accepting calls")) } @@ -905,7 +1418,7 @@ func (rt *Runtime) startTransport() { // subcomponent that received the graceful stop command must not treat the // later Kill as a no-op. func (rt *Runtime) closeTransport(graceful bool) { - if rt.transport == nil { + if isNil(rt.transport) { return } if rt.transportClosing.CompareAndSwap(false, true) { @@ -953,7 +1466,7 @@ func (rt *Runtime) watchCluster() { func (rt *Runtime) deactivateMovedActivations(view cluster.View) { for _, id := range rt.engine.GrainIds() { - owner, ok := cluster.Owner(view, store.GrainId(id)) + owner, ok := cluster.Owner(view, store.GrainId{GrainType: id.GrainType, GrainKey: id.GrainKey}) if !ok || owner != rt.nodeAddr { rt.engine.Deactivate(runtimepkg.GrainId(id)) } @@ -965,24 +1478,56 @@ type reminderInvoker struct { } func (i reminderInvoker) Invoke(ctx context.Context, id store.GrainId, method string, args any, reply any) error { - err := i.runtime.Invoke(ctx, GrainId(id), method, args, reply) - if err != nil && ctx.Err() == nil && i.runtime.onError != nil { - i.runtime.onError(BackgroundError{ - GrainId: GrainId(id), - Err: err, - Source: ReminderInvocation{Method: method}, - }) - } - return err + return i.runtime.Invoke(ctx, fromStoreGrainID(id), method, args, reply) } func (i reminderInvoker) Owns(id store.GrainId) bool { return i.runtime.Owns(id) } -// TypeName returns the registered name used for T by the generated runtime -// glue. Application code should use this helper rather than constructing type -// names by hand. -func TypeName[T any]() string { - return strings.TrimPrefix(fmt.Sprintf("%T", (*T)(nil)), "*") +func (rt *Runtime) reportReminderFailure(failure timer.Failure) { + event := BackgroundError{Err: failure.Err} + if failure.Reminder.GrainId != (store.GrainId{}) { + event.GrainId = fromStoreGrainID(failure.Reminder.GrainId) + } + switch failure.Kind { + case timer.FailureScan: + event.Source = ReminderScan{} + case timer.FailureDispatch: + event.Source = ReminderDispatch{Name: failure.Reminder.Name, Method: failure.Reminder.Method} + case timer.FailureClaim: + event.Source = ReminderClaim{Name: failure.Reminder.Name} + case timer.FailureTerminal: + event.Source = ReminderTerminal{Name: failure.Reminder.Name, Method: failure.Reminder.Method} + case timer.FailureInvoke: + event.Source = ReminderInvocation{ + Name: failure.Reminder.Name, + Method: failure.Reminder.Method, + TickStatus: TickStatus{ + ReminderName: failure.Reminder.Name, + FirstTickTime: failure.Reminder.FirstTickTime, + Period: failure.Reminder.Interval, + CurrentTickTime: failure.CurrentTickTime, + }, + } + default: + return + } + rt.reportBackgroundError(event) +} + +func toStoreGrainID(id GrainId) store.GrainId { + return store.GrainId{GrainType: string(id.GrainType), GrainKey: id.GrainKey} +} + +func fromStoreGrainID(id store.GrainId) GrainId { + return GrainId{GrainType: GrainType(id.GrainType), GrainKey: id.GrainKey} +} + +func toRuntimeGrainID(id GrainId) runtimepkg.GrainId { + return runtimepkg.GrainId{GrainType: string(id.GrainType), GrainKey: id.GrainKey} +} + +func fromRuntimeGrainID(id runtimepkg.GrainId) GrainId { + return GrainId{GrainType: GrainType(id.GrainType), GrainKey: id.GrainKey} } diff --git a/gor_test.go b/gor_test.go index 265950e..61751a6 100644 --- a/gor_test.go +++ b/gor_test.go @@ -57,6 +57,7 @@ type lifecycleAccount interface { Value(context.Context) (int, error) Panic(context.Context) error SetValue(context.Context, int) error + Deactivate(context.Context) error } type lifecycleAccountValueRequest struct{} @@ -71,14 +72,21 @@ type lifecycleAccountSetValueRequest struct { type lifecycleAccountSetValueReply struct{} +type lifecycleAccountDeactivateRequest struct{} + +type lifecycleAccountDeactivateReply struct{} + type lifecycleAccountValueReply struct { R0 int } -type lifecycleAccountEntity struct { +type lifecycleAccountGrain struct { + grain *GrainContext value State[int] activateErr error + activatePanic bool deactivateErr error + deactivatePanic bool deactivateCalls *atomic.Int32 deactivateContexts chan context.Context deactivateReasons chan DeactivationReason @@ -91,14 +99,17 @@ type lifecycleAccountProxy struct { id GrainId } -func (e *lifecycleAccountEntity) OnActivate(context.Context) error { +func (e *lifecycleAccountGrain) OnActivate(context.Context) error { + if e.activatePanic { + panic("lifecycleAccount activate boom") + } if e.events != nil { e.events <- fmt.Sprintf("activate:%d", e.value.Get()) } return e.activateErr } -func (e *lifecycleAccountEntity) OnDeactivate(ctx context.Context, reason DeactivationReason) error { +func (e *lifecycleAccountGrain) OnDeactivate(ctx context.Context, reason DeactivationReason) error { if e.deactivateCalls != nil { e.deactivateCalls.Add(1) } @@ -114,24 +125,33 @@ func (e *lifecycleAccountEntity) OnDeactivate(ctx context.Context, reason Deacti if e.releaseDeactivate != nil { <-e.releaseDeactivate } + if e.deactivatePanic { + panic("lifecycleAccount deactivate boom") + } return e.deactivateErr } -func (e *lifecycleAccountEntity) Value(context.Context) (int, error) { +func (e *lifecycleAccountGrain) Value(context.Context) (int, error) { if e.events != nil { e.events <- "value" } return e.value.Get(), nil } -func (e *lifecycleAccountEntity) Panic(context.Context) error { +func (e *lifecycleAccountGrain) Panic(context.Context) error { panic("lifecycleAccount boom") } -func (e *lifecycleAccountEntity) SetValue(ctx context.Context, value int) error { +func (e *lifecycleAccountGrain) SetValue(ctx context.Context, value int) error { return e.value.Set(ctx, value) } +func (e *lifecycleAccountGrain) Deactivate(context.Context) error { + DeactivateOnIdle(e.grain) + DeactivateOnIdle(e.grain) + return nil +} + func (p *lifecycleAccountProxy) Value(ctx context.Context) (int, error) { var reply lifecycleAccountValueReply err := p.invoker.Invoke(ctx, p.id, "Value", &lifecycleAccountValueRequest{}, &reply) @@ -146,6 +166,10 @@ func (p *lifecycleAccountProxy) SetValue(ctx context.Context, value int) error { return p.invoker.Invoke(ctx, p.id, "SetValue", &lifecycleAccountSetValueRequest{A0: value}, &lifecycleAccountSetValueReply{}) } +func (p *lifecycleAccountProxy) Deactivate(ctx context.Context) error { + return p.invoker.Invoke(ctx, p.id, "Deactivate", &lifecycleAccountDeactivateRequest{}, &lifecycleAccountDeactivateReply{}) +} + func dispatchLifecycleAccount(ctx context.Context, instance lifecycleAccount, method string, args any, reply any) error { switch method { case "Value": @@ -160,6 +184,8 @@ func dispatchLifecycleAccount(ctx context.Context, instance lifecycleAccount, me case "SetValue": typedArgs := args.(*lifecycleAccountSetValueRequest) return instance.SetValue(ctx, typedArgs.A0) + case "Deactivate": + return instance.Deactivate(ctx) default: return fmt.Errorf("unknown method %q", method) } @@ -173,23 +199,25 @@ func newLifecycleAccountCall(method string) (args any, reply any) { return &lifecycleAccountPanicRequest{}, &lifecycleAccountPanicReply{} case "SetValue": return &lifecycleAccountSetValueRequest{}, &lifecycleAccountSetValueReply{} + case "Deactivate": + return &lifecycleAccountDeactivateRequest{}, &lifecycleAccountDeactivateReply{} default: return nil, nil } } -func installLifecycleAccount(t *testing.T, rt *Runtime, factoryCalls *atomic.Int32, configure func(*lifecycleAccountEntity)) { +func installLifecycleAccount(t *testing.T, rt *Runtime, factoryCalls *atomic.Int32, configure func(*lifecycleAccountGrain)) { t.Helper() - if err := InstallType[lifecycleAccount](rt, dispatchLifecycleAccount, func(invoker Invoker, id GrainId) lifecycleAccount { + if err := InstallType[lifecycleAccount](rt, GeneratedCodeVersion, "gor.lifecycleAccount", dispatchLifecycleAccount, func(invoker Invoker, id GrainId) lifecycleAccount { return &lifecycleAccountProxy{invoker: invoker, id: id} - }, newLifecycleAccountCall, nil); err != nil { + }, newLifecycleAccountCall, noReminderCall); err != nil { t.Fatal(err) } - if err := Register[lifecycleAccount](rt, func(b *Binder) lifecycleAccount { + if err := Register[lifecycleAccount](rt, func(b *GrainContext) lifecycleAccount { factoryCalls.Add(1) - entity := &lifecycleAccountEntity{value: NewState[int](b, "value")} - configure(entity) - return entity + grain := &lifecycleAccountGrain{grain: b, value: NewState[int](b, "value")} + configure(grain) + return grain }); err != nil { t.Fatal(err) } @@ -214,16 +242,16 @@ type scopeAccountForwardDepositReply struct { R0 int64 } -type scopeAccountEntity struct { +type scopeAccountGrain struct { createdAt time.Time target Account } -func (a *scopeAccountEntity) CreatedAt(context.Context) (time.Time, error) { +func (a *scopeAccountGrain) CreatedAt(context.Context) (time.Time, error) { return a.createdAt, nil } -func (a *scopeAccountEntity) ForwardDeposit(ctx context.Context, amount int64) (int64, error) { +func (a *scopeAccountGrain) ForwardDeposit(ctx context.Context, amount int64) (int64, error) { return a.target.Deposit(ctx, amount) } @@ -257,6 +285,8 @@ func newScopeAccountCall(method string) (args any, reply any) { func mustNew(t *testing.T, options ...Option) *Runtime { t.Helper() + backend := store.NewMemory() + options = append([]Option{WithStore(backend), WithReminderStore(backend)}, options...) rt, err := New(options...) if err != nil { t.Fatal(err) @@ -264,6 +294,27 @@ func mustNew(t *testing.T, options ...Option) *Runtime { return rt } +func noReminderCall(string, TickStatus) (any, any) { + return nil, nil +} + +func mustStart(t *testing.T, rt *Runtime) { + t.Helper() + if err := rt.Start(context.Background()); err != nil { + t.Fatal(err) + } +} + +func closeRuntime(rt *Runtime) { + _ = rt.Shutdown(context.Background()) +} + +func killRuntime(rt *Runtime) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _ = rt.Shutdown(ctx) +} + func (a *account) Deposit(ctx context.Context, amount int64) (int64, error) { value := a.value.Get() + amount if err := a.value.Set(ctx, value); err != nil { @@ -278,16 +329,17 @@ func (a *account) Balance(context.Context) (int64, error) { func TestLifecycle_OnActivateRunsAfterLoadBeforeFirstCall(t *testing.T) { backend := store.NewMemory() - id := store.GrainId{GrainType: TypeName[lifecycleAccount](), GrainKey: "alice"} + id := store.GrainId{GrainType: "gor.lifecycleAccount", GrainKey: "alice"} if _, err := backend.Write(context.Background(), id, []byte(`{"value":7}`), 0); err != nil { t.Fatalf("seed Write: %v", err) } events := make(chan string, 3) rt := mustNew(t, WithStore(backend), WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() - installLifecycleAccount(t, rt, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.events = events + defer closeRuntime(rt) + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.events = events }) + mustStart(t, rt) value, err := Ref[lifecycleAccount](rt, "alice").Value(context.Background()) if err != nil || value != 7 { @@ -304,13 +356,14 @@ func TestLifecycle_OnActivateFailureDoesNotEstablishActivation(t *testing.T) { synctest.Test(t, func(t *testing.T) { activateErr := errors.New("activate failed") factoryCalls := new(atomic.Int32) - rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() - installLifecycleAccount(t, rt, factoryCalls, func(entity *lifecycleAccountEntity) { - entity.activateErr = activateErr + rt := mustNew(t, WithMaxActivations(1), WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + installLifecycleAccount(t, rt, factoryCalls, func(grain *lifecycleAccountGrain) { + grain.activateErr = activateErr }) + mustStart(t, rt) - id := GrainId{GrainType: TypeName[lifecycleAccount](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: "alice"} if err := rt.Invoke(context.Background(), id, "Value", &lifecycleAccountValueRequest{}, &lifecycleAccountValueReply{}); !errors.Is(err, activateErr) { t.Fatalf("first activation error = %v, want %v", err, activateErr) } @@ -339,14 +392,15 @@ func TestLifecycle_OnDeactivateFailureReportsAndRemovesActivation(t *testing.T) errorsSeen <- event }), ) - defer rt.Close() + defer closeRuntime(rt) deactivateCalls := new(atomic.Int32) - installLifecycleAccount(t, rt, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.deactivateErr = deactivateErr - entity.deactivateCalls = deactivateCalls + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateErr = deactivateErr + grain.deactivateCalls = deactivateCalls }) + mustStart(t, rt) - id := GrainId{GrainType: TypeName[lifecycleAccount](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: "alice"} if err := rt.Invoke(context.Background(), id, "Value", &lifecycleAccountValueRequest{}, &lifecycleAccountValueReply{}); err != nil { t.Fatalf("initial Value: %v", err) } @@ -383,18 +437,19 @@ func TestLifecycle_KillSkipsOnDeactivate(t *testing.T) { errorsSeen <- event }), ) - defer rt.Close() + defer closeRuntime(rt) deactivateCalls := new(atomic.Int32) - installLifecycleAccount(t, rt, new(atomic.Int32), func(entity *lifecycleAccountEntity) { - entity.deactivateCalls = deactivateCalls - entity.deactivateErr = errors.New("deactivate failed") + installLifecycleAccount(t, rt, new(atomic.Int32), func(grain *lifecycleAccountGrain) { + grain.deactivateCalls = deactivateCalls + grain.deactivateErr = errors.New("deactivate failed") }) + mustStart(t, rt) - id := GrainId{GrainType: TypeName[lifecycleAccount](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.lifecycleAccount"), GrainKey: "alice"} if err := rt.Invoke(context.Background(), id, "Value", &lifecycleAccountValueRequest{}, &lifecycleAccountValueReply{}); err != nil { t.Fatalf("initial Value: %v", err) } - rt.Kill() + killRuntime(rt) synctest.Wait() if got := deactivateCalls.Load(); got != 0 { t.Fatalf("OnDeactivate calls after Kill = %d, want 0", got) @@ -432,16 +487,17 @@ func dispatchScopeAccount(ctx context.Context, instance scopeAccount, method str func TestRegister_InvokesInstalledDispatch(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installAccount(t, rt) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) - id := GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"} var first accountDepositReply if err := rt.Invoke(context.Background(), id, "Deposit", &accountDepositRequest{A0: 2}, &first); err != nil { t.Fatalf("first invoke error = %v", err) @@ -465,47 +521,48 @@ func TestRegister_InvokesInstalledDispatch(t *testing.T) { func TestRegister_RejectsDuplicateType(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installAccount(t, rt) - if err := Register[Account](rt, func(*Binder) Account { return &account{} }); err != nil { + if err := Register[Account](rt, func(*GrainContext) Account { return &account{} }); err != nil { t.Fatal(err) } - if err := Register[Account](rt, func(*Binder) Account { return &account{} }); err == nil { + if err := Register[Account](rt, func(*GrainContext) Account { return &account{} }); err == nil { t.Fatal("duplicate registration returned nil error") } } func TestRegister_RejectsUninstalledType(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) - if err := Register[Account](rt, func(*Binder) Account { return &account{} }); !errors.Is(err, ErrTypeNotInstalled) { + if err := Register[Account](rt, func(*GrainContext) Account { return &account{} }); !errors.Is(err, ErrTypeNotInstalled) { t.Fatalf("Register error = %v, want ErrTypeNotInstalled", err) } } func TestInstallType_IsScopedToRuntime(t *testing.T) { first := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer first.Close() + defer closeRuntime(first) second := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer second.Close() + defer closeRuntime(second) installAccount(t, first) - if err := Register[Account](second, func(*Binder) Account { return &account{} }); !errors.Is(err, ErrTypeNotInstalled) { + if err := Register[Account](second, func(*GrainContext) Account { return &account{} }); !errors.Is(err, ErrTypeNotInstalled) { t.Fatalf("second runtime Register error = %v, want ErrTypeNotInstalled", err) } } func TestRef_ConstructsTypedProxyFromInstalledType(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installAccount(t, rt) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) accountRef := Ref[Account](rt, "alice") if _, ok := accountRef.(*accountProxy); !ok { @@ -517,32 +574,33 @@ func TestRef_ConstructsTypedProxyFromInstalledType(t *testing.T) { } } -func TestBinderScope_ProvidesClockAndTypedReferences(t *testing.T) { +func TestGrainContextScope_ProvidesClockAndTypedReferences(t *testing.T) { synctest.Test(t, func(t *testing.T) { start := time.Unix(100, 0).UTC() fakeClock := clock.NewFake(start) rt := mustNew(t, WithClock(fakeClock), WithIdleTimeout(time.Minute), WithEvictionInterval(time.Minute)) - defer rt.Close() + defer closeRuntime(rt) installAccount(t, rt) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } - if err := InstallType[scopeAccount](rt, dispatchScopeAccount, func(invoker Invoker, id GrainId) scopeAccount { + if err := InstallType[scopeAccount](rt, GeneratedCodeVersion, "gor.scopeAccount", dispatchScopeAccount, func(invoker Invoker, id GrainId) scopeAccount { return &scopeAccountProxy{invoker: invoker, id: id} - }, newScopeAccountCall, nil); err != nil { + }, newScopeAccountCall, noReminderCall); err != nil { t.Fatal(err) } - if err := Register[scopeAccount](rt, func(b *Binder) scopeAccount { - return &scopeAccountEntity{ + if err := Register[scopeAccount](rt, func(b *GrainContext) scopeAccount { + return &scopeAccountGrain{ createdAt: Now(b), target: Ref[Account](b, "target"), } }); err != nil { t.Fatal(err) } + mustStart(t, rt) source := Ref[scopeAccount](rt, "source") createdAt, err := source.CreatedAt(context.Background()) @@ -572,7 +630,7 @@ func TestBinderScope_ProvidesClockAndTypedReferences(t *testing.T) { func TestRef_PanicsForUninstalledType(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) defer func() { if recover() == nil { @@ -585,22 +643,23 @@ func TestRef_PanicsForUninstalledType(t *testing.T) { func TestRegister_LoadsAndPersistsState(t *testing.T) { synctest.Test(t, func(t *testing.T) { backend := store.NewMemory() - id := store.GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + id := store.GrainId{GrainType: "gor.Account", GrainKey: "alice"} if _, err := backend.Write(context.Background(), id, []byte(`{"value":7}`), 0); err != nil { t.Fatalf("seed Write: %v", err) } rt := mustNew(t, WithStore(backend), WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installAccount(t, rt) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) var balance accountBalanceReply - if err := rt.Invoke(context.Background(), GrainId(id), "Balance", &accountBalanceRequest{}, &balance); err != nil { + if err := rt.Invoke(context.Background(), fromStoreGrainID(id), "Balance", &accountBalanceRequest{}, &balance); err != nil { t.Fatalf("Balance invoke error = %v", err) } if balance.R0 != 7 { @@ -608,7 +667,7 @@ func TestRegister_LoadsAndPersistsState(t *testing.T) { } var deposited accountDepositReply - if err := rt.Invoke(context.Background(), GrainId(id), "Deposit", &accountDepositRequest{A0: 2}, &deposited); err != nil { + if err := rt.Invoke(context.Background(), fromStoreGrainID(id), "Deposit", &accountDepositRequest{A0: 2}, &deposited); err != nil { t.Fatalf("Deposit invoke error = %v", err) } if deposited.R0 != 9 { @@ -628,19 +687,21 @@ func TestRegister_LoadsAndPersistsState(t *testing.T) { func TestRuntime_RestartRestoresStateFromMemoryStore(t *testing.T) { synctest.Test(t, func(t *testing.T) { backend := store.NewMemory() - id := GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"} first := mustNew(t, WithStore(backend), WithIdleTimeout(0), WithEvictionInterval(0)) registerAccount(t, first) + mustStart(t, first) var written accountDepositReply if err := first.Invoke(context.Background(), id, "Deposit", &accountDepositRequest{A0: 42}, &written); err != nil { t.Fatalf("first Deposit invoke error = %v", err) } - first.Close() + closeRuntime(first) second := mustNew(t, WithStore(backend), WithIdleTimeout(0), WithEvictionInterval(0)) registerAccount(t, second) - defer second.Close() + mustStart(t, second) + defer closeRuntime(second) var restored accountBalanceReply if err := second.Invoke(context.Background(), id, "Balance", &accountBalanceRequest{}, &restored); err != nil { t.Fatalf("restarted Balance invoke error = %v", err) @@ -657,15 +718,16 @@ func TestRuntime_RestartRestoresStateFromSQLite(t *testing.T) { if err != nil { t.Fatalf("OpenSQLite first: %v", err) } - first := mustNew(t, WithStore(firstStore), WithIdleTimeout(0), WithEvictionInterval(0)) + first := mustNew(t, WithStore(firstStore), WithReminderStore(firstStore), WithIdleTimeout(0), WithEvictionInterval(0)) registerAccount(t, first) + mustStart(t, first) var written accountDepositReply - if err := first.Invoke(context.Background(), GrainId{GrainType: TypeName[Account](), GrainKey: "alice"}, "Deposit", &accountDepositRequest{A0: 42}, &written); err != nil { - first.Close() + if err := first.Invoke(context.Background(), GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"}, "Deposit", &accountDepositRequest{A0: 42}, &written); err != nil { + closeRuntime(first) firstStore.Close() t.Fatalf("first Deposit invoke error = %v", err) } - first.Close() + closeRuntime(first) if err := firstStore.Close(); err != nil { t.Fatalf("Close first store: %v", err) } @@ -674,12 +736,13 @@ func TestRuntime_RestartRestoresStateFromSQLite(t *testing.T) { if err != nil { t.Fatalf("OpenSQLite second: %v", err) } - second := mustNew(t, WithStore(secondStore), WithIdleTimeout(0), WithEvictionInterval(0)) + second := mustNew(t, WithStore(secondStore), WithReminderStore(secondStore), WithIdleTimeout(0), WithEvictionInterval(0)) registerAccount(t, second) - defer second.Close() + mustStart(t, second) + defer closeRuntime(second) defer secondStore.Close() var restored accountBalanceReply - if err := second.Invoke(context.Background(), GrainId{GrainType: TypeName[Account](), GrainKey: "alice"}, "Balance", &accountBalanceRequest{}, &restored); err != nil { + if err := second.Invoke(context.Background(), GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"}, "Balance", &accountBalanceRequest{}, &restored); err != nil { t.Fatalf("restarted Balance invoke error = %v", err) } if restored.R0 != 42 { @@ -698,18 +761,19 @@ func dispatchAccountWithWrappedError(ctx context.Context, instance Account, meth func TestRegister_ConflictDiscardsActivationBeforeReactivation(t *testing.T) { synctest.Test(t, func(t *testing.T) { backend := store.NewMemory() - id := GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"} rt := mustNew(t, WithStore(backend), WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) var factoryCalls atomic.Int32 installAccountWithDispatch(t, rt, dispatchAccountWithWrappedError) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { factoryCalls.Add(1) return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) var first accountDepositReply if err := rt.Invoke(context.Background(), id, "Deposit", &accountDepositRequest{A0: 10}, &first); err != nil { @@ -719,7 +783,7 @@ func TestRegister_ConflictDiscardsActivationBeforeReactivation(t *testing.T) { t.Fatalf("first balance = %d, want 10", first.R0) } - storeID := store.GrainId(id) + storeID := toStoreGrainID(id) if _, err := backend.Write(context.Background(), storeID, []byte(`{"value":100}`), 1); err != nil { t.Fatalf("external Write: %v", err) } @@ -758,18 +822,19 @@ func TestRegister_StateWriteFailureReturnsErrorAndDiscardsActivation(t *testing. synctest.Test(t, func(t *testing.T) { writeErr := errors.New("store unavailable") rt := mustNew(t, WithStore(failingWriteStore{err: writeErr}), WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) var factoryCalls atomic.Int32 installAccount(t, rt) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { factoryCalls.Add(1) return &writeIgnoringAccount{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) - id := GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"} var result accountDepositReply err := rt.Invoke(context.Background(), id, "Deposit", &accountDepositRequest{A0: 1}, &result) if err == nil { @@ -794,16 +859,17 @@ func TestRegister_StateWriteFailureJoinsMethodError(t *testing.T) { writeErr := errors.New("store unavailable") methodErr := errors.New("method failed") rt := mustNew(t, WithStore(failingWriteStore{err: writeErr}), WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) installAccount(t, rt) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &writeIgnoringAccount{value: NewState[int64](b, "value"), methodErr: methodErr} }); err != nil { t.Fatal(err) } + mustStart(t, rt) - id := GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"} var result accountDepositReply err := rt.Invoke(context.Background(), id, "Deposit", &accountDepositRequest{A0: 1}, &result) if err == nil { @@ -858,7 +924,7 @@ func newAccountCall(method string) (args any, reply any) { func registerAccount(t *testing.T, rt *Runtime) { t.Helper() installAccount(t, rt) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) @@ -889,9 +955,9 @@ func installAccount(t *testing.T, rt *Runtime) { func installAccountWithDispatch(t *testing.T, rt *Runtime, dispatch func(context.Context, Account, string, any, any) error) { t.Helper() - if err := InstallType[Account](rt, dispatch, func(invoker Invoker, id GrainId) Account { + if err := InstallType[Account](rt, GeneratedCodeVersion, "gor.Account", dispatch, func(invoker Invoker, id GrainId) Account { return &accountProxy{invoker: invoker, id: id} - }, newAccountCall, nil); err != nil { + }, newAccountCall, noReminderCall); err != nil { t.Fatal(err) } } diff --git a/grain_timer.go b/grain_timer.go new file mode 100644 index 0000000..2fb2df2 --- /dev/null +++ b/grain_timer.go @@ -0,0 +1,67 @@ +package gor + +import ( + "context" + "errors" + "time" + + runtimepkg "github.com/suraciii/gor/internal/runtime" +) + +// ErrGrainTimerStopped reports that Change targeted a stopped Grain Timer or +// an Activation which is no longer active. +var ErrGrainTimerStopped = runtimepkg.ErrGrainTimerStopped + +// GrainTimer controls one timer that belongs to the current Activation. +type GrainTimer interface { + // Change resets the next due time and repeat period. A zero period makes the + // timer one-shot. A Change during a callback takes effect after that callback. + // Change returns ErrGrainTimerStopped after Stop or Activation end. + Change(dueTime time.Duration, period time.Duration) error + // Stop prevents queued and later callbacks. It is safe to call Stop more than + // once, and it does not cancel a callback that is already running. + Stop() +} + +// GrainTimerOptions defines when a Grain Timer runs and whether its completed +// callbacks update the Activation's idle-use time. +type GrainTimerOptions struct { + // DueTime is the delay before the first callback. Zero makes it ready now. + DueTime time.Duration + // Period is the delay after one callback ends. Zero makes a one-shot timer. + Period time.Duration + // KeepAlive makes each completed callback refresh Activation idle time. + KeepAlive bool +} + +type grainTimerRegistrar func(func(context.Context) error, GrainTimerOptions) (GrainTimer, error) + +// RegisterGrainTimer creates a timer for the Activation bound to grainContext. +// A zero DueTime makes the first callback ready now. A zero Period makes the +// timer one-shot. DueTime and Period must not be negative. Each callback enters +// the Grain mailbox as a serialized turn. The timer stops when the Activation +// ends. +func RegisterGrainTimer(grainContext *GrainContext, callback func(context.Context) error, options GrainTimerOptions) (GrainTimer, error) { + if grainContext == nil { + return nil, errors.New("missing Grain Context") + } + if callback == nil { + return nil, errors.New("missing Grain Timer callback") + } + if options.DueTime < 0 { + return nil, errors.New("invalid Grain Timer DueTime: must not be negative") + } + if options.Period < 0 { + return nil, errors.New("invalid Grain Timer Period: must not be negative") + } + if grainContext.registerGrainTimer == nil { + return nil, ErrGrainTimerStopped + } + return grainContext.registerGrainTimer(func(ctx context.Context) error { + err := callback(ctx) + if result, discarded := grainContext.resultWithDiscard(err); discarded { + return runtimepkg.Discard{Err: result} + } + return err + }, options) +} diff --git a/grain_timer_test.go b/grain_timer_test.go new file mode 100644 index 0000000..9fe910c --- /dev/null +++ b/grain_timer_test.go @@ -0,0 +1,809 @@ +package gor + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/suraciii/gor/clock" +) + +type grainTimerProbe interface{} + +type grainTimerProbeOptions struct { + dueTime time.Duration + period time.Duration + keepAlive bool +} + +type grainTimerProbeChange struct { + dueTime time.Duration + period time.Duration +} + +type grainTimerProbeReply struct { + activation int +} + +type grainTimerProbeGrain struct { + grain *GrainContext + timer GrainTimer + state State[int] + activation int + callback func(*grainTimerProbeGrain, context.Context, int) error + ticks int + + blockStarted chan<- struct{} + blockRelease <-chan struct{} + timerCreated chan<- GrainTimer +} + +type grainTimerProbeFactory struct { + activations atomic.Int32 + callback func(*grainTimerProbeGrain, context.Context, int) error + withState bool + + blockStarted chan<- struct{} + blockRelease <-chan struct{} + timerCreated chan<- GrainTimer +} + +func (f *grainTimerProbeFactory) new(grain *GrainContext) grainTimerProbe { + probe := &grainTimerProbeGrain{ + grain: grain, + activation: int(f.activations.Add(1)), + callback: f.callback, + blockStarted: f.blockStarted, + blockRelease: f.blockRelease, + timerCreated: f.timerCreated, + } + if f.withState { + probe.state = NewState[int](grain, "value") + } + return probe +} + +func dispatchGrainTimerProbe(ctx context.Context, instance grainTimerProbe, method string, args any, reply any) error { + probe := instance.(*grainTimerProbeGrain) + switch method { + case "Start": + options := args.(*grainTimerProbeOptions) + timer, err := RegisterGrainTimer(probe.grain, func(callbackCtx context.Context) error { + probe.ticks++ + if probe.callback == nil { + return nil + } + return probe.callback(probe, callbackCtx, probe.ticks) + }, GrainTimerOptions{DueTime: options.dueTime, Period: options.period, KeepAlive: options.keepAlive}) + if err != nil { + return err + } + probe.timer = timer + if probe.timerCreated != nil { + probe.timerCreated <- timer + } + return nil + case "Change": + change := args.(*grainTimerProbeChange) + return probe.timer.Change(change.dueTime, change.period) + case "Stop": + probe.timer.Stop() + return nil + case "Block": + probe.blockStarted <- struct{}{} + <-probe.blockRelease + return nil + case "BlockAndDeactivate": + DeactivateOnIdle(probe.grain) + probe.blockStarted <- struct{}{} + <-probe.blockRelease + return nil + case "Deactivate": + DeactivateOnIdle(probe.grain) + return nil + case "Activation": + reply.(*grainTimerProbeReply).activation = probe.activation + return nil + default: + return errors.New("unknown Grain Timer probe method") + } +} + +func installGrainTimerProbe(t *testing.T, rt *Runtime, factory *grainTimerProbeFactory) { + t.Helper() + if err := InstallType[grainTimerProbe](rt, GeneratedCodeVersion, "gor.grainTimerProbe", dispatchGrainTimerProbe, func(Invoker, GrainId) grainTimerProbe { + return struct{}{} + }, func(string) (any, any) { return &struct{}{}, &struct{}{} }, noReminderCall); err != nil { + t.Fatal(err) + } + if err := Register[grainTimerProbe](rt, factory.new); err != nil { + t.Fatal(err) + } +} + +func invokeGrainTimerProbe(rt *Runtime, method string, args any, reply any) error { + return rt.Invoke(context.Background(), GrainId{GrainType: "gor.grainTimerProbe", GrainKey: "probe"}, method, args, reply) +} + +func receiveGrainTimerTestValue[T any](t *testing.T, channel <-chan T, name string) T { + t.Helper() + select { + case value := <-channel: + return value + default: + t.Fatalf("%s was not ready", name) + var zero T + return zero + } +} + +func startGrainTimerProbe(t *testing.T, sourceClock *clock.Fake, factory *grainTimerProbeFactory, options ...Option) *Runtime { + t.Helper() + options = append([]Option{WithClock(sourceClock), WithReminderInterval(0)}, options...) + rt := mustNew(t, options...) + installGrainTimerProbe(t, rt, factory) + mustStart(t, rt) + return rt +} + +func TestRegisterGrainTimerValidatesAndStoppedHandleRejectsChange(t *testing.T) { + if _, err := RegisterGrainTimer(nil, func(context.Context) error { return nil }, GrainTimerOptions{}); err == nil { + t.Fatal("RegisterGrainTimer accepted a nil Grain Context") + } + if _, err := RegisterGrainTimer(&GrainContext{}, nil, GrainTimerOptions{}); err == nil { + t.Fatal("RegisterGrainTimer accepted a nil callback") + } + + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(100, 0).UTC()) + created := make(chan GrainTimer, 1) + rt := startGrainTimerProbe(t, sourceClock, &grainTimerProbeFactory{timerCreated: created}, WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + + err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{dueTime: -1}, nil) + if err == nil || !strings.Contains(err.Error(), "DueTime") { + t.Fatalf("negative DueTime error = %v", err) + } + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{dueTime: time.Hour}, nil); err != nil { + t.Fatal(err) + } + timer := <-created + if err := timer.Change(-1, 0); err == nil { + t.Fatal("Change accepted a negative due time") + } + if err := timer.Change(0, -1); err == nil { + t.Fatal("Change accepted a negative period") + } + timer.Stop() + timer.Stop() + if err := timer.Change(0, 0); !errors.Is(err, ErrGrainTimerStopped) { + t.Fatalf("Change after Stop = %v, want ErrGrainTimerStopped", err) + } + }) +} + +func TestGrainTimerSerializesWithCallAndStartsPeriodAfterCompletion(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(200, 0).UTC() + sourceClock := clock.NewFake(start) + blockStarted := make(chan struct{}, 1) + blockRelease := make(chan struct{}, 1) + callbackStarted := make(chan time.Time, 4) + callbackRelease := make(chan struct{}, 4) + factory := &grainTimerProbeFactory{ + blockStarted: blockStarted, + blockRelease: blockRelease, + callback: func(_ *grainTimerProbeGrain, _ context.Context, _ int) error { + callbackStarted <- sourceClock.Now() + <-callbackRelease + return nil + }, + } + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{dueTime: time.Second, period: 2 * time.Second}, nil); err != nil { + t.Fatal(err) + } + + blockDone := make(chan error, 1) + go func() { blockDone <- invokeGrainTimerProbe(rt, "Block", nil, nil) }() + synctest.Wait() + <-blockStarted + sourceClock.Advance(time.Second) + synctest.Wait() + select { + case <-callbackStarted: + t.Fatal("Grain Timer callback overlapped a Call") + default: + } + + blockRelease <- struct{}{} + synctest.Wait() + if err := <-blockDone; err != nil { + t.Fatal(err) + } + if at := <-callbackStarted; !at.Equal(start.Add(time.Second)) { + t.Fatalf("first callback time = %s", at) + } + sourceClock.Advance(10 * time.Second) + synctest.Wait() + select { + case <-callbackStarted: + t.Fatal("repeating Grain Timer overlapped itself") + default: + } + + callbackRelease <- struct{}{} + synctest.Wait() + sourceClock.Advance(2*time.Second - time.Nanosecond) + synctest.Wait() + select { + case <-callbackStarted: + t.Fatal("next period started before callback completion plus Period") + default: + } + sourceClock.Advance(time.Nanosecond) + synctest.Wait() + if at := <-callbackStarted; !at.Equal(start.Add(13 * time.Second)) { + t.Fatalf("second callback time = %s, want %s", at, start.Add(13*time.Second)) + } + callbackRelease <- struct{}{} + }) +} + +func TestGrainTimerChangeDuringCallbackControlsNextDueTime(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(300, 0).UTC() + sourceClock := clock.NewFake(start) + started := make(chan int, 4) + release := make(chan struct{}, 4) + factory := &grainTimerProbeFactory{callback: func(probe *grainTimerProbeGrain, _ context.Context, tick int) error { + if tick == 1 { + if err := probe.timer.Change(3*time.Second, 4*time.Second); err != nil { + return err + } + } + started <- tick + <-release + return nil + }} + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{dueTime: time.Second, period: time.Hour}, nil); err != nil { + t.Fatal(err) + } + sourceClock.Advance(time.Second) + synctest.Wait() + if tick := <-started; tick != 1 { + t.Fatalf("first tick = %d", tick) + } + sourceClock.Advance(10 * time.Second) + release <- struct{}{} + synctest.Wait() + sourceClock.Advance(3*time.Second - time.Nanosecond) + synctest.Wait() + select { + case tick := <-started: + t.Fatalf("changed timer fired early at tick %d", tick) + default: + } + sourceClock.Advance(time.Nanosecond) + synctest.Wait() + if tick := receiveGrainTimerTestValue(t, started, "changed Grain Timer tick"); tick != 2 { + t.Fatalf("changed tick = %d, want 2", tick) + } + release <- struct{}{} + }) +} + +func TestGrainTimerStopDiscardsQueuedTick(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(400, 0).UTC()) + blockStarted := make(chan struct{}, 1) + blockRelease := make(chan struct{}, 1) + created := make(chan GrainTimer, 1) + callbackStarted := make(chan struct{}, 1) + factory := &grainTimerProbeFactory{ + blockStarted: blockStarted, + blockRelease: blockRelease, + timerCreated: created, + callback: func(*grainTimerProbeGrain, context.Context, int) error { + callbackStarted <- struct{}{} + return nil + }, + } + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{dueTime: time.Second}, nil); err != nil { + t.Fatal(err) + } + timer := <-created + blockDone := make(chan error, 1) + go func() { blockDone <- invokeGrainTimerProbe(rt, "Block", nil, nil) }() + synctest.Wait() + <-blockStarted + sourceClock.Advance(time.Second) + synctest.Wait() + timer.Stop() + blockRelease <- struct{}{} + synctest.Wait() + if err := <-blockDone; err != nil { + t.Fatal(err) + } + select { + case <-callbackStarted: + t.Fatal("stopped queued tick entered callback") + default: + } + }) +} + +func TestGrainTimerStopDoesNotCancelRunningCallback(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(450, 0).UTC()) + created := make(chan GrainTimer, 1) + callbackContexts := make(chan context.Context, 1) + release := make(chan struct{}) + factory := &grainTimerProbeFactory{ + timerCreated: created, + callback: func(_ *grainTimerProbeGrain, ctx context.Context, _ int) error { + callbackContexts <- ctx + <-release + return nil + }, + } + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{period: time.Second}, nil); err != nil { + t.Fatal(err) + } + timer := <-created + synctest.Wait() + callbackCtx := receiveGrainTimerTestValue(t, callbackContexts, "running Grain Timer callback context") + timer.Stop() + callbackErr := callbackCtx.Err() + close(release) + synctest.Wait() + if callbackErr != nil { + t.Fatalf("Stop canceled running callback: %v", callbackErr) + } + sourceClock.Advance(time.Second) + synctest.Wait() + select { + case <-callbackContexts: + t.Fatal("stopped Grain Timer ran again") + default: + } + }) +} + +func TestGrainTimerCallbackCanStopItsTimer(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(475, 0).UTC()) + callbacks := make(chan struct{}, 2) + factory := &grainTimerProbeFactory{callback: func(probe *grainTimerProbeGrain, _ context.Context, _ int) error { + probe.timer.Stop() + callbacks <- struct{}{} + return nil + }} + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{period: time.Second}, nil); err != nil { + t.Fatal(err) + } + synctest.Wait() + <-callbacks + sourceClock.Advance(time.Second) + synctest.Wait() + select { + case <-callbacks: + t.Fatal("self-stopped Grain Timer ran again") + default: + } + }) +} + +func TestGrainTimerDeactivationDiscardsOldTickAndHandle(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(500, 0).UTC()) + blockStarted := make(chan struct{}, 1) + blockRelease := make(chan struct{}, 1) + created := make(chan GrainTimer, 1) + callbackStarted := make(chan struct{}, 1) + factory := &grainTimerProbeFactory{ + blockStarted: blockStarted, + blockRelease: blockRelease, + timerCreated: created, + callback: func(*grainTimerProbeGrain, context.Context, int) error { + callbackStarted <- struct{}{} + return nil + }, + } + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{dueTime: time.Second, keepAlive: true}, nil); err != nil { + t.Fatal(err) + } + oldTimer := <-created + deactivateDone := make(chan error, 1) + go func() { deactivateDone <- invokeGrainTimerProbe(rt, "BlockAndDeactivate", nil, nil) }() + synctest.Wait() + <-blockStarted + sourceClock.Advance(time.Second) + synctest.Wait() + blockRelease <- struct{}{} + synctest.Wait() + if err := <-deactivateDone; err != nil { + t.Fatal(err) + } + var reply grainTimerProbeReply + if err := invokeGrainTimerProbe(rt, "Activation", nil, &reply); err != nil { + t.Fatal(err) + } + if reply.activation != 2 { + t.Fatalf("Activation after Deactivate on Idle = %d, want 2", reply.activation) + } + if err := oldTimer.Change(0, 0); !errors.Is(err, ErrGrainTimerStopped) { + t.Fatalf("old timer Change = %v, want ErrGrainTimerStopped", err) + } + select { + case <-callbackStarted: + t.Fatal("old Activation timer callback ran") + default: + } + }) +} + +func TestGrainTimerRequestedDeactivationDoesNotBlockNextCallBehindQueuedTick(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(550, 0).UTC()) + blockStarted := make(chan struct{}, 1) + blockRelease := make(chan struct{}, 1) + callbackStarted := make(chan struct{}, 1) + factory := &grainTimerProbeFactory{ + blockStarted: blockStarted, + blockRelease: blockRelease, + callback: func(*grainTimerProbeGrain, context.Context, int) error { + callbackStarted <- struct{}{} + return nil + }, + } + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(0), WithEvictionInterval(0)) + defer killRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{dueTime: time.Second}, nil); err != nil { + t.Fatal(err) + } + + deactivateDone := make(chan error, 1) + go func() { deactivateDone <- invokeGrainTimerProbe(rt, "BlockAndDeactivate", nil, nil) }() + synctest.Wait() + <-blockStarted + + type activationResult struct { + activation int + err error + } + nextDone := make(chan activationResult, 1) + go func() { + var reply grainTimerProbeReply + err := invokeGrainTimerProbe(rt, "Activation", nil, &reply) + nextDone <- activationResult{activation: reply.activation, err: err} + }() + synctest.Wait() + activations := rt.Activations() + if len(activations) != 1 || activations[0].Queued != 1 { + t.Fatalf("queued Calls before Timer tick = %#v, want one queued Call", activations) + } + + sourceClock.Advance(time.Second) + synctest.Wait() + blockRelease <- struct{}{} + synctest.Wait() + if err := <-deactivateDone; err != nil { + t.Fatal(err) + } + result := receiveGrainTimerTestValue(t, nextDone, "Call behind requested deactivation") + if result.err != nil { + t.Fatal(result.err) + } + if result.activation != 2 { + t.Fatalf("Activation after requested deactivation = %d, want 2", result.activation) + } + select { + case <-callbackStarted: + t.Fatal("old Activation Grain Timer callback ran") + default: + } + }) +} + +func TestGrainTimerErrorsContinueAndPanicFaultsActivation(t *testing.T) { + t.Run("error continues", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + callbackErr := errors.New("timer callback failed") + sourceClock := clock.NewFake(time.Unix(600, 0).UTC()) + events := make(chan BackgroundError, 4) + factory := &grainTimerProbeFactory{callback: func(*grainTimerProbeGrain, context.Context, int) error { return callbackErr }} + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(0), WithEvictionInterval(0), OnError(func(event BackgroundError) { events <- event })) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{dueTime: time.Second, period: time.Second}, nil); err != nil { + t.Fatal(err) + } + for tick := 1; tick <= 2; tick++ { + sourceClock.Advance(time.Second) + synctest.Wait() + event := receiveGrainTimerTestValue(t, events, "Grain Timer error event") + if !errors.Is(event.Err, callbackErr) || event.GrainId.GrainKey != "probe" { + t.Fatalf("OnError event %d = %#v", tick, event) + } + if _, ok := event.Source.(GrainTimerInvocation); !ok { + t.Fatalf("OnError source = %T, want GrainTimerInvocation", event.Source) + } + } + if got := factory.activations.Load(); got != 1 { + t.Fatalf("Activations after callback errors = %d, want 1", got) + } + }) + }) + + t.Run("panic faults", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(700, 0).UTC()) + events := make(chan BackgroundError, 1) + factory := &grainTimerProbeFactory{callback: func(*grainTimerProbeGrain, context.Context, int) error { panic("timer panic") }} + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(0), WithEvictionInterval(0), OnError(func(event BackgroundError) { events <- event })) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{dueTime: time.Second, period: time.Second}, nil); err != nil { + t.Fatal(err) + } + sourceClock.Advance(time.Second) + synctest.Wait() + event := receiveGrainTimerTestValue(t, events, "Grain Timer panic event") + if !errors.Is(event.Err, ErrPanic) { + t.Fatalf("panic OnError = %v, want ErrPanic", event.Err) + } + var reply grainTimerProbeReply + if err := invokeGrainTimerProbe(rt, "Activation", nil, &reply); err != nil { + t.Fatal(err) + } + if reply.activation != 2 { + t.Fatalf("Activation after timer panic = %d, want 2", reply.activation) + } + }) + }) +} + +func TestGrainTimerStateFailureFaultsActivation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + writeErr := errors.New("timer State write failed") + sourceClock := clock.NewFake(time.Unix(750, 0).UTC()) + events := make(chan BackgroundError, 1) + factory := &grainTimerProbeFactory{ + withState: true, + callback: func(probe *grainTimerProbeGrain, ctx context.Context, _ int) error { + _ = probe.state.Set(ctx, 1) + return nil + }, + } + rt := startGrainTimerProbe(t, sourceClock, factory, + WithStore(failingWriteStore{err: writeErr}), + WithIdleTimeout(0), + WithEvictionInterval(0), + OnError(func(event BackgroundError) { events <- event }), + ) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{}, nil); err != nil { + t.Fatal(err) + } + synctest.Wait() + event := receiveGrainTimerTestValue(t, events, "Grain Timer State error event") + if !errors.Is(event.Err, writeErr) || !errors.Is(event.Err, ErrPersistenceFailed) { + t.Fatalf("State failure event = %v", event.Err) + } + var reply grainTimerProbeReply + if err := invokeGrainTimerProbe(rt, "Activation", nil, &reply); err != nil { + t.Fatal(err) + } + if reply.activation != 2 { + t.Fatalf("Activation after timer State failure = %d, want 2", reply.activation) + } + }) +} + +func TestGrainTimerUsesRuntimeContextAndShutdownCancellationIsSilent(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(800, 0).UTC()) + callbackContexts := make(chan context.Context, 1) + events := make(chan BackgroundError, 1) + factory := &grainTimerProbeFactory{callback: func(_ *grainTimerProbeGrain, ctx context.Context, _ int) error { + callbackContexts <- ctx + <-ctx.Done() + return ctx.Err() + }} + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(0), WithEvictionInterval(0), OnError(func(event BackgroundError) { events <- event })) + + callCtx, cancel := context.WithCancel(context.Background()) + callCtx, err := WithRequestContext(callCtx, "trace_id", "call") + if err != nil { + t.Fatal(err) + } + if err := rt.Invoke(callCtx, GrainId{GrainType: "gor.grainTimerProbe", GrainKey: "probe"}, "Start", &grainTimerProbeOptions{}, nil); err != nil { + t.Fatal(err) + } + cancel() + synctest.Wait() + callbackCtx := receiveGrainTimerTestValue(t, callbackContexts, "Grain Timer callback context") + if err := callbackCtx.Err(); err != nil { + t.Fatalf("callback inherited Call cancellation: %v", err) + } + if _, ok := callbackCtx.Deadline(); ok { + t.Fatal("callback inherited a deadline") + } + if value, ok := RequestContextValue(callbackCtx, "trace_id"); ok || value != nil { + t.Fatalf("callback Request Context = (%v, %v), want absent", value, ok) + } + + shutdownDone := make(chan error, 1) + go func() { shutdownDone <- rt.Shutdown(context.Background()) }() + synctest.Wait() + if err := <-shutdownDone; err != nil { + t.Fatal(err) + } + select { + case event := <-events: + t.Fatalf("Runtime cancellation reached OnError: %#v", event) + default: + } + }) +} + +func TestGrainTimerContextRejectsSelfCallCycle(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(850, 0).UTC()) + callbackResults := make(chan error, 1) + var rt *Runtime + factory := &grainTimerProbeFactory{callback: func(*grainTimerProbeGrain, context.Context, int) error { return nil }} + factory.callback = func(_ *grainTimerProbeGrain, ctx context.Context, _ int) error { + var reply grainTimerProbeReply + callbackResults <- rt.Invoke(ctx, GrainId{GrainType: "gor.grainTimerProbe", GrainKey: "probe"}, "Activation", nil, &reply) + return nil + } + rt = startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{}, nil); err != nil { + t.Fatal(err) + } + synctest.Wait() + if err := receiveGrainTimerTestValue(t, callbackResults, "self Call result"); !errors.Is(err, ErrCallCycle) { + t.Fatalf("self Call error = %v, want ErrCallCycle", err) + } + }) +} + +func TestGrainTimerKeepAliveUsesCallbackCompletion(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(900, 0).UTC()) + created := make(chan GrainTimer, 1) + factory := &grainTimerProbeFactory{timerCreated: created} + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(5*time.Second), WithEvictionInterval(time.Second)) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{dueTime: time.Second, period: time.Second, keepAlive: true}, nil); err != nil { + t.Fatal(err) + } + timer := <-created + for range 7 { + sourceClock.Advance(time.Second) + synctest.Wait() + } + if got := len(rt.Activations()); got != 1 { + t.Fatalf("Activations with completed keep-alive ticks = %d, want 1", got) + } + timer.Stop() + sourceClock.Advance(6 * time.Second) + synctest.Wait() + if got := len(rt.Activations()); got != 0 { + t.Fatalf("Activations after keep-alive Stop = %d, want 0", got) + } + }) +} + +func TestGrainTimerKeepAliveRefreshesAfterBlockedCallbackCompletes(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(950, 0).UTC()) + started := make(chan struct{}, 1) + release := make(chan struct{}) + factory := &grainTimerProbeFactory{callback: func(*grainTimerProbeGrain, context.Context, int) error { + started <- struct{}{} + <-release + return nil + }} + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(5*time.Second), WithEvictionInterval(time.Second)) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{dueTime: time.Second, keepAlive: true}, nil); err != nil { + t.Fatal(err) + } + sourceClock.Advance(time.Second) + synctest.Wait() + <-started + sourceClock.Advance(5 * time.Second) + synctest.Wait() + close(release) + synctest.Wait() + + sourceClock.Advance(4 * time.Second) + synctest.Wait() + if got := len(rt.Activations()); got != 1 { + t.Fatalf("Activations four seconds after callback completion = %d, want 1", got) + } + sourceClock.Advance(2 * time.Second) + synctest.Wait() + if got := len(rt.Activations()); got != 0 { + t.Fatalf("Activations after refreshed idle timeout = %d, want 0", got) + } + }) +} + +func TestGrainTimerWithoutKeepAliveDoesNotPreventIdleDeactivation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(975, 0).UTC()) + created := make(chan GrainTimer, 1) + var callbacks atomic.Int32 + factory := &grainTimerProbeFactory{ + timerCreated: created, + callback: func(*grainTimerProbeGrain, context.Context, int) error { + callbacks.Add(1) + return nil + }, + } + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(3*time.Second), WithEvictionInterval(time.Second)) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{dueTime: time.Second, period: time.Second}, nil); err != nil { + t.Fatal(err) + } + timer := <-created + for range 5 { + sourceClock.Advance(time.Second) + synctest.Wait() + } + if got := callbacks.Load(); got == 0 { + t.Fatal("non-keep-alive Grain Timer did not run before idle deactivation") + } + if got := len(rt.Activations()); got != 0 { + t.Fatalf("Activations with a non-keep-alive Grain Timer = %d, want 0", got) + } + if err := timer.Change(0, 0); !errors.Is(err, ErrGrainTimerStopped) { + t.Fatalf("Change after idle deactivation = %v, want ErrGrainTimerStopped", err) + } + }) +} + +func TestGrainTimerOneShotRunsOnce(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + sourceClock := clock.NewFake(time.Unix(1000, 0).UTC()) + ticks := make(chan int, 2) + factory := &grainTimerProbeFactory{callback: func(_ *grainTimerProbeGrain, _ context.Context, tick int) error { + ticks <- tick + return nil + }} + rt := startGrainTimerProbe(t, sourceClock, factory, WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + if err := invokeGrainTimerProbe(rt, "Start", &grainTimerProbeOptions{}, nil); err != nil { + t.Fatal(err) + } + synctest.Wait() + if tick := receiveGrainTimerTestValue(t, ticks, "one-shot Grain Timer tick"); tick != 1 { + t.Fatalf("one-shot tick = %d", tick) + } + sourceClock.Advance(24 * time.Hour) + synctest.Wait() + select { + case tick := <-ticks: + t.Fatalf("one-shot timer repeated at tick %d", tick) + default: + } + }) +} diff --git a/internal/codegen/load.go b/internal/codegen/load.go index c3dc349..6b2a882 100644 --- a/internal/codegen/load.go +++ b/internal/codegen/load.go @@ -52,17 +52,31 @@ func Load(pattern string) (Loaded, error) { } for _, specification := range genDecl.Specs { typeSpec := specification.(*ast.TypeSpec) - if !hasGrainMarker(genDecl, typeSpec) { + grainType, marked, err := grainMarker(pkg, genDecl, typeSpec) + if err != nil { + return Loaded{}, err + } + if !marked { continue } + if pkg.Name == "main" { + return Loaded{}, locatedError(pkg.Fset, typeSpec.Pos(), "Grain interface %s must be in an importable package, not package main", typeSpec.Name.Name) + } + if !ast.IsExported(typeSpec.Name.Name) { + return Loaded{}, locatedError(pkg.Fset, typeSpec.Pos(), "Grain interface %s must be exported", typeSpec.Name.Name) + } + if typeSpec.TypeParams != nil && typeSpec.TypeParams.NumFields() != 0 { + return Loaded{}, locatedError(pkg.Fset, typeSpec.Pos(), "Grain interface %s must not have type parameters", typeSpec.Name.Name) + } if _, ok := typeSpec.Type.(*ast.InterfaceType); !ok { return Loaded{}, locatedError(pkg.Fset, typeSpec.Pos(), "%s.%s is marked gor:grain but is not an interface", pkg.Name, typeSpec.Name.Name) } - entity, err := loadInterface(pkg, typeSpec, imports) + grain, err := loadInterface(pkg, typeSpec, imports) if err != nil { return Loaded{}, err } - pending = append(pending, entity) + grain.grainType = grainType + pending = append(pending, grain) } } } @@ -71,13 +85,13 @@ func Load(pattern string) (Loaded, error) { } // Import names cannot be chosen while loading: whether two packages // collide is only known once every signature has been collected. - aliases := effectiveImportNames(imports, pkg.Name, pkg.PkgPath) + aliases := effectiveImportNames(imports, pkg.Name, pkg.PkgPath, generatedNames(pending)) if alias, ok := aliases[pkg.PkgPath]; ok { model.SourceImportName = alias } model.Interfaces = make([]Interface, len(pending)) - for i, entity := range pending { - model.Interfaces[i] = materialize(entity, aliases) + for i, grain := range pending { + model.Interfaces[i] = materialize(grain, aliases) } for path, imported := range imports { if alias, ok := aliases[path]; ok { @@ -97,8 +111,9 @@ func Load(pattern string) (Loaded, error) { // type strings are only rendered once every import has been collected and // aliases have been decided. type pendingInterface struct { - name string - methods []pendingMethod + name string + grainType string + methods []pendingMethod } type pendingMethod struct { @@ -122,23 +137,32 @@ func loadInterface(pkg *packages.Package, specification *ast.TypeSpec, imports m if !ok { return pendingInterface{}, locatedError(pkg.Fset, specification.Pos(), "type %s has no type information", specification.Name.Name) } - entity, ok := typeName.Type().Underlying().(*types.Interface) + grain, ok := typeName.Type().Underlying().(*types.Interface) if !ok { return pendingInterface{}, locatedError(pkg.Fset, specification.Pos(), "type %s is not an interface", specification.Name.Name) } - model := pendingInterface{name: specification.Name.Name, methods: make([]pendingMethod, entity.NumMethods())} - for i := 0; i < entity.NumMethods(); i++ { - method := entity.Method(i) + model := pendingInterface{name: specification.Name.Name, methods: make([]pendingMethod, grain.NumMethods())} + for i := 0; i < grain.NumMethods(); i++ { + method := grain.Method(i) + if !method.Exported() { + return pendingInterface{}, locatedError(pkg.Fset, method.Pos(), "%s.%s method must be exported", model.name, method.Name()) + } signature, ok := method.Type().(*types.Signature) if !ok { return pendingInterface{}, locatedError(pkg.Fset, method.Pos(), "%s.%s has no method signature", model.name, method.Name()) } + if signature.Variadic() { + return pendingInterface{}, locatedError(pkg.Fset, method.Pos(), "%s.%s must not be variadic", model.name, method.Name()) + } if signature.Params().Len() == 0 || !isContext(signature.Params().At(0).Type()) { return pendingInterface{}, locatedError(pkg.Fset, method.Pos(), "%s.%s must have context.Context as its first parameter", model.name, method.Name()) } if signature.Results().Len() == 0 || !isError(signature.Results().At(signature.Results().Len()-1).Type()) { return pendingInterface{}, locatedError(pkg.Fset, method.Pos(), "%s.%s must have error as its last result", model.name, method.Name()) } + if issue := contractAccessibilityIssue(signature); issue != nil { + return pendingInterface{}, locatedError(pkg.Fset, method.Pos(), "%s.%s uses %s %s", model.name, method.Name(), issue.kind, issue.name) + } loaded := pendingMethod{name: method.Name(), reminder: isReminderMethod(signature)} for parameter := 0; parameter < signature.Params().Len(); parameter++ { variable := signature.Params().At(parameter) @@ -158,6 +182,128 @@ func loadInterface(pkg *packages.Package, specification *ast.TypeSpec, imports m return model, nil } +type accessibilityIssue struct { + kind string + name string +} + +func contractAccessibilityIssue(signature *types.Signature) *accessibilityIssue { + seen := make(map[types.Type]bool) + for i := 0; i < signature.Params().Len(); i++ { + if issue := inspectContractType(signature.Params().At(i).Type(), seen); issue != nil { + return issue + } + } + for i := 0; i < signature.Results().Len(); i++ { + if issue := inspectContractType(signature.Results().At(i).Type(), seen); issue != nil { + return issue + } + } + return nil +} + +func inspectContractType(value types.Type, seen map[types.Type]bool) *accessibilityIssue { + if value == nil || seen[value] { + return nil + } + seen[value] = true + + switch value := value.(type) { + case *types.Basic: + return nil + case *types.Alias: + if object := value.Obj(); object.Pkg() != nil && !object.Exported() { + return &accessibilityIssue{kind: "unexported contract type", name: contractTypeName(value)} + } + return inspectTypeList(value.TypeArgs(), seen) + case *types.Named: + if object := value.Obj(); object.Pkg() != nil && !object.Exported() { + return &accessibilityIssue{kind: "unexported contract type", name: contractTypeName(value)} + } + return inspectTypeList(value.TypeArgs(), seen) + case *types.Pointer: + return inspectContractType(value.Elem(), seen) + case *types.Slice: + return inspectContractType(value.Elem(), seen) + case *types.Array: + return inspectContractType(value.Elem(), seen) + case *types.Map: + if issue := inspectContractType(value.Key(), seen); issue != nil { + return issue + } + return inspectContractType(value.Elem(), seen) + case *types.Chan: + return inspectContractType(value.Elem(), seen) + case *types.Struct: + for i := 0; i < value.NumFields(); i++ { + field := value.Field(i) + if field.Name() != "_" && field.Pkg() != nil && !field.Exported() { + return &accessibilityIssue{kind: "unexported contract field", name: field.Name()} + } + if issue := inspectContractType(field.Type(), seen); issue != nil { + return issue + } + } + case *types.Interface: + value = value.Complete() + for i := 0; i < value.NumEmbeddeds(); i++ { + if issue := inspectContractType(value.EmbeddedType(i), seen); issue != nil { + return issue + } + } + for i := 0; i < value.NumMethods(); i++ { + method := value.Method(i) + if !method.Exported() { + return &accessibilityIssue{kind: "unexported contract method", name: method.Name()} + } + if issue := inspectContractType(method.Type(), seen); issue != nil { + return issue + } + } + case *types.Signature: + if value.TypeParams() != nil && value.TypeParams().Len() != 0 { + return &accessibilityIssue{kind: "unsupported contract type parameters", name: contractTypeName(value)} + } + if issue := inspectContractType(value.Params(), seen); issue != nil { + return issue + } + return inspectContractType(value.Results(), seen) + case *types.Tuple: + for i := 0; i < value.Len(); i++ { + if issue := inspectContractType(value.At(i).Type(), seen); issue != nil { + return issue + } + } + case *types.TypeParam: + return &accessibilityIssue{kind: "unsupported contract type parameter", name: value.Obj().Name()} + case *types.Union: + for i := 0; i < value.Len(); i++ { + if issue := inspectContractType(value.Term(i).Type(), seen); issue != nil { + return issue + } + } + } + return nil +} + +func inspectTypeList(values *types.TypeList, seen map[types.Type]bool) *accessibilityIssue { + if values == nil { + return nil + } + for i := 0; i < values.Len(); i++ { + if issue := inspectContractType(values.At(i), seen); issue != nil { + return issue + } + } + return nil +} + +func contractTypeName(value types.Type) string { + return types.TypeString(value, func(pkg *types.Package) string { + return pkg.Name() + }) +} + func isReminderMethod(signature *types.Signature) bool { if signature.Params().Len() != 2 || signature.Results().Len() != 1 { return false @@ -188,7 +334,7 @@ func recordImports(value types.Type, imports map[string]Import) { // materialize renders pending types into model type strings, qualifying each // package by the alias effectiveImportNames assigned to it, if any. -func materialize(entity pendingInterface, aliases map[string]string) Interface { +func materialize(grain pendingInterface, aliases map[string]string) Interface { qualifier := func(imported *types.Package) string { if imported == nil { return "" @@ -198,8 +344,8 @@ func materialize(entity pendingInterface, aliases map[string]string) Interface { } return imported.Name() } - model := Interface{Name: entity.name, Methods: make([]Method, len(entity.methods))} - for i, method := range entity.methods { + model := Interface{Name: grain.name, GrainType: grain.grainType, Methods: make([]Method, len(grain.methods))} + for i, method := range grain.methods { loaded := Method{Name: method.name, Reminder: method.reminder} for _, parameter := range method.params { loaded.Params = append(loaded.Params, Parameter{Name: parameter.name, Type: types.TypeString(parameter.typ, qualifier)}) @@ -221,21 +367,47 @@ var templateImportPaths = map[string]bool{ "github.com/suraciii/gor": true, } +// generatedNames returns all package-level names that the generator will use. +func generatedNames(grains []pendingInterface) map[string]bool { + names := map[string]bool{ + "generatedCodeVersion": true, + "Install": true, + } + for _, grain := range grains { + for _, suffix := range []string{"GrainType", "Proxy", "Dispatch", "NewProxy", "NewCall", "NewReminderCall"} { + names[generatedGrainName(grain.name, suffix)] = true + } + names["Install"+grain.name] = true + for _, method := range grain.methods { + names[generatedMethodName(grain.name, method.name, "Request")] = true + names[generatedMethodName(grain.name, method.name, "Reply")] = true + } + } + return names +} + // effectiveImportNames decides the name under which each import appears in // the generated file: the package's own name, or an alias when that name // would collide with another import or with a name the generated file uses -// itself (the template's context, fmt and gor imports, and the source package +// itself (the template imports, generated declarations, and the source package // name). Only colliding imports are aliased; everything else keeps its // package name, so adding a new import never churns existing generated // output. The source package's own import line is part of the same -// allocation: when its name is one of the template's fixed names, the map +// allocation: when its name is reserved by the generated file, the map // holds an alias for the source import path. The result is a deterministic // function of the import set: the same input always yields the same aliases. -func effectiveImportNames(imports map[string]Import, sourcePackageName, sourceImportPath string) map[string]string { +func effectiveImportNames(imports map[string]Import, sourcePackageName, sourceImportPath string, generated map[string]bool) map[string]string { reserved := map[string]bool{ "context": true, "fmt": true, "gor": true, + "init": true, + } + for _, name := range types.Universe.Names() { + reserved[name] = true + } + for name := range generated { + reserved[name] = true } used := make(map[string]bool, len(reserved)+len(imports)+1) for name := range reserved { @@ -255,9 +427,8 @@ func effectiveImportNames(imports map[string]Import, sourcePackageName, sourceIm conflicting = append(conflicting, imported) } } - // The source import line can only collide with the template's fixed - // names: a signature import that shares the source name is aliased away - // instead, so aliasing the source here never churns existing artifacts. + // A signature import that shares the source name is aliased away instead, + // so aliasing the source here never churns existing artifacts. if reserved[sourcePackageName] { conflicting = append(conflicting, Import{Name: sourcePackageName, Path: sourceImportPath}) } @@ -331,7 +502,11 @@ func aliasCandidate(imported Import, depth int) string { if depth > len(segments) { return "" } - return sanitizeIdentifier(strings.Join(segments[len(segments)-depth:], "")) + candidate := sanitizeIdentifier(strings.Join(segments[len(segments)-depth:], "")) + if candidate == "_" || candidate == "init" || !token.IsIdentifier(candidate) { + return "" + } + return candidate } // sanitizeIdentifier keeps letters, digits and underscores and replaces every @@ -366,14 +541,41 @@ func isError(value types.Type) bool { return types.Identical(value, types.Universe.Lookup("error").Type()) } -func hasGrainMarker(declaration *ast.GenDecl, specification *ast.TypeSpec) bool { - return commentsHaveMarker(declaration.Doc) || commentsHaveMarker(specification.Doc) +func grainMarker(pkg *packages.Package, declaration *ast.GenDecl, specification *ast.TypeSpec) (string, bool, error) { + var markers []parsedGrainMarker + for _, group := range []*ast.CommentGroup{declaration.Doc, specification.Doc} { + markers = append(markers, commentsWithGrainMarker(group)...) + } + if len(markers) == 0 { + return "", false, nil + } + if markers[0].err != nil { + return "", false, locatedError(pkg.Fset, markers[0].pos, "%s.%s has invalid gor:grain marker: %s", pkg.Name, specification.Name.Name, markers[0].err) + } + if len(markers) > 1 { + return "", false, locatedError(pkg.Fset, markers[1].pos, "%s.%s has repeated gor:grain markers", pkg.Name, specification.Name.Name) + } + value := markers[0].value + if value == "" { + value = pkg.Name + "." + specification.Name.Name + } + if err := validateGrainType(value); err != nil { + return "", false, locatedError(pkg.Fset, markers[0].pos, "%s.%s has invalid GrainType %q: %s", pkg.Name, specification.Name.Name, value, err) + } + return value, true, nil +} + +type parsedGrainMarker struct { + value string + pos token.Pos + err error } -func commentsHaveMarker(group *ast.CommentGroup) bool { +func commentsWithGrainMarker(group *ast.CommentGroup) []parsedGrainMarker { if group == nil { - return false + return nil } + var markers []parsedGrainMarker for _, comment := range group.List { text := strings.TrimSpace(comment.Text) if strings.HasPrefix(text, "//") { @@ -381,11 +583,60 @@ func commentsHaveMarker(group *ast.CommentGroup) bool { } else if strings.HasPrefix(text, "/*") && strings.HasSuffix(text, "*/") { text = strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(text, "/*"), "*/")) } - if text == "gor:grain" { - return true + if !strings.HasPrefix(text, "gor:grain") { + continue } + value, err := parseGrainMarker(text) + markers = append(markers, parsedGrainMarker{value: value, pos: comment.Pos(), err: err}) + } + return markers +} + +func parseGrainMarker(text string) (string, error) { + const prefix = "gor:grain" + remainder := strings.TrimPrefix(text, prefix) + if remainder == text { + return "", fmt.Errorf("marker must start with gor:grain") } - return false + if remainder == "" { + return "", nil + } + if !strings.ContainsAny(remainder[:1], " \t\r\n") { + return "", fmt.Errorf("marker must be bare or contain one GrainType") + } + remainder = strings.TrimSpace(remainder) + if remainder == "" { + return "", nil + } + if strings.ContainsAny(remainder, " \t\r\n") { + return "", fmt.Errorf("marker must be bare or contain one GrainType") + } + return remainder, nil +} + +func validateGrainType(value string) error { + if len(value) == 0 || len(value) > 255 { + return fmt.Errorf("must contain 1 to 255 ASCII bytes") + } + for index := 0; index < len(value); index++ { + if value[index] > 0x7f { + return fmt.Errorf("must contain only ASCII characters") + } + if index == 0 || index == len(value)-1 { + if !isASCIIAlphaNumeric(value[index]) { + return fmt.Errorf("first and last bytes must be letters or digits") + } + continue + } + if !isASCIIAlphaNumeric(value[index]) && !strings.ContainsRune("./_-", rune(value[index])) { + return fmt.Errorf("middle bytes must be letters, digits, '.', '/', '_', or '-'") + } + } + return nil +} + +func isASCIIAlphaNumeric(value byte) bool { + return value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z' || value >= '0' && value <= '9' } func locatedError(fileSet *token.FileSet, position token.Pos, format string, args ...any) error { diff --git a/internal/codegen/load_test.go b/internal/codegen/load_test.go new file mode 100644 index 0000000..ff445c6 --- /dev/null +++ b/internal/codegen/load_test.go @@ -0,0 +1,161 @@ +package codegen + +import ( + "strings" + "testing" +) + +const fixtureModule = "github.com/suraciii/gor/internal/codegen/testfixture/" + +func TestLoad_GrainTypes(t *testing.T) { + loaded, err := Load(fixtureModule + "domain") + if err != nil { + t.Fatal(err) + } + if len(loaded.Model.Interfaces) != 2 { + t.Fatalf("interfaces = %d, want 2", len(loaded.Model.Interfaces)) + } + if got := loaded.Model.Interfaces[0].GrainType; got != "domain.Account" { + t.Fatalf("Account GrainType = %q, want %q", got, "domain.Account") + } + if got := loaded.Model.Interfaces[1].GrainType; got != "finance.ledger" { + t.Fatalf("Ledger GrainType = %q, want %q", got, "finance.ledger") + } +} + +func TestLoad_GrainMarkerErrorsHaveSourcePosition(t *testing.T) { + tests := []struct { + name string + packageName string + line string + message string + }{ + {name: "malformed", packageName: "invalidmarker", line: "domain.go:5:", message: "invalid GrainType"}, + {name: "repeated", packageName: "repeatedmarker", line: "domain.go:6:", message: "repeated gor:grain markers"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := Load(fixtureModule + test.packageName) + if err == nil { + t.Fatal("Load accepted an invalid gor:grain marker") + } + message := err.Error() + if !strings.Contains(message, test.line) { + t.Fatalf("error = %q, want source position %q", message, test.line) + } + if !strings.Contains(message, test.message) { + t.Fatalf("error = %q, want %q", message, test.message) + } + }) + } +} + +func TestLoad_ContractErrorsHaveSourcePosition(t *testing.T) { + tests := []struct { + name string + packageName string + line string + message string + }{ + {name: "generic interface", packageName: "genericinterface", line: "domain.go:6:", message: "must not have type parameters"}, + {name: "unexported interface", packageName: "unexportedinterface", line: "domain.go:6:", message: "must be exported"}, + {name: "variadic method", packageName: "variadicmethod", line: "domain.go:7:", message: "must not be variadic"}, + {name: "unexported method", packageName: "unexportedmethod", line: "domain.go:7:", message: "method must be exported"}, + {name: "nested unexported type", packageName: "unexportedtype", line: "domain.go:9:", message: "uses unexported contract type"}, + {name: "unexported type argument", packageName: "unexportedtypeargument", line: "domain.go:11:", message: "uses unexported contract type"}, + {name: "anonymous unexported field", packageName: "unexportedfield", line: "domain.go:7:", message: "uses unexported contract field"}, + {name: "anonymous unexported method", packageName: "unexportedcontractmethod", line: "domain.go:7:", message: "uses unexported contract method"}, + {name: "anonymous unexported embedded type", packageName: "unexportedembedded", line: "domain.go:11:", message: "uses unexported contract type"}, + {name: "main package", packageName: "mainpackage", line: "domain.go:6:", message: "not package main"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := Load(fixtureModule + test.packageName) + if err == nil { + t.Fatal("Load accepted an invalid Grain contract") + } + message := err.Error() + if !strings.Contains(message, test.line) { + t.Fatalf("error = %q, want source position %q", message, test.line) + } + if !strings.Contains(message, test.message) { + t.Fatalf("error = %q, want %q", message, test.message) + } + }) + } +} + +func TestAliasCandidateRejectsInvalidImportNames(t *testing.T) { + tests := []struct { + name string + path string + }{ + {name: "keyword", path: "example.com/type"}, + {name: "blank identifier", path: "example.com/-"}, + {name: "init", path: "example.com/init"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := aliasCandidate(Import{Name: "context", Path: test.path}, 1); got != "" { + t.Fatalf("aliasCandidate(%q) = %q, want empty", test.path, got) + } + }) + } +} + +func TestValidateGrainType(t *testing.T) { + valid := []string{"a", "Account", "billing.account", "billing/account_2-v1"} + for _, value := range valid { + if err := validateGrainType(value); err != nil { + t.Errorf("validateGrainType(%q) = %v, want nil", value, err) + } + } + invalid := []string{"", "_account", "account_", "billing account", "billing:account", "é"} + for _, value := range invalid { + if err := validateGrainType(value); err == nil { + t.Errorf("validateGrainType(%q) = nil, want error", value) + } + } + if err := validateGrainType(strings.Repeat("a", 255)); err != nil { + t.Fatalf("validateGrainType(255 bytes) = %v, want nil", err) + } + if err := validateGrainType(strings.Repeat("a", 256)); err == nil { + t.Fatal("validateGrainType(256 bytes) = nil, want error") + } +} + +func FuzzParseGrainMarker(f *testing.F) { + for _, marker := range []string{ + "gor:grain", + "gor:grain ", + "gor:grain finance.account", + "gor:grain\tfinance/account_2-v1", + "gor:grainfinance.account", + "gor:grain finance account", + "grain", + "", + } { + f.Add(marker) + } + + f.Fuzz(func(t *testing.T, marker string) { + value, err := parseGrainMarker(marker) + if err != nil { + return + } + if !strings.HasPrefix(marker, "gor:grain") { + t.Fatalf("successful marker = %q, want gor:grain prefix", marker) + } + if strings.ContainsAny(value, " \t\r\n") { + t.Fatalf("parsed GrainType = %q, want one token", value) + } + canonical := "gor:grain" + if value != "" { + canonical += " " + value + } + roundTrip, err := parseGrainMarker(canonical) + if err != nil || roundTrip != value { + t.Fatalf("canonical marker %q = (%q, %v), want %q", canonical, roundTrip, err, value) + } + }) +} diff --git a/internal/codegen/model.go b/internal/codegen/model.go index ef37cdd..d8e335a 100644 --- a/internal/codegen/model.go +++ b/internal/codegen/model.go @@ -15,8 +15,9 @@ type Import struct { } type Interface struct { - Name string - Methods []Method + Name string + GrainType string + Methods []Method } type Method struct { diff --git a/internal/codegen/render.go b/internal/codegen/render.go index 6b53938..5a79eff 100644 --- a/internal/codegen/render.go +++ b/internal/codegen/render.go @@ -7,8 +7,8 @@ import ( "sort" "strings" "text/template" - "unicode" - "unicode/utf8" + + "github.com/suraciii/gor/internal/generatedcode" ) type renderModel struct { @@ -16,6 +16,7 @@ type renderModel struct { SourcePackage string SourceImportPath string SourceImportName string + GeneratedVersion int Imports []renderImport Interfaces []renderInterface } @@ -26,11 +27,15 @@ type renderImport struct { } type renderInterface struct { - Name string - ProxyName string - DispatchName string - ConstructorName string - Methods []renderMethod + Name string + GrainType string + GrainTypeName string + ProxyName string + DispatchName string + ConstructorName string + CallName string + ReminderCallName string + Methods []renderMethod } type renderMethod struct { @@ -64,7 +69,9 @@ type renderAssignment struct { func Render(model Model) ([]byte, error) { prepared := prepare(model) var source bytes.Buffer - _ = generatedTemplate.Execute(&source, prepared) + if err := generatedTemplate.Execute(&source, prepared); err != nil { + return nil, fmt.Errorf("render generated source: %w", err) + } formatted, err := format.Source(source.Bytes()) if err != nil { return nil, fmt.Errorf("format generated source: %w", err) @@ -82,19 +89,24 @@ func prepare(model Model) renderModel { SourcePackage: sourcePackage, SourceImportPath: model.SourceImportPath, SourceImportName: model.SourceImportName, + GeneratedVersion: generatedcode.Version, Imports: renderImports(model), Interfaces: make([]renderInterface, len(model.Interfaces)), } - for i, entity := range model.Interfaces { + for i, grain := range model.Interfaces { prepared.Interfaces[i] = renderInterface{ - Name: entity.Name, - ProxyName: lowerFirst(entity.Name) + "Proxy", - DispatchName: "dispatch" + entity.Name, - ConstructorName: "new" + entity.Name + "Proxy", - Methods: make([]renderMethod, len(entity.Methods)), + Name: grain.Name, + GrainType: grain.GrainType, + GrainTypeName: generatedGrainName(grain.Name, "GrainType"), + ProxyName: generatedGrainName(grain.Name, "Proxy"), + DispatchName: generatedGrainName(grain.Name, "Dispatch"), + ConstructorName: generatedGrainName(grain.Name, "NewProxy"), + CallName: generatedGrainName(grain.Name, "NewCall"), + ReminderCallName: generatedGrainName(grain.Name, "NewReminderCall"), + Methods: make([]renderMethod, len(grain.Methods)), } - for j, method := range entity.Methods { - prepared.Interfaces[i].Methods[j] = prepareMethod(entity, method) + for j, method := range grain.Methods { + prepared.Interfaces[i].Methods[j] = prepareMethod(grain, method) } } return prepared @@ -113,22 +125,23 @@ func renderImports(model Model) []renderImport { return imports } -func prepareMethod(entity Interface, method Method) renderMethod { +func prepareMethod(grain Interface, method Method) renderMethod { + params := generatedParameters(method.Params) values := method.Results[:len(method.Results)-1] rendered := renderMethod{ Name: method.Name, Reminder: method.Reminder, - Params: joinParameters(method.Params), + Params: joinParameters(params), Results: joinResults(method.Results), - ContextName: method.Params[0].Name, - ArgsName: lowerFirst(entity.Name) + method.Name + "Request", - ReplyName: lowerFirst(entity.Name) + method.Name + "Reply", + ContextName: params[0].Name, + ArgsName: generatedMethodName(grain.Name, method.Name, "Request"), + ReplyName: generatedMethodName(grain.Name, method.Name, "Reply"), HasValues: len(values) > 0, DispatchCall: dispatchCall(method, values), } - rendered.Args = joinArgs(rendered.ArgsName, method.Params[1:]) + rendered.Args = joinArgs(rendered.ArgsName, params[1:]) rendered.ReplyPointer = "&reply" - for i, param := range method.Params[1:] { + for i, param := range params[1:] { rendered.ArgsFields = append(rendered.ArgsFields, renderResult{Name: fmt.Sprintf("A%d", i), Type: param.Type}) } for i, result := range values { @@ -146,6 +159,26 @@ func prepareMethod(entity Interface, method Method) renderMethod { return rendered } +func generatedParameters(params []Parameter) []Parameter { + generated := make([]Parameter, len(params)) + for i, param := range params { + name := "ctx" + if i > 0 { + name = fmt.Sprintf("arg%d", i-1) + } + generated[i] = Parameter{Name: name, Type: param.Type} + } + return generated +} + +func generatedMethodName(grainName, methodName, suffix string) string { + return fmt.Sprintf("gorgen%d_%s%d_%s%s", len(grainName), grainName, len(methodName), methodName, suffix) +} + +func generatedGrainName(grainName, suffix string) string { + return fmt.Sprintf("gorgen%d_%s%s", len(grainName), grainName, suffix) +} + func dispatchCall(method Method, values []string) string { resultNames := make([]string, len(values)+1) for i := range values { @@ -190,12 +223,9 @@ func joinArgs(name string, params []Parameter) string { return "&" + name + "{" + strings.Join(parts, ", ") + "}" } -func lowerFirst(value string) string { - runeValue, size := utf8.DecodeRuneInString(value) - return string(unicode.ToLower(runeValue)) + value[size:] -} +var generatedTemplate = template.Must(template.New("generated").Parse(`// Code generated by gorgen. DO NOT EDIT. -var generatedTemplate = template.Must(template.New("generated").Parse(`package {{.PackageName}} +package {{.PackageName}} import ( "context" @@ -207,7 +237,11 @@ import ( {{end}} ) -{{range .Interfaces}}{{ $entity := . }} +const generatedCodeVersion = {{.GeneratedVersion}} + +{{range .Interfaces}}{{ $grain := . }} +const {{$grain.GrainTypeName}} gor.GrainType = {{printf "%q" $grain.GrainType}} + type {{.ProxyName}} struct { id gor.GrainId rt gor.Invoker @@ -221,14 +255,14 @@ type {{.ProxyName}} struct { {{if .ReplyFields}}type {{.ReplyName}} struct { {{range .ReplyFields}} {{.Name}} {{.Type}} {{end}}}{{else}}type {{.ReplyName}} struct{}{{end}} -func (p *{{$entity.ProxyName}}) {{.Name}}({{.Params}}) {{.Results}} { +func (p *{{$grain.ProxyName}}) {{.Name}}({{.Params}}) {{.Results}} { var reply {{.ReplyName}} err := p.rt.Invoke({{.ContextName}}, p.id, "{{.Name}}", {{.Args}}, {{.ReplyPointer}}) {{if .HasValues}} return {{.ResultNames}}, err {{else}} return err {{end}}} {{end}} -func {{$entity.DispatchName}}(ctx context.Context, instance {{$.SourcePackage}}.{{$entity.Name}}, method string, args any, reply any) error { +func {{$grain.DispatchName}}(ctx context.Context, instance {{$.SourcePackage}}.{{$grain.Name}}, method string, args any, reply any) error { switch method { {{range .Methods}} case "{{.Name}}": {{if .ArgsFields}} typedArgs := args.(*{{.ArgsName}}) @@ -241,7 +275,7 @@ func {{$entity.DispatchName}}(ctx context.Context, instance {{$.SourcePackage}}. } } -func new{{ $entity.Name }}Call(method string) (args any, reply any) { +func {{$grain.CallName}}(method string) (args any, reply any) { switch method { {{range .Methods}} case "{{.Name}}": return &{{.ArgsName}}{}, &{{.ReplyName}}{} @@ -250,7 +284,7 @@ func new{{ $entity.Name }}Call(method string) (args any, reply any) { } } -func new{{ $entity.Name }}ReminderCall(method string, status gor.TickStatus) (args any, reply any) { +func {{$grain.ReminderCallName}}(method string, status gor.TickStatus) (args any, reply any) { switch method { {{range .Methods}}{{if .Reminder}} case "{{.Name}}": return &{{.ArgsName}}{A0: status}, &{{.ReplyName}}{} @@ -259,17 +293,23 @@ func new{{ $entity.Name }}ReminderCall(method string, status gor.TickStatus) (ar } } -func {{$entity.ConstructorName}}(rt gor.Invoker, id gor.GrainId) {{$.SourcePackage}}.{{$entity.Name}} { +func {{$grain.ConstructorName}}(rt gor.Invoker, id gor.GrainId) {{$.SourcePackage}}.{{$grain.Name}} { return &{{.ProxyName}}{id: id, rt: rt} } +{{end}} +{{range .Interfaces}}// Install{{.Name}} installs the generated bindings for {{.Name}} in rt. +func Install{{.Name}}(rt *gor.Runtime) error { + return gor.InstallType[{{$.SourcePackage}}.{{.Name}}](rt, generatedCodeVersion, {{.GrainTypeName}}, {{.DispatchName}}, {{.ConstructorName}}, {{.CallName}}, {{.ReminderCallName}}) +} + {{end}} // Install installs the generated Grain bindings in rt. // Call it once after creating rt and before registering or referencing any of // the generated Grain types. After it returns nil, gor.Register and gor.Ref // can use those types with rt. func Install(rt *gor.Runtime) error { -{{range .Interfaces}} if err := gor.InstallType[{{$.SourcePackage}}.{{.Name}}](rt, {{.DispatchName}}, {{.ConstructorName}}, new{{.Name}}Call, new{{.Name}}ReminderCall); err != nil { +{{range .Interfaces}} if err := Install{{.Name}}(rt); err != nil { return err } {{end}} return nil diff --git a/internal/codegen/render_test.go b/internal/codegen/render_test.go index 068f480..b8fa141 100644 --- a/internal/codegen/render_test.go +++ b/internal/codegen/render_test.go @@ -4,16 +4,18 @@ import ( "bytes" "os" "path/filepath" + "strings" "testing" ) func TestRender_MatchesCompilingGolden(t *testing.T) { model := Model{ - PackageName: "generated", + PackageName: "gorgen", SourcePackageName: "domain", SourceImportPath: "github.com/suraciii/gor/internal/codegen/testfixture/domain", Interfaces: []Interface{{ - Name: "Account", + Name: "Account", + GrainType: "domain.Account", Methods: []Method{ { Name: "Lookup", @@ -27,7 +29,8 @@ func TestRender_MatchesCompilingGolden(t *testing.T) { }, }, }, { - Name: "Ledger", + Name: "Ledger", + GrainType: "finance.ledger", Methods: []Method{ { Name: "Balance", @@ -50,3 +53,53 @@ func TestRender_MatchesCompilingGolden(t *testing.T) { t.Fatalf("rendered source differs from golden:\n--- got ---\n%s\n--- want ---\n%s", got, want) } } + +func TestRender_UsesStableNamesForValidContracts(t *testing.T) { + model := Model{ + PackageName: "gorgen", + SourcePackageName: "domain", + SourceImportPath: "example.com/application/domain", + Interfaces: []Interface{ + { + Name: "A", + GrainType: "domain.A", + Methods: []Method{{ + Name: "BC", + Params: []Parameter{ + {Name: "_", Type: "context.Context"}, + {Name: "p", Type: "int"}, + {Name: "reply", Type: "string"}, + }, + Results: []string{"error"}, + }}, + }, + { + Name: "AB", + GrainType: "domain.AB", + Methods: []Method{{ + Name: "C", + Params: []Parameter{{Name: "arg0", Type: "context.Context"}}, + Results: []string{"error"}, + }}, + }, + }, + } + + got, err := Render(model) + if err != nil { + t.Fatal(err) + } + source := string(got) + for _, want := range []string{ + "// Code generated by gorgen. DO NOT EDIT.", + "const generatedCodeVersion = 1", + "type gorgen1_A2_BCRequest struct", + "type gorgen2_AB1_CRequest struct{}", + "func (p *gorgen1_AProxy) BC(ctx context.Context, arg0 int, arg1 string) error", + "return gor.InstallType[domain.A](rt, generatedCodeVersion", + } { + if !strings.Contains(source, want) { + t.Fatalf("generated source misses %q:\n%s", want, source) + } + } +} diff --git a/internal/codegen/testfixture/domain/domain.go b/internal/codegen/testfixture/domain/domain.go index 7b4e6d5..e7c9172 100644 --- a/internal/codegen/testfixture/domain/domain.go +++ b/internal/codegen/testfixture/domain/domain.go @@ -2,11 +2,13 @@ package domain import "context" +//gor:grain type Account interface { Lookup(ctx context.Context, key string) (int64, string, error) Reset(ctx context.Context) error } +//gor:grain finance.ledger type Ledger interface { Balance(ctx context.Context) (int64, error) } diff --git a/internal/codegen/testfixture/generated/generated.go b/internal/codegen/testfixture/generated/generated.go index cc77fee..8d6cffd 100644 --- a/internal/codegen/testfixture/generated/generated.go +++ b/internal/codegen/testfixture/generated/generated.go @@ -1,4 +1,6 @@ -package generated +// Code generated by gorgen. DO NOT EDIT. + +package gorgen import ( "context" @@ -8,39 +10,43 @@ import ( "github.com/suraciii/gor/internal/codegen/testfixture/domain" ) -type accountProxy struct { +const generatedCodeVersion = 1 + +const gorgen7_AccountGrainType gor.GrainType = "domain.Account" + +type gorgen7_AccountProxy struct { id gor.GrainId rt gor.Invoker } -type accountLookupRequest struct { +type gorgen7_Account6_LookupRequest struct { A0 string } -type accountLookupReply struct { +type gorgen7_Account6_LookupReply struct { R0 int64 R1 string } -func (p *accountProxy) Lookup(ctx context.Context, key string) (int64, string, error) { - var reply accountLookupReply - err := p.rt.Invoke(ctx, p.id, "Lookup", &accountLookupRequest{A0: key}, &reply) +func (p *gorgen7_AccountProxy) Lookup(ctx context.Context, arg0 string) (int64, string, error) { + var reply gorgen7_Account6_LookupReply + err := p.rt.Invoke(ctx, p.id, "Lookup", &gorgen7_Account6_LookupRequest{A0: arg0}, &reply) return reply.R0, reply.R1, err } -type accountResetRequest struct{} -type accountResetReply struct{} +type gorgen7_Account5_ResetRequest struct{} +type gorgen7_Account5_ResetReply struct{} -func (p *accountProxy) Reset(ctx context.Context) error { - var reply accountResetReply - err := p.rt.Invoke(ctx, p.id, "Reset", &accountResetRequest{}, &reply) +func (p *gorgen7_AccountProxy) Reset(ctx context.Context) error { + var reply gorgen7_Account5_ResetReply + err := p.rt.Invoke(ctx, p.id, "Reset", &gorgen7_Account5_ResetRequest{}, &reply) return err } -func dispatchAccount(ctx context.Context, instance domain.Account, method string, args any, reply any) error { +func gorgen7_AccountDispatch(ctx context.Context, instance domain.Account, method string, args any, reply any) error { switch method { case "Lookup": - typedArgs := args.(*accountLookupRequest) - typedReply := reply.(*accountLookupReply) + typedArgs := args.(*gorgen7_Account6_LookupRequest) + typedReply := reply.(*gorgen7_Account6_LookupReply) r0, r1, err := instance.Lookup(ctx, typedArgs.A0) typedReply.R0 = r0 typedReply.R1 = r1 @@ -53,48 +59,50 @@ func dispatchAccount(ctx context.Context, instance domain.Account, method string } } -func newAccountCall(method string) (args any, reply any) { +func gorgen7_AccountNewCall(method string) (args any, reply any) { switch method { case "Lookup": - return &accountLookupRequest{}, &accountLookupReply{} + return &gorgen7_Account6_LookupRequest{}, &gorgen7_Account6_LookupReply{} case "Reset": - return &accountResetRequest{}, &accountResetReply{} + return &gorgen7_Account5_ResetRequest{}, &gorgen7_Account5_ResetReply{} default: return nil, nil } } -func newAccountReminderCall(method string, status gor.TickStatus) (args any, reply any) { +func gorgen7_AccountNewReminderCall(method string, status gor.TickStatus) (args any, reply any) { switch method { default: return nil, nil } } -func newAccountProxy(rt gor.Invoker, id gor.GrainId) domain.Account { - return &accountProxy{id: id, rt: rt} +func gorgen7_AccountNewProxy(rt gor.Invoker, id gor.GrainId) domain.Account { + return &gorgen7_AccountProxy{id: id, rt: rt} } -type ledgerProxy struct { +const gorgen6_LedgerGrainType gor.GrainType = "finance.ledger" + +type gorgen6_LedgerProxy struct { id gor.GrainId rt gor.Invoker } -type ledgerBalanceRequest struct{} -type ledgerBalanceReply struct { +type gorgen6_Ledger7_BalanceRequest struct{} +type gorgen6_Ledger7_BalanceReply struct { R0 int64 } -func (p *ledgerProxy) Balance(ctx context.Context) (int64, error) { - var reply ledgerBalanceReply - err := p.rt.Invoke(ctx, p.id, "Balance", &ledgerBalanceRequest{}, &reply) +func (p *gorgen6_LedgerProxy) Balance(ctx context.Context) (int64, error) { + var reply gorgen6_Ledger7_BalanceReply + err := p.rt.Invoke(ctx, p.id, "Balance", &gorgen6_Ledger7_BalanceRequest{}, &reply) return reply.R0, err } -func dispatchLedger(ctx context.Context, instance domain.Ledger, method string, args any, reply any) error { +func gorgen6_LedgerDispatch(ctx context.Context, instance domain.Ledger, method string, args any, reply any) error { switch method { case "Balance": - typedReply := reply.(*ledgerBalanceReply) + typedReply := reply.(*gorgen6_Ledger7_BalanceReply) r0, err := instance.Balance(ctx) typedReply.R0 = r0 return err @@ -103,24 +111,34 @@ func dispatchLedger(ctx context.Context, instance domain.Ledger, method string, } } -func newLedgerCall(method string) (args any, reply any) { +func gorgen6_LedgerNewCall(method string) (args any, reply any) { switch method { case "Balance": - return &ledgerBalanceRequest{}, &ledgerBalanceReply{} + return &gorgen6_Ledger7_BalanceRequest{}, &gorgen6_Ledger7_BalanceReply{} default: return nil, nil } } -func newLedgerReminderCall(method string, status gor.TickStatus) (args any, reply any) { +func gorgen6_LedgerNewReminderCall(method string, status gor.TickStatus) (args any, reply any) { switch method { default: return nil, nil } } -func newLedgerProxy(rt gor.Invoker, id gor.GrainId) domain.Ledger { - return &ledgerProxy{id: id, rt: rt} +func gorgen6_LedgerNewProxy(rt gor.Invoker, id gor.GrainId) domain.Ledger { + return &gorgen6_LedgerProxy{id: id, rt: rt} +} + +// InstallAccount installs the generated bindings for Account in rt. +func InstallAccount(rt *gor.Runtime) error { + return gor.InstallType[domain.Account](rt, generatedCodeVersion, gorgen7_AccountGrainType, gorgen7_AccountDispatch, gorgen7_AccountNewProxy, gorgen7_AccountNewCall, gorgen7_AccountNewReminderCall) +} + +// InstallLedger installs the generated bindings for Ledger in rt. +func InstallLedger(rt *gor.Runtime) error { + return gor.InstallType[domain.Ledger](rt, generatedCodeVersion, gorgen6_LedgerGrainType, gorgen6_LedgerDispatch, gorgen6_LedgerNewProxy, gorgen6_LedgerNewCall, gorgen6_LedgerNewReminderCall) } // Install installs the generated Grain bindings in rt. @@ -128,10 +146,10 @@ func newLedgerProxy(rt gor.Invoker, id gor.GrainId) domain.Ledger { // the generated Grain types. After it returns nil, gor.Register and gor.Ref // can use those types with rt. func Install(rt *gor.Runtime) error { - if err := gor.InstallType[domain.Account](rt, dispatchAccount, newAccountProxy, newAccountCall, newAccountReminderCall); err != nil { + if err := InstallAccount(rt); err != nil { return err } - if err := gor.InstallType[domain.Ledger](rt, dispatchLedger, newLedgerProxy, newLedgerCall, newLedgerReminderCall); err != nil { + if err := InstallLedger(rt); err != nil { return err } return nil diff --git a/internal/codegen/testfixture/genericinterface/domain.go b/internal/codegen/testfixture/genericinterface/domain.go new file mode 100644 index 0000000..9d89d6f --- /dev/null +++ b/internal/codegen/testfixture/genericinterface/domain.go @@ -0,0 +1,8 @@ +package genericinterface + +import "context" + +//gor:grain +type Account[T any] interface { + Get(context.Context) (T, error) +} diff --git a/internal/codegen/testfixture/invalidmarker/domain.go b/internal/codegen/testfixture/invalidmarker/domain.go new file mode 100644 index 0000000..e9d97bd --- /dev/null +++ b/internal/codegen/testfixture/invalidmarker/domain.go @@ -0,0 +1,8 @@ +package invalidmarker + +import "context" + +//gor:grain _account +type Account interface { + Balance(ctx context.Context) error +} diff --git a/internal/codegen/testfixture/mainpackage/domain.go b/internal/codegen/testfixture/mainpackage/domain.go new file mode 100644 index 0000000..b5ba605 --- /dev/null +++ b/internal/codegen/testfixture/mainpackage/domain.go @@ -0,0 +1,10 @@ +package main + +import "context" + +//gor:grain +type Account interface { + Read(context.Context) error +} + +func main() {} diff --git a/internal/codegen/testfixture/repeatedmarker/domain.go b/internal/codegen/testfixture/repeatedmarker/domain.go new file mode 100644 index 0000000..e892ce4 --- /dev/null +++ b/internal/codegen/testfixture/repeatedmarker/domain.go @@ -0,0 +1,9 @@ +package repeatedmarker + +import "context" + +//gor:grain +//gor:grain billing.account +type Account interface { + Balance(ctx context.Context) error +} diff --git a/internal/codegen/testfixture/unexportedcontractmethod/domain.go b/internal/codegen/testfixture/unexportedcontractmethod/domain.go new file mode 100644 index 0000000..0993088 --- /dev/null +++ b/internal/codegen/testfixture/unexportedcontractmethod/domain.go @@ -0,0 +1,8 @@ +package unexportedcontractmethod + +import "context" + +//gor:grain +type Account interface { + Set(context.Context, interface{ private() }) error +} diff --git a/internal/codegen/testfixture/unexportedembedded/domain.go b/internal/codegen/testfixture/unexportedembedded/domain.go new file mode 100644 index 0000000..b909cb1 --- /dev/null +++ b/internal/codegen/testfixture/unexportedembedded/domain.go @@ -0,0 +1,12 @@ +package domain + +import "context" + +type private interface { + Read(context.Context) error +} + +//gor:grain +type Account interface { + Use(context.Context, interface{ private }) error +} diff --git a/internal/codegen/testfixture/unexportedfield/domain.go b/internal/codegen/testfixture/unexportedfield/domain.go new file mode 100644 index 0000000..69f4651 --- /dev/null +++ b/internal/codegen/testfixture/unexportedfield/domain.go @@ -0,0 +1,8 @@ +package unexportedfield + +import "context" + +//gor:grain +type Account interface { + Set(context.Context, struct{ private int }) error +} diff --git a/internal/codegen/testfixture/unexportedinterface/domain.go b/internal/codegen/testfixture/unexportedinterface/domain.go new file mode 100644 index 0000000..512ee4c --- /dev/null +++ b/internal/codegen/testfixture/unexportedinterface/domain.go @@ -0,0 +1,10 @@ +package unexportedinterface + +import "context" + +//gor:grain +type account interface { + Get(context.Context) (int, error) +} + +var _ account diff --git a/internal/codegen/testfixture/unexportedmethod/domain.go b/internal/codegen/testfixture/unexportedmethod/domain.go new file mode 100644 index 0000000..6c21813 --- /dev/null +++ b/internal/codegen/testfixture/unexportedmethod/domain.go @@ -0,0 +1,8 @@ +package unexportedmethod + +import "context" + +//gor:grain +type Account interface { + get(context.Context) (int, error) +} diff --git a/internal/codegen/testfixture/unexportedtype/domain.go b/internal/codegen/testfixture/unexportedtype/domain.go new file mode 100644 index 0000000..8ca3037 --- /dev/null +++ b/internal/codegen/testfixture/unexportedtype/domain.go @@ -0,0 +1,10 @@ +package unexportedtype + +import "context" + +type privateValue struct{} + +//gor:grain +type Account interface { + Set(context.Context, map[string][]*privateValue) error +} diff --git a/internal/codegen/testfixture/unexportedtypeargument/domain.go b/internal/codegen/testfixture/unexportedtypeargument/domain.go new file mode 100644 index 0000000..add0f65 --- /dev/null +++ b/internal/codegen/testfixture/unexportedtypeargument/domain.go @@ -0,0 +1,12 @@ +package unexportedtypeargument + +import "context" + +type Box[T any] struct{} + +type privateValue struct{} + +//gor:grain +type Account interface { + Set(context.Context, Box[privateValue]) error +} diff --git a/internal/codegen/testfixture/variadicmethod/domain.go b/internal/codegen/testfixture/variadicmethod/domain.go new file mode 100644 index 0000000..c10cb16 --- /dev/null +++ b/internal/codegen/testfixture/variadicmethod/domain.go @@ -0,0 +1,8 @@ +package variadicmethod + +import "context" + +//gor:grain +type Account interface { + Add(context.Context, ...int) error +} diff --git a/internal/generatedcode/version.go b/internal/generatedcode/version.go new file mode 100644 index 0000000..2c94799 --- /dev/null +++ b/internal/generatedcode/version.go @@ -0,0 +1,5 @@ +// Package generatedcode owns the private contract between gorgen and Runtime. +package generatedcode + +// Version changes when generated bindings and Runtime stop being compatible. +const Version = 1 diff --git a/internal/mail/mail.go b/internal/mail/mail.go new file mode 100644 index 0000000..6547a4a --- /dev/null +++ b/internal/mail/mail.go @@ -0,0 +1,218 @@ +// Package mail provides the per-Grain mailboxes gor uses to serialize Calls. +// +// It is an implementation package, not an application dependency. Invoke +// Grains through the root gor package instead of importing mail directly. +package mail + +import ( + "context" + "errors" + "sync" +) + +var ( + ErrOverloaded = errors.New("mailbox overloaded") + ErrClosed = errors.New("mailbox closed") +) + +type Call func(context.Context) (any, error) + +type Result struct { + Value any + Err error +} + +type notDispatchedMarker struct{} + +type Box struct { + in chan *call + done chan struct{} + closedSignal chan struct{} + space chan struct{} + onCanceled func() + + mu sync.Mutex + closed bool +} + +type call struct { + fn Call + reply chan Result + ctx context.Context +} + +// New creates a Box and calls onCanceled after it skips a canceled Call. The +// owner can use the callback to release state held by that Call. +func New(capacity int, onCanceled func()) *Box { + b := &Box{ + in: make(chan *call, capacity), + done: make(chan struct{}), + closedSignal: make(chan struct{}), + space: make(chan struct{}, 1), + onCanceled: onCanceled, + } + go b.run() + return b +} + +func (b *Box) Call(ctx context.Context, fn Call) (any, error) { + result, _ := b.CallResult(ctx, fn) + return result.Value, result.Err +} + +// CallResult returns the result and reports whether that result came from the +// Call callback. False means the mailbox or caller context returned the error. +func (b *Box) CallResult(ctx context.Context, fn Call) (Result, bool) { + c := &call{fn: fn, reply: make(chan Result, 1), ctx: ctx} + if err := b.enqueue(ctx, c, false); err != nil { + return Result{Err: err}, false + } + return waitForResult(ctx, c) +} + +// CallWaitResult waits for mailbox capacity before it enqueues fn. It is for +// Runtime-owned turns that must not be dropped when Application Calls fill the +// bounded mailbox. After enqueue, it waits for the Box to report whether fn +// entered. Canceling ctx prevents a queued fn from entering, but does not hide +// a callback that already entered. +func (b *Box) CallWaitResult(ctx context.Context, fn Call) (Result, bool) { + return b.callWaitResultUntil(ctx, nil, fn) +} + +// CallWaitResultUntil is CallWaitResult with an abrupt-stop signal. A closed +// stop channel lets Runtime infrastructure stop waiting for user code which +// already entered and did not return after cancellation. +func (b *Box) CallWaitResultUntil(ctx context.Context, stop <-chan struct{}, fn Call) (Result, bool) { + return b.callWaitResultUntil(ctx, stop, fn) +} + +func (b *Box) callWaitResultUntil(ctx context.Context, stop <-chan struct{}, fn Call) (Result, bool) { + c := &call{fn: fn, reply: make(chan Result, 1), ctx: ctx} + if err := b.enqueue(ctx, c, true); err != nil { + return Result{Err: err}, false + } + select { + case result := <-c.reply: + return callbackResult(result) + case <-stop: + return Result{Err: ErrClosed}, false + } +} + +func waitForResult(ctx context.Context, c *call) (Result, bool) { + select { + case result := <-c.reply: + return callbackResult(result) + case <-ctx.Done(): + return Result{Err: ctx.Err()}, false + } +} + +func callbackResult(result Result) (Result, bool) { + if _, ok := result.Value.(notDispatchedMarker); ok { + result.Value = nil + return result, false + } + return result, true +} + +func (b *Box) enqueue(ctx context.Context, c *call, wait bool) error { + for { + b.mu.Lock() + if b.closed { + b.mu.Unlock() + return ErrClosed + } + select { + case b.in <- c: + b.mu.Unlock() + return nil + default: + b.mu.Unlock() + } + if !wait { + return ErrOverloaded + } + select { + case <-b.space: + case <-ctx.Done(): + return ctx.Err() + case <-b.closedSignal: + return ErrClosed + } + } +} + +func (b *Box) Close() { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return + } + b.closed = true + close(b.closedSignal) + close(b.in) +} + +func (b *Box) Done() <-chan struct{} { + return b.done +} + +func (b *Box) Len() int { + return len(b.in) +} + +func (b *Box) run() { + defer close(b.done) + + for c := range b.in { + b.signalSpace() + if b.isClosed() { + c.reply <- Result{Value: notDispatchedMarker{}, Err: ErrClosed} + b.rejectQueued() + return + } + if err := c.ctx.Err(); err != nil { + c.reply <- Result{Value: notDispatchedMarker{}, Err: err} + if b.onCanceled != nil { + b.onCanceled() + } + continue + } + value, err := c.fn(c.ctx) + c.reply <- Result{Value: value, Err: err} + if b.isClosed() { + b.rejectQueued() + return + } + } + + b.rejectQueued() +} + +func (b *Box) signalSpace() { + select { + case b.space <- struct{}{}: + default: + } +} + +func (b *Box) isClosed() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.closed +} + +func (b *Box) rejectQueued() { + for { + select { + case c, ok := <-b.in: + if !ok { + return + } + c.reply <- Result{Value: notDispatchedMarker{}, Err: ErrClosed} + default: + return + } + } +} diff --git a/internal/mail/mail_test.go b/internal/mail/mail_test.go new file mode 100644 index 0000000..4216e36 --- /dev/null +++ b/internal/mail/mail_test.go @@ -0,0 +1,473 @@ +package mail + +import ( + "context" + "errors" + "testing" + "testing/synctest" +) + +type callResult struct { + value any + err error +} + +func TestMailbox_SerializesCalls(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + box := New(2, nil) + defer box.Close() + + started := make(chan struct{}) + release := make(chan struct{}) + secondStarted := make(chan struct{}) + firstDone := make(chan callResult, 1) + secondDone := make(chan callResult, 1) + + go func() { + value, err := box.Call(context.Background(), func(context.Context) (any, error) { + close(started) + <-release + return "first", nil + }) + firstDone <- callResult{value: value, err: err} + }() + synctest.Wait() + <-started + + go func() { + value, err := box.Call(context.Background(), func(context.Context) (any, error) { + close(secondStarted) + return "second", nil + }) + secondDone <- callResult{value: value, err: err} + }() + synctest.Wait() + select { + case <-secondStarted: + t.Fatal("second call ran before first call completed") + default: + } + + close(release) + synctest.Wait() + if result := <-firstDone; result.value != "first" || result.err != nil { + t.Fatalf("first result = %#v, want first result", result) + } + if result := <-secondDone; result.value != "second" || result.err != nil { + t.Fatalf("second result = %#v, want second result", result) + } + }) +} + +func TestMailbox_RejectsCallsWhenQueueIsFull(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + box := New(1, nil) + defer box.Close() + + started := make(chan struct{}) + release := make(chan struct{}) + firstDone := make(chan error, 1) + secondDone := make(chan error, 1) + + go func() { + _, err := box.Call(context.Background(), func(context.Context) (any, error) { + close(started) + <-release + return nil, nil + }) + firstDone <- err + }() + synctest.Wait() + <-started + + go func() { + _, err := box.Call(context.Background(), func(context.Context) (any, error) { return nil, nil }) + secondDone <- err + }() + synctest.Wait() + + if _, err := box.Call(context.Background(), func(context.Context) (any, error) { return nil, nil }); !errors.Is(err, ErrOverloaded) { + t.Fatalf("third call error = %v, want ErrOverloaded", err) + } + + close(release) + synctest.Wait() + if err := <-firstDone; err != nil { + t.Fatalf("first call error = %v", err) + } + if err := <-secondDone; err != nil { + t.Fatalf("second call error = %v", err) + } + }) +} + +func TestMailbox_CallWaitResultWaitsForCapacity(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + box := New(1, nil) + defer box.Close() + + firstStarted := make(chan struct{}) + firstRelease := make(chan struct{}) + firstDone := make(chan error, 1) + go func() { + _, err := box.Call(context.Background(), func(context.Context) (any, error) { + close(firstStarted) + <-firstRelease + return nil, nil + }) + firstDone <- err + }() + synctest.Wait() + <-firstStarted + + secondDone := make(chan error, 1) + go func() { + _, err := box.Call(context.Background(), func(context.Context) (any, error) { + return nil, nil + }) + secondDone <- err + }() + synctest.Wait() + + waitedEntered := make(chan struct{}) + waitedDone := make(chan error, 1) + go func() { + result, dispatched := box.CallWaitResult(context.Background(), func(context.Context) (any, error) { + close(waitedEntered) + return nil, nil + }) + if !dispatched { + waitedDone <- result.Err + return + } + waitedDone <- result.Err + }() + synctest.Wait() + select { + case <-waitedEntered: + t.Fatal("waiting turn entered while mailbox was full") + default: + } + + close(firstRelease) + synctest.Wait() + if err := <-firstDone; err != nil { + t.Fatalf("first call: %v", err) + } + if err := <-secondDone; err != nil { + t.Fatalf("second call: %v", err) + } + if err := <-waitedDone; err != nil { + t.Fatalf("waiting turn: %v", err) + } + select { + case <-waitedEntered: + default: + t.Fatal("waiting turn did not enter after capacity was free") + } + }) +} + +func TestMailbox_CallWaitResultCancellationStopsAdmission(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + box := New(1, nil) + defer box.Close() + + firstStarted := make(chan struct{}) + firstRelease := make(chan struct{}) + go func() { + _, _ = box.Call(context.Background(), func(context.Context) (any, error) { + close(firstStarted) + <-firstRelease + return nil, nil + }) + }() + synctest.Wait() + <-firstStarted + go func() { + _, _ = box.Call(context.Background(), func(context.Context) (any, error) { return nil, nil }) + }() + synctest.Wait() + + ctx, cancel := context.WithCancel(context.Background()) + entered := make(chan struct{}) + done := make(chan callResult, 1) + go func() { + result, dispatched := box.CallWaitResult(ctx, func(context.Context) (any, error) { + close(entered) + return nil, nil + }) + done <- callResult{value: dispatched, err: result.Err} + }() + synctest.Wait() + cancel() + synctest.Wait() + result := <-done + if result.value.(bool) || !errors.Is(result.err, context.Canceled) { + t.Fatalf("waiting canceled result = (%v, %v), want not dispatched context.Canceled", result.value, result.err) + } + close(firstRelease) + synctest.Wait() + select { + case <-entered: + t.Fatal("canceled waiting turn entered") + default: + } + }) +} + +func TestMailbox_CallWaitResultCancellationAfterEnqueueWaitsForDisposition(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + box := New(1, nil) + defer box.Close() + + firstStarted := make(chan struct{}) + firstRelease := make(chan struct{}, 1) + defer func() { + select { + case firstRelease <- struct{}{}: + default: + } + }() + go func() { + _, _ = box.Call(context.Background(), func(context.Context) (any, error) { + close(firstStarted) + <-firstRelease + return nil, nil + }) + }() + synctest.Wait() + <-firstStarted + + ctx, cancel := context.WithCancel(context.Background()) + entered := make(chan struct{}) + done := make(chan callResult, 1) + go func() { + result, dispatched := box.CallWaitResult(ctx, func(context.Context) (any, error) { + close(entered) + return nil, nil + }) + done <- callResult{value: dispatched, err: result.Err} + }() + synctest.Wait() + if got := box.Len(); got != 1 { + t.Fatalf("queued Runtime turn count = %d, want 1", got) + } + + cancel() + synctest.Wait() + select { + case result := <-done: + t.Fatalf("queued Runtime turn returned before mailbox disposition: %#v", result) + default: + } + + firstRelease <- struct{}{} + synctest.Wait() + result := <-done + if result.value.(bool) || !errors.Is(result.err, context.Canceled) { + t.Fatalf("canceled queued result = (%v, %v), want not dispatched context.Canceled", result.value, result.err) + } + select { + case <-entered: + t.Fatal("canceled queued Runtime turn entered") + default: + } + }) +} + +func TestMailbox_ContinuesAfterCallerTimeout(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + box := New(1, nil) + defer box.Close() + + started := make(chan struct{}) + release := make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + callDone := make(chan error, 1) + go func() { + _, err := box.Call(ctx, func(context.Context) (any, error) { + close(started) + <-release + return "timed out", nil + }) + callDone <- err + }() + + synctest.Wait() + <-started + cancel() + synctest.Wait() + if err := <-callDone; !errors.Is(err, context.Canceled) { + t.Fatalf("timed out call error = %v, want context.Canceled", err) + } + + close(release) + synctest.Wait() + value, err := box.Call(context.Background(), func(context.Context) (any, error) { + return "after timeout", nil + }) + if err != nil || value != "after timeout" { + t.Fatalf("follow-up call = %#v, %v", value, err) + } + }) +} + +func TestMailbox_CanceledQueuedCallDoesNotEnter(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + box := New(1, nil) + defer box.Close() + + started := make(chan struct{}) + release := make(chan struct{}) + firstDone := make(chan error, 1) + go func() { + _, err := box.Call(context.Background(), func(context.Context) (any, error) { + close(started) + <-release + return nil, nil + }) + firstDone <- err + }() + synctest.Wait() + <-started + + ctx, cancel := context.WithCancel(context.Background()) + entered := make(chan struct{}) + queuedDone := make(chan error, 1) + go func() { + _, err := box.Call(ctx, func(context.Context) (any, error) { + close(entered) + return nil, nil + }) + queuedDone <- err + }() + synctest.Wait() + cancel() + synctest.Wait() + if err := <-queuedDone; !errors.Is(err, context.Canceled) { + t.Fatalf("queued call error = %v, want context.Canceled", err) + } + + close(release) + synctest.Wait() + if err := <-firstDone; err != nil { + t.Fatalf("first call error = %v", err) + } + select { + case <-entered: + t.Fatal("canceled queued call entered its method") + default: + } + }) +} + +func TestMailbox_CallResultSeparatesCallbackErrorFromControlError(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + box := New(1, nil) + value := map[string]int{"value": 1} + result, dispatched := box.CallResult(context.Background(), func(context.Context) (any, error) { + return value, ErrClosed + }) + if !dispatched || !errors.Is(result.Err, ErrClosed) { + t.Fatalf("callback result = (%v, %v), want dispatched ErrClosed", dispatched, result.Err) + } + if got := result.Value.(map[string]int)["value"]; got != 1 { + t.Fatalf("callback value = %d, want 1", got) + } + + box.Close() + entered := make(chan struct{}, 1) + result, dispatched = box.CallResult(context.Background(), func(context.Context) (any, error) { + entered <- struct{}{} + return nil, nil + }) + if dispatched || !errors.Is(result.Err, ErrClosed) { + t.Fatalf("closed result = (%v, %v), want not dispatched ErrClosed", dispatched, result.Err) + } + select { + case <-entered: + t.Fatal("closed mailbox dispatched callback") + default: + } + }) +} + +func TestMailbox_RejectsQueuedCallsOnClose(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + box := New(1, nil) + + started := make(chan struct{}) + release := make(chan struct{}) + firstDone := make(chan callResult, 1) + queuedDone := make(chan callResult, 1) + go func() { + value, err := box.Call(context.Background(), func(context.Context) (any, error) { + close(started) + <-release + return "first", nil + }) + firstDone <- callResult{value: value, err: err} + }() + synctest.Wait() + <-started + + go func() { + value, err := box.Call(context.Background(), func(context.Context) (any, error) { + return "queued", nil + }) + queuedDone <- callResult{value: value, err: err} + }() + synctest.Wait() + box.Close() + close(release) + synctest.Wait() + + if result := <-firstDone; result.value != "first" || result.err != nil { + t.Fatalf("first result = %#v, want first result", result) + } + if result := <-queuedDone; !errors.Is(result.err, ErrClosed) { + t.Fatalf("queued result error = %v, want ErrClosed", result.err) + } + }) +} + +func TestMailbox_LenReportsQueuedCalls(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + box := New(1, nil) + defer box.Close() + + started := make(chan struct{}) + release := make(chan struct{}) + firstDone := make(chan error, 1) + queuedDone := make(chan error, 1) + go func() { + _, err := box.Call(context.Background(), func(context.Context) (any, error) { + close(started) + <-release + return nil, nil + }) + firstDone <- err + }() + synctest.Wait() + <-started + + go func() { + _, err := box.Call(context.Background(), func(context.Context) (any, error) { return nil, nil }) + queuedDone <- err + }() + synctest.Wait() + if got := box.Len(); got != 1 { + t.Fatalf("Len = %d, want 1", got) + } + + close(release) + synctest.Wait() + if err := <-firstDone; err != nil { + t.Fatalf("first call error = %v", err) + } + if err := <-queuedDone; err != nil { + t.Fatalf("queued call error = %v", err) + } + }) +} diff --git a/internal/platformcheck/main.go b/internal/platformcheck/main.go new file mode 100644 index 0000000..4d99b14 --- /dev/null +++ b/internal/platformcheck/main.go @@ -0,0 +1,41 @@ +// Command platformcheck verifies the Go host system used by a release job. +package main + +import ( + "flag" + "fmt" + "io" + "os" + "runtime" +) + +func main() { + os.Exit(run(os.Args[1:], os.Stderr)) +} + +func run(args []string, stderr io.Writer) int { + flags := flag.NewFlagSet("platformcheck", flag.ContinueOnError) + flags.SetOutput(stderr) + wantOS := flags.String("os", "", "required GOOS") + wantArch := flags.String("arch", "", "required GOARCH") + if err := flags.Parse(args); err != nil { + return 2 + } + if flags.NArg() != 0 || *wantOS == "" || *wantArch == "" { + fmt.Fprintln(stderr, "platformcheck: -os and -arch are required") + return 2 + } + if err := check(*wantOS, *wantArch, runtime.GOOS, runtime.GOARCH); err != nil { + fmt.Fprintf(stderr, "platformcheck: %v\n", err) + return 1 + } + fmt.Fprintf(stderr, "platformcheck: %s/%s\n", runtime.GOOS, runtime.GOARCH) + return 0 +} + +func check(wantOS, wantArch, actualOS, actualArch string) error { + if actualOS != wantOS || actualArch != wantArch { + return fmt.Errorf("host is %s/%s, want %s/%s", actualOS, actualArch, wantOS, wantArch) + } + return nil +} diff --git a/internal/platformcheck/main_test.go b/internal/platformcheck/main_test.go new file mode 100644 index 0000000..d0d2af0 --- /dev/null +++ b/internal/platformcheck/main_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "bytes" + "runtime" + "strings" + "testing" +) + +func TestCheckAcceptsExactPlatform(t *testing.T) { + if err := check("linux", "amd64", "linux", "amd64"); err != nil { + t.Fatalf("check exact platform: %v", err) + } +} + +func TestCheckRejectsDifferentPlatform(t *testing.T) { + err := check("darwin", "arm64", "windows", "amd64") + if err == nil || !strings.Contains(err.Error(), "host is windows/amd64, want darwin/arm64") { + t.Fatalf("check different platform error = %v", err) + } +} + +func TestRunRequiresBothFlags(t *testing.T) { + var stderr bytes.Buffer + if code := run([]string{"-os", runtime.GOOS}, &stderr); code != 2 { + t.Fatalf("run exit = %d, want 2", code) + } + if !strings.Contains(stderr.String(), "-os and -arch are required") { + t.Fatalf("stderr = %q, want required flag error", stderr.String()) + } +} + +func TestRunAcceptsHostPlatform(t *testing.T) { + var stderr bytes.Buffer + code := run([]string{"-os", runtime.GOOS, "-arch", runtime.GOARCH}, &stderr) + if code != 0 { + t.Fatalf("run exit = %d, want 0; stderr = %q", code, stderr.String()) + } +} diff --git a/internal/runtime/activation_context.go b/internal/runtime/activation_context.go new file mode 100644 index 0000000..c30a84f --- /dev/null +++ b/internal/runtime/activation_context.go @@ -0,0 +1,28 @@ +package runtime + +import "context" + +type deactivateOnIdleKey struct{} +type grainTimerRegistrarKey struct{} + +func withDeactivateOnIdle(ctx context.Context, request func()) context.Context { + return context.WithValue(ctx, deactivateOnIdleKey{}, request) +} + +// DeactivateOnIdleFrom returns the lifecycle control attached to an +// Activation context. The root package binds it to the public Grain Context. +func DeactivateOnIdleFrom(ctx context.Context) func() { + request, _ := ctx.Value(deactivateOnIdleKey{}).(func()) + return request +} + +func withGrainTimerRegistrar(ctx context.Context, register GrainTimerRegistrar) context.Context { + return context.WithValue(ctx, grainTimerRegistrarKey{}, register) +} + +// GrainTimerRegistrarFrom returns the timer capability attached to an +// Activation context. The root package binds it to the public Grain Context. +func GrainTimerRegistrarFrom(ctx context.Context) GrainTimerRegistrar { + register, _ := ctx.Value(grainTimerRegistrarKey{}).(GrainTimerRegistrar) + return register +} diff --git a/internal/runtime/grain_timer.go b/internal/runtime/grain_timer.go new file mode 100644 index 0000000..e0adee4 --- /dev/null +++ b/internal/runtime/grain_timer.go @@ -0,0 +1,308 @@ +package runtime + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/suraciii/gor/clock" +) + +// ErrGrainTimerStopped reports that Change targeted a stopped Grain Timer or +// an Activation which is no longer active. +var ErrGrainTimerStopped = errors.New("grain timer stopped") + +// GrainTimer is the Activation-local timer handle implemented by the Runtime. +type GrainTimer interface { + Change(dueTime time.Duration, period time.Duration) error + Stop() +} + +// GrainTimerOptions defines the first due time, repeat period, and idle-use +// behavior of one Grain Timer. +type GrainTimerOptions struct { + DueTime time.Duration + Period time.Duration + KeepAlive bool +} + +// GrainTimerRegistrar creates a Grain Timer for the Activation in ctx. +type GrainTimerRegistrar func(func(context.Context) error, GrainTimerOptions) (GrainTimer, error) + +type grainTimerCommandKind uint8 + +const ( + grainTimerChange grainTimerCommandKind = iota + grainTimerStop +) + +type grainTimerCommand struct { + kind grainTimerCommandKind + dueTime time.Duration + period time.Duration + reply chan error +} + +type grainTimerTurnResult struct { + admitted bool +} + +type grainTimer struct { + runtime *Runtime + activation *activation + callback func(context.Context) error + keepAlive bool + clock clock.Clock + queueCtx context.Context + queueCancel context.CancelFunc + + commands chan grainTimerCommand + completed chan grainTimerTurnResult + ready chan error + done chan struct{} +} + +func validateGrainTimerSchedule(dueTime time.Duration, period time.Duration) error { + if dueTime < 0 { + return errors.New("grain timer due time must not be negative") + } + if period < 0 { + return errors.New("grain timer period must not be negative") + } + return nil +} + +func (r *Runtime) registerGrainTimer(act *activation, callback func(context.Context) error, options GrainTimerOptions) (GrainTimer, error) { + if callback == nil { + return nil, errors.New("grain timer callback is required") + } + if err := validateGrainTimerSchedule(options.DueTime, options.Period); err != nil { + return nil, err + } + + r.mu.Lock() + if r.state != engineRunning || (act.state != ActivationActivating && act.state != ActivationActive) || act.lifecycleCtx.Err() != nil { + r.mu.Unlock() + return nil, ErrGrainTimerStopped + } + timer := &grainTimer{ + runtime: r, + activation: act, + callback: callback, + keepAlive: options.KeepAlive, + clock: r.clock, + commands: make(chan grainTimerCommand), + completed: make(chan grainTimerTurnResult, 1), + ready: make(chan error, 1), + done: make(chan struct{}), + } + timer.queueCtx, timer.queueCancel = context.WithCancel(act.lifecycleCtx) + act.timers[timer] = struct{}{} + r.timers[timer] = struct{}{} + r.mu.Unlock() + + go timer.run(options) + if err := <-timer.ready; err != nil { + return nil, err + } + return timer, nil +} + +func (t *grainTimer) Change(dueTime time.Duration, period time.Duration) error { + if err := validateGrainTimerSchedule(dueTime, period); err != nil { + return err + } + reply := make(chan error, 1) + command := grainTimerCommand{kind: grainTimerChange, dueTime: dueTime, period: period, reply: reply} + select { + case t.commands <- command: + case <-t.done: + return ErrGrainTimerStopped + } + select { + case err := <-reply: + return err + case <-t.done: + return ErrGrainTimerStopped + } +} + +func (t *grainTimer) Stop() { + reply := make(chan error, 1) + command := grainTimerCommand{kind: grainTimerStop, reply: reply} + select { + case t.commands <- command: + case <-t.done: + return + } + select { + case <-reply: + case <-t.done: + } +} + +func (t *grainTimer) run(options GrainTimerOptions) { + defer t.finish() + + var ( + ticker clock.Ticker + tick <-chan time.Time + immediate <-chan struct{} + dueTime = options.DueTime + period = options.Period + changed bool + firing bool + stopping bool + lifecycle = t.activation.lifecycleCtx.Done() + ) + disarm := func() { + if ticker != nil { + ticker.Stop() + ticker = nil + } + tick = nil + immediate = nil + } + arm := func(delay time.Duration) { + disarm() + if delay == 0 { + ready := make(chan struct{}) + close(ready) + immediate = ready + return + } + ticker = t.clock.NewTicker(delay) + tick = ticker.C() + } + + if err := t.activation.lifecycleCtx.Err(); err != nil { + t.ready <- ErrGrainTimerStopped + return + } + arm(dueTime) + t.ready <- nil + + for { + if stopping && !firing { + disarm() + return + } + select { + case <-lifecycle: + stopping = true + disarm() + lifecycle = nil + case command := <-t.commands: + switch command.kind { + case grainTimerChange: + if stopping || t.activation.lifecycleCtx.Err() != nil { + stopping = true + disarm() + command.reply <- ErrGrainTimerStopped + continue + } + dueTime = command.dueTime + period = command.period + changed = true + if !firing { + arm(dueTime) + } + command.reply <- nil + case grainTimerStop: + stopping = true + disarm() + t.queueCancel() + command.reply <- nil + } + case <-immediate: + immediate = nil + if !stopping { + firing = true + changed = false + go t.invoke() + } + case <-tick: + disarm() + if !stopping { + firing = true + changed = false + go t.invoke() + } + case result := <-t.completed: + firing = false + if !result.admitted { + stopping = true + continue + } + if stopping { + continue + } + if changed { + arm(dueTime) + } else if period > 0 { + arm(period) + } + } + } +} + +func (t *grainTimer) invoke() { + result := t.runtime.invokeGrainTimerTurn(t.activation, t.queueCtx, t.callback, t.keepAlive) + t.completed <- result +} + +func (t *grainTimer) finish() { + t.queueCancel() + t.runtime.mu.Lock() + delete(t.activation.timers, t) + delete(t.runtime.timers, t) + close(t.done) + t.runtime.mu.Unlock() +} + +func (r *Runtime) invokeGrainTimerTurn(act *activation, queueCtx context.Context, callback func(context.Context) error, keepAlive bool) grainTimerTurnResult { + result, dispatched := act.lane.mailbox.CallWaitResultUntil(queueCtx, act.timerWaitDone, func(context.Context) (any, error) { + if !r.beginTurnWithKeepAlive(act, false) { + return grainTimerTurnResult{}, nil + } + err, faulted := callGrainTimerCallback(act.timerCtx, callback) + runtimeErr := act.lifecycleCtx.Err() + runtimeCanceled := runtimeErr != nil && errors.Is(err, runtimeErr) + if err != nil && !runtimeCanceled && act.onGrainTimerError != nil { + reportGrainTimerError(act.onGrainTimerError, act.id, err) + } + r.finishGrainTimerTurn(act, faulted, keepAlive) + return grainTimerTurnResult{admitted: true}, nil + }) + if !dispatched { + return grainTimerTurnResult{} + } + outcome, ok := result.Value.(grainTimerTurnResult) + if !ok { + return grainTimerTurnResult{} + } + return outcome +} + +func reportGrainTimerError(handler func(GrainId, error), id GrainId, err error) { + defer func() { + recover() + }() + handler(id, err) +} + +func callGrainTimerCallback(ctx context.Context, callback func(context.Context) error) (err error, faulted bool) { + defer func() { + if value := recover(); value != nil { + err = fmt.Errorf("%w: Grain Timer callback panicked: %v", ErrPanic, value) + faulted = true + } + }() + err = callback(ctx) + var discard Discard + if errors.As(err, &discard) { + return discard.Err, true + } + return err, false +} diff --git a/internal/runtime/grain_timer_test.go b/internal/runtime/grain_timer_test.go new file mode 100644 index 0000000..dd0aa59 --- /dev/null +++ b/internal/runtime/grain_timer_test.go @@ -0,0 +1,320 @@ +package runtime + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/suraciii/gor/clock" +) + +func TestGrainTimerGracefulCloseWaitsForCallbackAndStopsTimerBeforeHook(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + callbackStarted := make(chan struct{}) + callbackRelease := make(chan struct{}, 1) + defer func() { + select { + case callbackRelease <- struct{}{}: + default: + } + }() + timerCreated := make(chan *grainTimer, 1) + hookResult := make(chan bool, 1) + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) + if err := rt.Register("probe", Registration{ + Factory: func(ctx context.Context, _ GrainId) (any, error) { + handle, err := GrainTimerRegistrarFrom(ctx)(func(context.Context) error { + close(callbackStarted) + <-callbackRelease + return nil + }, GrainTimerOptions{}) + if err != nil { + return nil, err + } + timerCreated <- handle.(*grainTimer) + return struct{}{}, nil + }, + Dispatch: func(context.Context, any, string, any, any) error { return nil }, + OnDeactivate: func(context.Context, GrainId, DeactivationReason, any) { + timer := <-timerCreated + select { + case <-timer.done: + hookResult <- true + default: + hookResult <- false + } + }, + }); err != nil { + t.Fatal(err) + } + if err := rt.Invoke(context.Background(), GrainId{GrainType: "probe", GrainKey: "one"}, "Start", nil, nil); err != nil { + t.Fatal(err) + } + synctest.Wait() + <-callbackStarted + + rt.BeginClose() + synctest.Wait() + select { + case <-rt.Done(): + t.Fatal("graceful close ended before the Grain Timer callback") + default: + } + select { + case <-hookResult: + t.Fatal("OnDeactivate ran before the Grain Timer callback ended") + default: + } + + callbackRelease <- struct{}{} + synctest.Wait() + if stopped := <-hookResult; !stopped { + t.Fatal("OnDeactivate started before the Grain Timer owner stopped") + } + select { + case <-rt.Done(): + default: + t.Fatal("graceful close did not end after timer cleanup") + } + }) +} + +func TestGrainTimerAndDeactivationHoldActivationSlot(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + callbackStarted := make(chan struct{}, 1) + releaseCallback := make(chan struct{}) + hookStarted := make(chan struct{}, 1) + releaseHook := make(chan struct{}) + var factoryCalls atomic.Int32 + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1, MaxActivations: 1}) + defer stopEngine(rt) + if err := rt.Register("probe", Registration{ + Factory: func(ctx context.Context, _ GrainId) (any, error) { + if factoryCalls.Add(1) != 1 { + return struct{}{}, nil + } + if _, err := GrainTimerRegistrarFrom(ctx)(func(context.Context) error { + callbackStarted <- struct{}{} + <-releaseCallback + return nil + }, GrainTimerOptions{}); err != nil { + return nil, err + } + return struct{}{}, nil + }, + Dispatch: func(context.Context, any, string, any, any) error { return nil }, + OnDeactivate: func(context.Context, GrainId, DeactivationReason, any) { + hookStarted <- struct{}{} + <-releaseHook + }, + }); err != nil { + t.Fatal(err) + } + + firstID := GrainId{GrainType: "probe", GrainKey: "one"} + if err := rt.Invoke(context.Background(), firstID, "Start", nil, nil); err != nil { + t.Fatal(err) + } + synctest.Wait() + <-callbackStarted + + rt.Deactivate(firstID) + synctest.Wait() + secondID := GrainId{GrainType: "probe", GrainKey: "two"} + if err := rt.Invoke(context.Background(), secondID, "Start", nil, nil); !errors.Is(err, ErrActivationLimit) { + t.Fatalf("Invoke while Grain Timer callback runs = %v, want ErrActivationLimit", err) + } + + close(releaseCallback) + synctest.Wait() + <-hookStarted + if err := rt.Invoke(context.Background(), secondID, "Start", nil, nil); !errors.Is(err, ErrActivationLimit) { + t.Fatalf("Invoke while OnDeactivate runs = %v, want ErrActivationLimit", err) + } + + close(releaseHook) + synctest.Wait() + if err := rt.Invoke(context.Background(), secondID, "Start", nil, nil); err != nil { + t.Fatalf("Invoke after timer and deactivation cleanup = %v", err) + } + }) +} + +func TestGrainTimerKillWaitsForOwnerButNotCallback(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + callbackStarted := make(chan struct{}) + callbackRelease := make(chan struct{}, 1) + defer func() { + select { + case callbackRelease <- struct{}{}: + default: + } + }() + callbackDone := make(chan struct{}) + timerCreated := make(chan *grainTimer, 1) + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) + if err := rt.Register("probe", Registration{ + Factory: func(ctx context.Context, _ GrainId) (any, error) { + handle, err := GrainTimerRegistrarFrom(ctx)(func(context.Context) error { + close(callbackStarted) + <-callbackRelease + close(callbackDone) + return nil + }, GrainTimerOptions{}) + if err != nil { + return nil, err + } + timerCreated <- handle.(*grainTimer) + return struct{}{}, nil + }, + Dispatch: func(context.Context, any, string, any, any) error { return nil }, + }); err != nil { + t.Fatal(err) + } + if err := rt.Invoke(context.Background(), GrainId{GrainType: "probe", GrainKey: "one"}, "Start", nil, nil); err != nil { + t.Fatal(err) + } + timer := <-timerCreated + synctest.Wait() + <-callbackStarted + + rt.BeginKill() + synctest.Wait() + select { + case <-rt.Done(): + default: + t.Fatal("Kill did not stop Runtime infrastructure") + } + select { + case <-timer.done: + default: + t.Fatal("Runtime.Done closed before the Grain Timer owner stopped") + } + rt.mu.Lock() + remainingTimers := len(rt.timers) + rt.mu.Unlock() + if remainingTimers != 0 { + t.Fatalf("tracked Grain Timers after Runtime.Done = %d, want 0", remainingTimers) + } + select { + case <-callbackDone: + t.Fatal("Kill waited for user callback code") + default: + } + + callbackRelease <- struct{}{} + synctest.Wait() + select { + case <-callbackDone: + default: + t.Fatal("released Grain Timer callback did not end") + } + }) +} + +func TestGrainTimerRegistrationStopsWithRuntime(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) + act := &activation{ + lifecycleCtx: context.Background(), + state: ActivationActive, + timers: make(map[*grainTimer]struct{}), + } + rt.BeginKill() + <-rt.Done() + if _, err := rt.registerGrainTimer(act, func(context.Context) error { return nil }, GrainTimerOptions{DueTime: time.Second}); err != ErrGrainTimerStopped { + t.Fatalf("registration after Kill = %v, want ErrGrainTimerStopped", err) + } +} + +func TestGrainTimerActivationFailureDiscardsQueuedTimer(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + factoryErr := errors.New("activation failed") + callbackEntered := make(chan struct{}, 1) + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) + defer func() { + rt.BeginKill() + <-rt.Done() + }() + if err := rt.Register("probe", Registration{ + Factory: func(ctx context.Context, _ GrainId) (any, error) { + _, err := GrainTimerRegistrarFrom(ctx)(func(context.Context) error { + callbackEntered <- struct{}{} + return nil + }, GrainTimerOptions{}) + if err != nil { + return nil, err + } + return nil, factoryErr + }, + Dispatch: func(context.Context, any, string, any, any) error { return nil }, + }); err != nil { + t.Fatal(err) + } + if err := rt.Invoke(context.Background(), GrainId{GrainType: "probe", GrainKey: "one"}, "Start", nil, nil); !errors.Is(err, factoryErr) { + t.Fatalf("activation error = %v, want %v", err, factoryErr) + } + synctest.Wait() + select { + case <-callbackEntered: + t.Fatal("Grain Timer callback entered after Activation setup failed") + default: + } + rt.mu.Lock() + remainingTimers := len(rt.timers) + rt.mu.Unlock() + if remainingTimers != 0 { + t.Fatalf("tracked Grain Timers after Activation setup failure = %d, want 0", remainingTimers) + } + }) +} + +func TestGrainTimerErrorHandlerPanicDoesNotBlockCleanup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + callbackErr := errors.New("timer callback failed") + reported := make(chan error, 1) + timerCreated := make(chan *grainTimer, 1) + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) + defer stopEngine(rt) + if err := rt.Register("probe", Registration{ + Factory: func(ctx context.Context, _ GrainId) (any, error) { + handle, err := GrainTimerRegistrarFrom(ctx)(func(context.Context) error { + return callbackErr + }, GrainTimerOptions{}) + if err != nil { + return nil, err + } + timerCreated <- handle.(*grainTimer) + return struct{}{}, nil + }, + Dispatch: func(context.Context, any, string, any, any) error { return nil }, + OnGrainTimerError: func(_ GrainId, err error) { + reported <- err + panic("error handler failed") + }, + }); err != nil { + t.Fatal(err) + } + id := GrainId{GrainType: "probe", GrainKey: "one"} + if err := rt.Invoke(context.Background(), id, "Start", nil, nil); err != nil { + t.Fatal(err) + } + timer := <-timerCreated + synctest.Wait() + if err := <-reported; !errors.Is(err, callbackErr) { + t.Fatalf("reported error = %v, want %v", err, callbackErr) + } + if err := rt.Invoke(context.Background(), id, "AfterError", nil, nil); err != nil { + t.Fatalf("Call after error handler panic: %v", err) + } + timer.Stop() + synctest.Wait() + select { + case <-timer.done: + default: + t.Fatal("Grain Timer owner did not stop after error handler panic") + } + }) +} diff --git a/internal/runtime/lifecycle_hardening_test.go b/internal/runtime/lifecycle_hardening_test.go new file mode 100644 index 0000000..ab6edb1 --- /dev/null +++ b/internal/runtime/lifecycle_hardening_test.go @@ -0,0 +1,469 @@ +package runtime + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "testing/synctest" + + "github.com/suraciii/gor/clock" + "github.com/suraciii/gor/internal/mail" +) + +type activationTestContextKey struct{} + +func TestRuntime_TriggerCancellationDoesNotCancelSharedActivation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 2}) + defer stopEngine(rt) + + factoryContext := make(chan context.Context, 1) + releaseFactory := make(chan struct{}) + var factoryCalls atomic.Int32 + var methodCalls atomic.Int32 + if err := rt.Register("account", Registration{ + ActivationContext: func(lifecycleCtx context.Context, callCtx context.Context) context.Context { + return context.WithValue(lifecycleCtx, activationTestContextKey{}, callCtx.Value(activationTestContextKey{})) + }, + Factory: func(ctx context.Context, _ GrainId) (any, error) { + factoryCalls.Add(1) + factoryContext <- ctx + <-releaseFactory + return &testGrain{}, nil + }, + Dispatch: func(context.Context, any, string, any, any) error { + methodCalls.Add(1) + return nil + }, + }); err != nil { + t.Fatal(err) + } + + id := GrainId{GrainType: "account", GrainKey: "alice"} + firstBase := context.WithValue(context.Background(), activationTestContextKey{}, "first") + firstCtx, cancelFirst := context.WithCancel(firstBase) + firstDone := make(chan error, 1) + secondDone := make(chan error, 1) + go func() { firstDone <- rt.Invoke(firstCtx, id, "Value", nil, nil) }() + synctest.Wait() + activationCtx := <-factoryContext + go func() { secondDone <- rt.Invoke(context.Background(), id, "Value", nil, nil) }() + synctest.Wait() + + cancelFirst() + synctest.Wait() + if err := <-firstDone; !errors.Is(err, context.Canceled) { + t.Fatalf("triggering call error = %v, want context.Canceled", err) + } + if err := activationCtx.Err(); err != nil { + t.Fatalf("Activation context after caller cancellation = %v, want nil", err) + } + if _, ok := activationCtx.Deadline(); ok { + t.Fatal("Activation context copied the caller deadline") + } + if got := activationCtx.Value(activationTestContextKey{}); got != "first" { + t.Fatalf("Activation context value = %v, want first", got) + } + + close(releaseFactory) + synctest.Wait() + if err := <-secondDone; err != nil { + t.Fatalf("waiting call error = %v", err) + } + if got := factoryCalls.Load(); got != 1 { + t.Fatalf("factory calls = %d, want 1", got) + } + if got := methodCalls.Load(); got != 1 { + t.Fatalf("method calls = %d, want only the waiting Call", got) + } + }) +} + +func TestRuntime_CancellationLinearizesAtMethodEntry(t *testing.T) { + t.Run("cancellation wins", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) + defer stopEngine(rt) + + firstStarted := make(chan struct{}) + releaseFirst := make(chan struct{}) + queuedEntered := make(chan struct{}) + if err := rt.Register("account", Registration{ + Factory: func(context.Context, GrainId) (any, error) { return &testGrain{}, nil }, + Dispatch: func(_ context.Context, _ any, method string, _ any, _ any) error { + switch method { + case "Block": + close(firstStarted) + <-releaseFirst + case "Queued": + close(queuedEntered) + } + return nil + }, + }); err != nil { + t.Fatal(err) + } + + id := GrainId{GrainType: "account", GrainKey: "alice"} + firstDone := make(chan error, 1) + go func() { firstDone <- rt.Invoke(context.Background(), id, "Block", nil, nil) }() + synctest.Wait() + <-firstStarted + + ctx, cancel := context.WithCancel(context.Background()) + queuedDone := make(chan error, 1) + go func() { queuedDone <- rt.Invoke(ctx, id, "Queued", nil, nil) }() + synctest.Wait() + cancel() + synctest.Wait() + if err := <-queuedDone; !errors.Is(err, context.Canceled) { + t.Fatalf("queued call error = %v, want context.Canceled", err) + } + + close(releaseFirst) + synctest.Wait() + if err := <-firstDone; err != nil { + t.Fatalf("first call error = %v", err) + } + select { + case <-queuedEntered: + t.Fatal("canceled queued Call entered its method") + default: + } + }) + }) + + t.Run("method entry wins", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) + defer stopEngine(rt) + + entered := make(chan struct{}) + if err := rt.Register("account", Registration{ + Factory: func(context.Context, GrainId) (any, error) { return &testGrain{}, nil }, + Dispatch: func(ctx context.Context, _ any, _ string, _ any, _ any) error { + close(entered) + <-ctx.Done() + return ctx.Err() + }, + }); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- rt.Invoke(ctx, GrainId{GrainType: "account", GrainKey: "alice"}, "Wait", nil, nil) + }() + synctest.Wait() + <-entered + cancel() + synctest.Wait() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("started call error = %v, want context.Canceled", err) + } + }) + }) +} + +func TestRuntime_CanceledFirstCallReleasesEmptyCallLane(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) + defer stopEngine(rt) + + var factoryCalls atomic.Int32 + if err := rt.Register("account", Registration{ + Factory: func(context.Context, GrainId) (any, error) { + factoryCalls.Add(1) + return &testGrain{}, nil + }, + Dispatch: func(context.Context, any, string, any, any) error { return nil }, + }); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := rt.Invoke(ctx, GrainId{GrainType: "account", GrainKey: "alice"}, "Value", nil, nil) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled Call error = %v, want context.Canceled", err) + } + synctest.Wait() + if got := factoryCalls.Load(); got != 0 { + t.Fatalf("factory calls = %d, want 0", got) + } + rt.mu.Lock() + lanes := len(rt.lanes) + rt.mu.Unlock() + if lanes != 0 { + t.Fatalf("Call lanes after canceled first Call = %d, want 0", lanes) + } + }) +} + +type rotatingActivation struct { + generation int + request func() +} + +func TestRuntime_RequestedDeactivationPreservesQueuedCallOrder(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 3}) + defer stopEngine(rt) + + rotateStarted := make(chan struct{}) + releaseRotate := make(chan struct{}) + hookStarted := make(chan struct{}) + releaseHook := make(chan struct{}) + reasons := make(chan DeactivationReason, 2) + entered := make(chan int, 2) + var factoryCalls atomic.Int32 + if err := rt.Register("account", Registration{ + Factory: func(ctx context.Context, _ GrainId) (any, error) { + return &rotatingActivation{ + generation: int(factoryCalls.Add(1)), + request: DeactivateOnIdleFrom(ctx), + }, nil + }, + Dispatch: func(_ context.Context, instance any, method string, args any, reply any) error { + activation := instance.(*rotatingActivation) + switch method { + case "Rotate": + activation.request() + activation.request() + close(rotateStarted) + <-releaseRotate + case "Value": + entered <- args.(int) + *(reply.(*int)) = activation.generation + default: + return errors.New("unknown method") + } + return nil + }, + OnDeactivate: func(_ context.Context, _ GrainId, reason DeactivationReason, _ any) { + if reason != ApplicationRequested { + return + } + reasons <- reason + close(hookStarted) + <-releaseHook + }, + }); err != nil { + t.Fatal(err) + } + + id := GrainId{GrainType: "account", GrainKey: "alice"} + rotateDone := make(chan error, 1) + go func() { rotateDone <- rt.Invoke(context.Background(), id, "Rotate", nil, nil) }() + synctest.Wait() + <-rotateStarted + + type valueResult struct { + value int + err error + } + firstDone := make(chan valueResult, 1) + secondDone := make(chan valueResult, 1) + go func() { + var value int + err := rt.Invoke(context.Background(), id, "Value", 1, &value) + firstDone <- valueResult{value: value, err: err} + }() + synctest.Wait() + go func() { + var value int + err := rt.Invoke(context.Background(), id, "Value", 2, &value) + secondDone <- valueResult{value: value, err: err} + }() + synctest.Wait() + + close(releaseRotate) + synctest.Wait() + if err := <-rotateDone; err != nil { + t.Fatalf("Rotate error = %v", err) + } + <-hookStarted + if reason := <-reasons; reason != ApplicationRequested { + t.Fatalf("deactivation reason = %v, want ApplicationRequested", reason) + } + select { + case label := <-entered: + t.Fatalf("queued Call %d entered before deactivation completed", label) + default: + } + + close(releaseHook) + synctest.Wait() + first := <-firstDone + second := <-secondDone + if first.err != nil || second.err != nil { + t.Fatalf("queued Call errors = %v, %v", first.err, second.err) + } + if first.value != 2 || second.value != 2 { + t.Fatalf("queued Call generations = %d, %d; want 2, 2", first.value, second.value) + } + if got := []int{<-entered, <-entered}; got[0] != 1 || got[1] != 2 { + t.Fatalf("queued Call order = %v, want [1 2]", got) + } + if got := factoryCalls.Load(); got != 2 { + t.Fatalf("factory calls = %d, want 2", got) + } + }) +} + +func TestRuntime_FaultWinsOverRequestedDeactivation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) + defer stopEngine(rt) + + started := make(chan struct{}) + release := make(chan struct{}) + reasons := make(chan DeactivationReason, 2) + var valueEntered atomic.Bool + if err := rt.Register("account", Registration{ + Factory: func(ctx context.Context, _ GrainId) (any, error) { + return &rotatingActivation{request: DeactivateOnIdleFrom(ctx)}, nil + }, + Dispatch: func(_ context.Context, instance any, method string, _ any, _ any) error { + if method == "Fault" { + instance.(*rotatingActivation).request() + close(started) + <-release + panic("broken state") + } + valueEntered.Store(true) + return nil + }, + OnDeactivate: func(_ context.Context, _ GrainId, reason DeactivationReason, _ any) { + reasons <- reason + }, + }); err != nil { + t.Fatal(err) + } + + id := GrainId{GrainType: "account", GrainKey: "alice"} + faultDone := make(chan error, 1) + queuedDone := make(chan error, 1) + go func() { faultDone <- rt.Invoke(context.Background(), id, "Fault", nil, nil) }() + synctest.Wait() + <-started + go func() { queuedDone <- rt.Invoke(context.Background(), id, "Value", nil, nil) }() + synctest.Wait() + close(release) + synctest.Wait() + + if err := <-faultDone; !errors.Is(err, ErrPanic) { + t.Fatalf("faulting Call error = %v, want ErrPanic", err) + } + if err := <-queuedDone; !errors.Is(err, mail.ErrClosed) { + t.Fatalf("queued Call error = %v, want mailbox closed", err) + } + if valueEntered.Load() { + t.Fatal("queued Call entered after the Activation faulted") + } + if reason := <-reasons; reason != Faulted { + t.Fatalf("deactivation reason = %v, want Faulted", reason) + } + }) +} + +func TestRuntime_DiscardWinsOverRequestedDeactivation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) + defer stopEngine(rt) + + discardErr := errors.New("state is not trusted") + started := make(chan struct{}) + release := make(chan struct{}) + reasons := make(chan DeactivationReason, 1) + var valueEntered atomic.Bool + if err := rt.Register("account", Registration{ + Factory: func(ctx context.Context, _ GrainId) (any, error) { + return &rotatingActivation{request: DeactivateOnIdleFrom(ctx)}, nil + }, + Dispatch: func(_ context.Context, instance any, method string, _ any, _ any) error { + if method == "Discard" { + instance.(*rotatingActivation).request() + close(started) + <-release + return Discard{Err: discardErr} + } + valueEntered.Store(true) + return nil + }, + OnDeactivate: func(_ context.Context, _ GrainId, reason DeactivationReason, _ any) { + reasons <- reason + }, + }); err != nil { + t.Fatal(err) + } + + id := GrainId{GrainType: "account", GrainKey: "alice"} + discardDone := make(chan error, 1) + queuedDone := make(chan error, 1) + go func() { discardDone <- rt.Invoke(context.Background(), id, "Discard", nil, nil) }() + synctest.Wait() + <-started + go func() { queuedDone <- rt.Invoke(context.Background(), id, "Value", nil, nil) }() + synctest.Wait() + close(release) + synctest.Wait() + + if err := <-discardDone; !errors.Is(err, discardErr) { + t.Fatalf("discarding Call error = %v, want %v", err, discardErr) + } + if err := <-queuedDone; !errors.Is(err, mail.ErrClosed) { + t.Fatalf("queued Call error = %v, want mailbox closed", err) + } + if valueEntered.Load() { + t.Fatal("queued Call entered after the Activation became untrusted") + } + if reason := <-reasons; reason != Faulted { + t.Fatalf("deactivation reason = %v, want Faulted", reason) + } + }) +} + +func TestRuntime_DeactivationPanicDoesNotSkipCleanup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) + defer stopEngine(rt) + + var factoryCalls atomic.Int32 + if err := rt.Register("account", Registration{ + Factory: func(context.Context, GrainId) (any, error) { + return int(factoryCalls.Add(1)), nil + }, + Dispatch: func(_ context.Context, instance any, _ string, _ any, reply any) error { + *(reply.(*int)) = instance.(int) + return nil + }, + OnDeactivate: func(context.Context, GrainId, DeactivationReason, any) { + panic("hook failure") + }, + }); err != nil { + t.Fatal(err) + } + + id := GrainId{GrainType: "account", GrainKey: "alice"} + var first int + if err := rt.Invoke(context.Background(), id, "Value", nil, &first); err != nil { + t.Fatal(err) + } + rt.Deactivate(id) + synctest.Wait() + + var second int + if err := rt.Invoke(context.Background(), id, "Value", nil, &second); err != nil { + t.Fatalf("Call after panicking hook = %v", err) + } + if first != 1 || second != 2 { + t.Fatalf("Activation generations = %d, %d; want 1, 2", first, second) + } + if got := factoryCalls.Load(); got != 2 { + t.Fatalf("factory calls = %d, want 2", got) + } + }) +} diff --git a/runtime/occupied.go b/internal/runtime/occupied.go similarity index 82% rename from runtime/occupied.go rename to internal/runtime/occupied.go index 9af9d30..86a9b7a 100644 --- a/runtime/occupied.go +++ b/internal/runtime/occupied.go @@ -7,14 +7,14 @@ import ( "strings" ) -// ErrCallCycle reports that a call targeted an entity already occupied by the -// same call chain, so the call could never start. The wrapped message names -// the entities in the cycle. +// ErrCallCycle reports that a Call targeted a Grain already occupied by the +// same Call chain, so the Call could never start. The wrapped message names +// the Grains in the cycle. var ErrCallCycle = errors.New("call cycle detected") type occupiedKey struct{} -// OccupiedFrom returns the entities the current call chain already occupies, +// OccupiedFrom returns the Grains the current Call chain already occupies, // outermost first. The chain travels on the context because Go has no // AsyncLocal; a call whose target is on the chain would close a call cycle. func OccupiedFrom(ctx context.Context) []GrainId { @@ -22,7 +22,7 @@ func OccupiedFrom(ctx context.Context) []GrainId { return chain } -// WithOccupied returns a context that carries chain as its occupied entities. +// WithOccupied returns a context that carries chain as its occupied Grains. func WithOccupied(ctx context.Context, chain []GrainId) context.Context { return context.WithValue(ctx, occupiedKey{}, chain) } @@ -37,7 +37,7 @@ func withOccupied(ctx context.Context, id GrainId) context.Context { } // checkCycle rejects a call whose target already occupies the context's -// chain. The error names the cycle: the occupied entities from the first +// chain. The error names the cycle: the occupied Grains from the first // occurrence of the target to the end of the chain, then the target again. func checkCycle(ctx context.Context, id GrainId) error { chain := OccupiedFrom(ctx) diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go new file mode 100644 index 0000000..885b208 --- /dev/null +++ b/internal/runtime/runtime.go @@ -0,0 +1,1007 @@ +// Package runtime is gor's local Grain engine. It manages Activation, +// lifecycle, mailboxes, and dispatch within one process. +// +// Application code should use the root package's gor.New, gor.Register, +// gor.Ref, and *gor.Runtime APIs instead of importing runtime directly. +package runtime + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + "time" + + "github.com/suraciii/gor/clock" + "github.com/suraciii/gor/internal/mail" +) + +var ( + ErrTypeNotRegistered = errors.New("grain type is not registered") + ErrRuntimeClosed = errors.New("runtime closed") + ErrActivationLimit = errors.New("activation limit reached") + ErrPanic = errors.New("runtime panic") + errCallLaneClosed = errors.New("call lane closed") + errOwnershipLost = errors.New("grain ownership lost") +) + +type GrainId struct { + GrainType string + GrainKey string +} + +type Activation struct { + GrainId GrainId + Queued int +} + +type Dispatch func(context.Context, any, string, any, any) error + +// InvokeOutcome reports the result of one inner Runtime invocation. +// OwnershipLost is true only when the Call did not enter a Grain method and +// the root Runtime must read the latest Cluster View before it routes again. +type InvokeOutcome struct { + Err error + OwnershipLost bool +} + +// DeactivationReason describes why an activation left the active state. It is +// fixed at the first transition out of active and is never rewritten by later +// events. +type DeactivationReason uint8 + +const ( + Idle DeactivationReason = iota + 1 + ApplicationRequested + OwnershipLost + RuntimeClosed + Faulted +) + +type Registration struct { + Factory func(context.Context, GrainId) (any, error) + ActivationContext func(context.Context, context.Context) context.Context + Dispatch Dispatch + OnDeactivate func(context.Context, GrainId, DeactivationReason, any) + OnGrainTimerError func(GrainId, error) +} + +type Discard struct { + Err error +} + +func (d Discard) Error() string { + if d.Err == nil { + return "activation discarded" + } + return d.Err.Error() +} + +func (d Discard) Unwrap() error { + return d.Err +} + +type Config struct { + Clock clock.Clock + MailboxCapacity int + MaxActivations int + IdleTimeout time.Duration + EvictionInterval time.Duration +} + +// engineState tracks the local engine lifecycle independently of the root +// runtime: graceful and sudden stops are distinct, and a sudden stop can +// escalate an in-progress graceful stop. +type engineState uint8 + +const ( + engineRunning engineState = iota + engineClosing + engineKilling +) + +type Runtime struct { + clock clock.Clock + mailboxCapacity int + maxActivations int + idleTimeout time.Duration + + mu sync.Mutex + state engineState + registrations map[string]Registration + activations map[GrainId]*activation + lanes map[GrainId]*callLane + reservedActivations int + timers map[*grainTimer]struct{} + + stop chan struct{} + ticker clock.Ticker + evictionDone chan struct{} + pendingDeactivations int + deactivationsDone chan struct{} + killing chan struct{} + done chan struct{} + killCtx context.Context + killCancel context.CancelFunc +} + +// ActivationState is the explicit state machine an activation walks: an +// activation is created activating, becomes active, then leaves active exactly +// once (deactivating) before it is stopped. The deactivation reason is written +// on that single departure. +type ActivationState uint8 + +const ( + ActivationActivating ActivationState = iota + ActivationActive + ActivationDeactivating + ActivationStopped +) + +type activation struct { + id GrainId + instance any + onDeactivate func(context.Context, GrainId, DeactivationReason, any) + onGrainTimerError func(GrainId, error) + lifecycleCtx context.Context + timerCtx context.Context + timerWaitDone <-chan struct{} + timerWaitCancel context.CancelFunc + lifecycleCancel context.CancelFunc + stopKillCancel func() bool + timers map[*grainTimer]struct{} + skipOnDeactivate bool + hookStarted bool + requested bool + reason DeactivationReason + lane *callLane + lastUsed time.Time + calls int + state ActivationState + done chan struct{} +} + +type callLane struct { + id GrainId + mailbox *mail.Box + done chan struct{} + act *activation + users int + activating bool + accepting bool + activationSlotHeld bool + closeMode callLaneCloseMode +} + +type callExecutionOutcome struct { + err error + laneClosed bool +} + +type callLaneCloseMode uint8 + +const ( + callLaneOpen callLaneCloseMode = iota + callLaneRetry + callLaneOwnershipLost + callLaneReject +) + +func New(config Config) *Runtime { + if config.MaxActivations == 0 { + config.MaxActivations = 10000 + } + killCtx, killCancel := context.WithCancel(context.Background()) + r := &Runtime{ + clock: config.Clock, + mailboxCapacity: config.MailboxCapacity, + maxActivations: config.MaxActivations, + idleTimeout: config.IdleTimeout, + registrations: make(map[string]Registration), + activations: make(map[GrainId]*activation), + lanes: make(map[GrainId]*callLane), + timers: make(map[*grainTimer]struct{}), + stop: make(chan struct{}), + evictionDone: make(chan struct{}), + killing: make(chan struct{}), + done: make(chan struct{}), + killCtx: killCtx, + killCancel: killCancel, + } + if config.IdleTimeout > 0 && config.EvictionInterval > 0 { + r.ticker = config.Clock.NewTicker(config.EvictionInterval) + go r.evictLoop() + } else { + close(r.evictionDone) + } + return r +} + +func (r *Runtime) Register(name string, registration Registration) error { + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.registrations[name]; exists { + return fmt.Errorf("grain type %q is already registered", name) + } + r.registrations[name] = registration + return nil +} + +func (r *Runtime) Invoke(ctx context.Context, id GrainId, method string, args any, reply any) error { + return r.InvokeOutcome(ctx, id, method, args, reply).Err +} + +// InvokeOutcome invokes one Call and keeps an ownership reroute signal +// separate from errors returned by Grain code. +func (r *Runtime) InvokeOutcome(ctx context.Context, id GrainId, method string, args any, reply any) InvokeOutcome { + if err := checkCycle(ctx, id); err != nil { + return InvokeOutcome{Err: err} + } + callCtx, cancel := context.WithCancel(ctx) + defer cancel() + defer context.AfterFunc(r.killCtx, cancel)() + + // The call occupies id from admission on: activation and the method body + // both run with id on the occupied chain, so a nested call that targets + // id again is rejected as a cycle. + callCtx = withOccupied(callCtx, id) + + registration, err := r.registration(id.GrainType) + if err != nil { + return InvokeOutcome{Err: err} + } + for { + lane, err := r.callLaneFor(callCtx, id) + if err != nil { + return InvokeOutcome{Err: err} + } + result, dispatched := lane.mailbox.CallResult(callCtx, func(callCtx context.Context) (any, error) { + execution := r.executeCall(callCtx, lane, registration, method, args, reply) + if execution.laneClosed { + return nil, errCallLaneClosed + } + return nil, execution.err + }) + r.releaseCallLane(lane) + if !dispatched { + if errors.Is(result.Err, mail.ErrClosed) { + retry, outcome := r.closedCallOutcome(lane) + if retry { + continue + } + return outcome + } + return InvokeOutcome{Err: result.Err} + } + if result.Err == errCallLaneClosed { + retry, outcome := r.closedCallOutcome(lane) + if retry { + continue + } + return outcome + } + return InvokeOutcome{Err: result.Err} + } +} + +// Deactivate stops the activation for id because this node no longer owns it +// (or the view has no active owner). The deactivation hook runs with +// OwnershipLost. +func (r *Runtime) Deactivate(id GrainId) { + r.mu.Lock() + act, ok := r.activations[id] + if !ok { + r.mu.Unlock() + return + } + _, closeLane := r.deactivateLocked(act, OwnershipLost, true) + r.mu.Unlock() + if closeLane { + act.lane.mailbox.Close() + } +} + +func (r *Runtime) GrainIds() []GrainId { + r.mu.Lock() + defer r.mu.Unlock() + identities := make([]GrainId, 0, len(r.activations)) + for id := range r.activations { + identities = append(identities, id) + } + return identities +} + +func (r *Runtime) Activations() []Activation { + r.mu.Lock() + activations := make([]Activation, 0, len(r.activations)) + for _, act := range r.activations { + if act.state != ActivationActive { + continue + } + activations = append(activations, Activation{ + GrainId: act.id, + Queued: act.lane.mailbox.Len(), + }) + } + r.mu.Unlock() + + sort.Slice(activations, func(i, j int) bool { + if activations[i].GrainId.GrainType != activations[j].GrainId.GrainType { + return activations[i].GrainId.GrainType < activations[j].GrainId.GrainType + } + return activations[i].GrainId.GrainKey < activations[j].GrainId.GrainKey + }) + return activations +} + +// Done returns a channel that closes when the engine has stopped. For a +// graceful stop that is after deactivation hooks finish; for a sudden stop it +// is after the engine's own goroutines exit, not after user methods. +func (r *Runtime) Done() <-chan struct{} { + return r.done +} + +// BeginClose starts a graceful stop and returns immediately. See Close. +func (r *Runtime) BeginClose() { + r.mu.Lock() + if r.state != engineRunning { + r.mu.Unlock() + return + } + r.state = engineClosing + close(r.stop) + lanes, timerDone := r.beginStopDeactivationsLocked(false) + ticker := r.ticker + r.mu.Unlock() + if ticker != nil { + ticker.Stop() + } + for _, lane := range lanes { + lane.mailbox.Close() + } + go r.drain(lanes, timerDone) +} + +// BeginKill starts a sudden stop, or escalates an in-progress graceful stop, +// and returns immediately. See Kill. +func (r *Runtime) BeginKill() { + r.mu.Lock() + switch r.state { + case engineRunning: + r.state = engineKilling + close(r.stop) + close(r.killing) + lanes, timerDone := r.beginStopDeactivationsLocked(true) + ticker := r.ticker + r.mu.Unlock() + if ticker != nil { + ticker.Stop() + } + r.killCancel() + for _, lane := range lanes { + lane.mailbox.Close() + } + go r.drain(lanes, timerDone) + case engineClosing: + r.state = engineKilling + close(r.killing) + r.markSkipOnDeactivateLocked() + r.mu.Unlock() + r.killCancel() + default: + r.mu.Unlock() + } +} + +// drain waits for the eviction loop to exit, then for either all deactivations +// to finish (graceful) or a sudden stop to overtake them. It never waits for +// user methods once the engine is killing. +func (r *Runtime) drain(lanes []*callLane, timerDone []<-chan struct{}) { + defer close(r.done) + <-r.evictionDone + waitForChannels(timerDone) + for _, lane := range lanes { + select { + case <-lane.mailbox.Done(): + case <-r.killing: + return + } + } + r.mu.Lock() + deactivationsDone := r.deactivationsDone + r.mu.Unlock() + if deactivationsDone == nil { + return + } + select { + case <-deactivationsDone: + case <-r.killing: + } +} + +// beginStopDeactivationsLocked marks deactivation hooks to skip when requested, +// begins deactivation of every active activation, and arms the channel that +// closes once every in-flight deactivation has finished. The caller must hold +// r.mu. +func (r *Runtime) beginStopDeactivationsLocked(skip bool) ([]*callLane, []<-chan struct{}) { + lanes := make([]*callLane, 0, len(r.lanes)) + for _, lane := range r.lanes { + closeCallLane(lane, RuntimeClosed) + lanes = append(lanes, lane) + } + for _, act := range r.activations { + if skip && !act.hookStarted { + act.skipOnDeactivate = true + } + if beginDeactivation(act, RuntimeClosed) { + act.stopLifecycle() + r.startDeactivationWaiterLocked(act, true) + } + } + r.deactivationsDone = make(chan struct{}) + if r.pendingDeactivations == 0 { + close(r.deactivationsDone) + } + timerDone := make([]<-chan struct{}, 0, len(r.timers)) + for timer := range r.timers { + timerDone = append(timerDone, timer.done) + } + return lanes, timerDone +} + +// markSkipOnDeactivateLocked marks every activation's deactivation hook to be +// skipped if it has not started yet. The caller must hold r.mu. +func (r *Runtime) markSkipOnDeactivateLocked() { + for _, act := range r.activations { + if !act.hookStarted { + act.skipOnDeactivate = true + } + } +} + +func (r *Runtime) registration(name string) (Registration, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.state != engineRunning { + return Registration{}, ErrRuntimeClosed + } + registration, ok := r.registrations[name] + if !ok { + return Registration{}, fmt.Errorf("%w: %s", ErrTypeNotRegistered, name) + } + return registration, nil +} + +func (r *Runtime) callLaneFor(ctx context.Context, id GrainId) (*callLane, error) { + for { + r.mu.Lock() + if r.state != engineRunning { + r.mu.Unlock() + return nil, ErrRuntimeClosed + } + if lane, ok := r.lanes[id]; ok { + if lane.accepting { + lane.users++ + r.mu.Unlock() + return lane, nil + } + done := lane.done + r.mu.Unlock() + select { + case <-done: + continue + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + if err := ctx.Err(); err != nil { + r.mu.Unlock() + return nil, err + } + if err := r.reserveActivationSlotLocked(); err != nil { + r.mu.Unlock() + return nil, err + } + lane := &callLane{id: id, done: make(chan struct{}), users: 1, accepting: true, activationSlotHeld: true} + lane.mailbox = mail.New(r.mailboxCapacity, func() { + r.cleanupCanceledCall(lane) + }) + r.lanes[id] = lane + r.mu.Unlock() + return lane, nil + } +} + +func (r *Runtime) releaseCallLane(lane *callLane) { + r.mu.Lock() + if lane.users > 0 { + lane.users-- + } + closeLane := r.cleanupCallLaneLocked(lane) + r.mu.Unlock() + if closeLane { + lane.mailbox.Close() + } +} + +func (r *Runtime) closedCallOutcome(lane *callLane) (retry bool, outcome InvokeOutcome) { + r.mu.Lock() + defer r.mu.Unlock() + switch lane.closeMode { + case callLaneRetry: + if r.state == engineRunning { + return true, InvokeOutcome{} + } + case callLaneOwnershipLost: + return false, InvokeOutcome{Err: errOwnershipLost, OwnershipLost: true} + } + return false, InvokeOutcome{Err: mail.ErrClosed} +} + +func (r *Runtime) cleanupCanceledCall(lane *callLane) { + r.mu.Lock() + closeLane := r.cleanupCallLaneLocked(lane) + r.mu.Unlock() + if closeLane { + lane.mailbox.Close() + } +} + +func (r *Runtime) cleanupCallLaneLocked(lane *callLane) bool { + if current, ok := r.lanes[lane.id]; !ok || current != lane || lane.users != 0 || lane.activating || lane.act != nil || lane.mailbox.Len() != 0 { + return false + } + delete(r.lanes, lane.id) + r.releaseActivationLocked(lane) + lane.accepting = false + close(lane.done) + return true +} + +// reserveActivationLocked admits one new Grain lane. The caller must hold +// r.mu. The same lane can keep its reservation across a failed start retry. +func (r *Runtime) reserveActivationLocked(lane *callLane) error { + if lane.activationSlotHeld { + return nil + } + if err := r.reserveActivationSlotLocked(); err != nil { + return err + } + lane.activationSlotHeld = true + return nil +} + +func (r *Runtime) reserveActivationSlotLocked() error { + if r.reservedActivations >= r.maxActivations { + return ErrActivationLimit + } + r.reservedActivations++ + return nil +} + +// releaseActivationLocked releases a lane's reservation once. The caller must +// hold r.mu. A lane owns the reservation until its Activation is fully stopped. +func (r *Runtime) releaseActivationLocked(lane *callLane) { + if !lane.activationSlotHeld { + return + } + lane.activationSlotHeld = false + if r.reservedActivations > 0 { + r.reservedActivations-- + } +} + +func (r *Runtime) executeCall(ctx context.Context, lane *callLane, registration Registration, method string, args any, reply any) callExecutionOutcome { + for { + act, laneClosed, err := r.activationForTurn(ctx, lane, registration) + if err != nil { + return callExecutionOutcome{err: err, laneClosed: laneClosed} + } + if err := ctx.Err(); err != nil { + return callExecutionOutcome{err: err} + } + if !r.beginTurn(act) { + continue + } + err, faulted := r.dispatch(registration, act, ctx, method, args, reply) + r.finishTurn(act, faulted) + return callExecutionOutcome{err: err} + } +} + +func (r *Runtime) activationForTurn(ctx context.Context, lane *callLane, registration Registration) (*activation, bool, error) { + for { + r.mu.Lock() + if r.state != engineRunning { + r.mu.Unlock() + return nil, false, ErrRuntimeClosed + } + if !lane.accepting || r.lanes[lane.id] != lane { + r.mu.Unlock() + return nil, true, mail.ErrClosed + } + act := lane.act + if act == nil { + if err := r.reserveActivationLocked(lane); err != nil { + r.mu.Unlock() + return nil, false, err + } + lane.activating = true + r.mu.Unlock() + + act, err := r.createActivation(ctx, lane, registration) + + r.mu.Lock() + lane.activating = false + if err == nil && (r.state != engineRunning || !lane.accepting || r.lanes[lane.id] != lane) { + err = ErrRuntimeClosed + } + if err == nil && !finishActivation(act) { + err = fmt.Errorf("%w: invalid activation transition", ErrPanic) + } + var timerDone []<-chan struct{} + if err == nil { + act.lastUsed = r.clock.Now() + lane.act = act + r.activations[lane.id] = act + } else { + if act != nil { + act.stopLifecycle() + act.stopTimerWait() + timerDone = grainTimerDoneChannelsLocked(act) + } + } + var closeLane bool + if err == nil { + closeLane = r.cleanupCallLaneLocked(lane) + } + r.mu.Unlock() + if closeLane { + lane.mailbox.Close() + } + if err != nil { + waitForChannels(timerDone) + r.mu.Lock() + r.releaseActivationLocked(lane) + closeLane = r.cleanupCallLaneLocked(lane) + r.mu.Unlock() + if closeLane { + lane.mailbox.Close() + } + return nil, false, err + } + return act, false, nil + } + + switch act.state { + case ActivationActive: + r.mu.Unlock() + return act, false, nil + case ActivationDeactivating: + done := act.done + r.mu.Unlock() + select { + case <-done: + continue + case <-ctx.Done(): + return nil, false, ctx.Err() + } + case ActivationStopped: + if lane.act == act { + lane.act = nil + } + r.mu.Unlock() + } + } +} + +func (r *Runtime) createActivation(callCtx context.Context, lane *callLane, registration Registration) (act *activation, err error) { + lifecycleCtx, lifecycleCancel := context.WithCancel(context.Background()) + timerWaitCtx, timerWaitCancel := context.WithCancel(r.killCtx) + act = &activation{ + id: lane.id, + onDeactivate: registration.OnDeactivate, + onGrainTimerError: registration.OnGrainTimerError, + lifecycleCtx: lifecycleCtx, + timerCtx: WithOccupied(lifecycleCtx, []GrainId{lane.id}), + timerWaitDone: timerWaitCtx.Done(), + timerWaitCancel: timerWaitCancel, + lifecycleCancel: lifecycleCancel, + stopKillCancel: context.AfterFunc(r.killCtx, lifecycleCancel), + timers: make(map[*grainTimer]struct{}), + lane: lane, + state: ActivationActivating, + done: make(chan struct{}), + } + defer func() { + if value := recover(); value != nil { + err = fmt.Errorf("%w: activation callback panicked: %v", ErrPanic, value) + } + }() + + chain := append([]GrainId(nil), OccupiedFrom(callCtx)...) + activationCtx := WithOccupied(lifecycleCtx, chain) + if registration.ActivationContext != nil { + activationCtx = registration.ActivationContext(activationCtx, callCtx) + } + activationCtx = withDeactivateOnIdle(activationCtx, func() { + r.requestDeactivateOnIdle(act) + }) + activationCtx = withGrainTimerRegistrar(activationCtx, func(callback func(context.Context) error, options GrainTimerOptions) (GrainTimer, error) { + return r.registerGrainTimer(act, callback, options) + }) + instance, err := registration.Factory(activationCtx, lane.id) + if err != nil { + return act, err + } + act.instance = instance + return act, nil +} + +func (act *activation) stopLifecycle() { + if act.stopKillCancel != nil { + act.stopKillCancel() + } + if act.lifecycleCancel != nil { + act.lifecycleCancel() + } +} + +func (act *activation) stopTimerWait() { + if act.timerWaitCancel != nil { + act.timerWaitCancel() + } +} + +func (r *Runtime) beginTurn(act *activation) bool { + return r.beginTurnWithKeepAlive(act, true) +} + +func (r *Runtime) beginTurnWithKeepAlive(act *activation, keepAlive bool) bool { + r.mu.Lock() + defer r.mu.Unlock() + if act.state != ActivationActive || act.lane.act != act || !act.lane.accepting { + return false + } + if keepAlive { + act.lastUsed = r.clock.Now() + } + act.calls++ + return true +} + +func (r *Runtime) finishTurn(act *activation, faulted bool) { + r.finishTurnWithIdleUse(act, faulted, false) +} + +func (r *Runtime) finishGrainTimerTurn(act *activation, faulted bool, keepAlive bool) { + r.finishTurnWithIdleUse(act, faulted, keepAlive) +} + +func (r *Runtime) finishTurnWithIdleUse(act *activation, faulted bool, keepAliveAfterCompletion bool) { + r.mu.Lock() + if act.calls > 0 { + act.calls-- + } + var closeLane bool + if faulted { + _, closeLane = r.deactivateLocked(act, Faulted, true) + } else if act.requested { + _, closeLane = r.deactivateLocked(act, ApplicationRequested, false) + } else if keepAliveAfterCompletion && act.state == ActivationActive { + act.lastUsed = r.clock.Now() + } + r.mu.Unlock() + if closeLane { + act.lane.mailbox.Close() + } +} + +func (r *Runtime) dispatch(registration Registration, act *activation, ctx context.Context, method string, args any, reply any) (err error, faulted bool) { + defer func() { + if value := recover(); value != nil { + err = fmt.Errorf("%w: grain method panicked: %v", ErrPanic, value) + faulted = true + } + }() + err = registration.Dispatch(ctx, act.instance, method, args, reply) + var discard Discard + if errors.As(err, &discard) { + return discard.Err, true + } + return err, false +} + +func (r *Runtime) evictLoop() { + defer close(r.evictionDone) + for { + select { + case now := <-r.ticker.C(): + r.evict(now) + case <-r.stop: + return + } + } +} + +func (r *Runtime) evict(now time.Time) { + r.mu.Lock() + if r.state != engineRunning { + r.mu.Unlock() + return + } + victims := make([]*callLane, 0) + for _, act := range r.activations { + if act.state == ActivationActive && act.calls == 0 && act.lane.mailbox.Len() == 0 && !now.Before(act.lastUsed.Add(r.idleTimeout)) { + if _, closeLane := r.deactivateLocked(act, Idle, true); closeLane { + victims = append(victims, act.lane) + } + } + } + r.mu.Unlock() + + for _, lane := range victims { + lane.mailbox.Close() + } +} + +func (r *Runtime) requestDeactivateOnIdle(act *activation) { + r.mu.Lock() + if act.state == ActivationActivating || act.state == ActivationActive { + act.requested = true + } + if act.state == ActivationActive && act.calls == 0 { + r.deactivateLocked(act, ApplicationRequested, false) + } + r.mu.Unlock() +} + +func (r *Runtime) deactivateLocked(act *activation, reason DeactivationReason, terminal bool) (started bool, closeLane bool) { + if terminal { + closeCallLane(act.lane, reason) + closeLane = true + } + if !beginDeactivation(act, reason) { + return false, closeLane + } + act.stopLifecycle() + if reason == ApplicationRequested { + act.stopTimerWait() + } + r.startDeactivationWaiterLocked(act, terminal) + return true, closeLane +} + +func closeCallLane(lane *callLane, reason DeactivationReason) { + lane.accepting = false + switch reason { + case RuntimeClosed, Faulted: + lane.closeMode = callLaneReject + case OwnershipLost: + if lane.closeMode != callLaneReject { + lane.closeMode = callLaneOwnershipLost + } + case Idle: + if lane.closeMode == callLaneOpen { + lane.closeMode = callLaneRetry + } + } +} + +func (r *Runtime) startDeactivationWaiterLocked(act *activation, waitForMailbox bool) { + r.pendingDeactivations++ + timerDone := grainTimerDoneChannelsLocked(act) + go r.waitForDeactivation(act, waitForMailbox, timerDone) +} + +func (r *Runtime) waitForDeactivation(act *activation, waitForMailbox bool, timerDone []<-chan struct{}) { + defer r.deactivationFinished() + defer r.finishDeactivation(act) + if waitForMailbox { + <-act.lane.mailbox.Done() + } + waitForChannels(timerDone) + act.stopTimerWait() + r.mu.Lock() + var ( + onDeactivate func(context.Context, GrainId, DeactivationReason, any) + reason DeactivationReason + ) + if !act.skipOnDeactivate { + act.hookStarted = true + onDeactivate = act.onDeactivate + reason = act.reason + } + r.mu.Unlock() + if onDeactivate != nil { + callDeactivationHook(onDeactivate, act.id, reason, act.instance) + } +} + +func grainTimerDoneChannelsLocked(act *activation) []<-chan struct{} { + done := make([]<-chan struct{}, 0, len(act.timers)) + for timer := range act.timers { + done = append(done, timer.done) + } + return done +} + +func waitForChannels(channels []<-chan struct{}) { + for _, channel := range channels { + <-channel + } +} + +func callDeactivationHook(hook func(context.Context, GrainId, DeactivationReason, any), id GrainId, reason DeactivationReason, instance any) { + defer func() { + recover() + }() + hook(context.Background(), id, reason, instance) +} + +// deactivationFinished records one deactivation waiter completing. When the +// last in-flight deactivation finishes after a stop has begun, it closes the +// channel the graceful drain waits on. +func (r *Runtime) deactivationFinished() { + r.mu.Lock() + r.pendingDeactivations-- + if r.pendingDeactivations == 0 && r.deactivationsDone != nil { + close(r.deactivationsDone) + r.deactivationsDone = nil + } + r.mu.Unlock() +} + +func (r *Runtime) finishDeactivation(act *activation) { + r.mu.Lock() + if !finishDeactivation(act) { + r.mu.Unlock() + return + } + if current, ok := r.activations[act.id]; ok && current == act { + delete(r.activations, act.id) + } + if act.lane.act == act { + act.lane.act = nil + } + r.releaseActivationLocked(act.lane) + if !act.lane.accepting { + if current, ok := r.lanes[act.id]; ok && current == act.lane { + delete(r.lanes, act.id) + close(act.lane.done) + } + } + closeLane := r.cleanupCallLaneLocked(act.lane) + r.mu.Unlock() + if closeLane { + act.lane.mailbox.Close() + } +} + +func beginDeactivation(act *activation, reason DeactivationReason) bool { + if act.state != ActivationActive { + return false + } + act.state = ActivationDeactivating + act.reason = reason + return true +} + +func finishActivation(act *activation) bool { + if act.state != ActivationActivating { + return false + } + act.state = ActivationActive + return true +} + +func finishDeactivation(act *activation) bool { + if act.state != ActivationDeactivating { + return false + } + act.state = ActivationStopped + close(act.done) + return true +} diff --git a/runtime/runtime_test.go b/internal/runtime/runtime_test.go similarity index 63% rename from runtime/runtime_test.go rename to internal/runtime/runtime_test.go index c1905cf..d9f2ca0 100644 --- a/runtime/runtime_test.go +++ b/internal/runtime/runtime_test.go @@ -3,6 +3,8 @@ package runtime import ( "context" "errors" + "fmt" + "math/rand" "reflect" "sync" "sync/atomic" @@ -11,7 +13,7 @@ import ( "time" "github.com/suraciii/gor/clock" - "github.com/suraciii/gor/mail" + "github.com/suraciii/gor/internal/mail" ) // stopEngine is the test-only equivalent of the root coordinator's wait: begin @@ -21,7 +23,7 @@ func stopEngine(r *Runtime) { <-r.Done() } -type testEntity struct{} +type testGrain struct{} func TestRuntime_ConcurrentFirstCallsDeduplicateActivation(t *testing.T) { synctest.Test(t, func(t *testing.T) { @@ -36,7 +38,7 @@ func TestRuntime_ConcurrentFirstCallsDeduplicateActivation(t *testing.T) { factoryCalls.Add(1) close(factoryStarted) <-releaseFactory - return &testEntity{}, nil + return &testGrain{}, nil }, Dispatch: func(_ context.Context, instance any, method string, _ any, reply any) error { if method != "Ping" { @@ -84,6 +86,242 @@ func TestRuntime_ConcurrentFirstCallsDeduplicateActivation(t *testing.T) { }) } +func TestRuntime_ActivationLimitRejectsBeforeLaneAllocation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1, MaxActivations: 1}) + defer stopEngine(rt) + + factoryStarted := make(chan struct{}) + releaseFactory := make(chan struct{}) + var factoryCalls atomic.Int32 + if err := rt.Register("account", Registration{ + Factory: func(context.Context, GrainId) (any, error) { + factoryCalls.Add(1) + close(factoryStarted) + <-releaseFactory + return &testGrain{}, nil + }, + Dispatch: func(context.Context, any, string, any, any) error { return nil }, + }); err != nil { + t.Fatal(err) + } + + firstID := GrainId{GrainType: "account", GrainKey: "alice"} + firstDone := make(chan error, 1) + go func() { firstDone <- rt.Invoke(context.Background(), firstID, "Ping", nil, nil) }() + synctest.Wait() + <-factoryStarted + + secondID := GrainId{GrainType: "account", GrainKey: "bob"} + err := rt.Invoke(context.Background(), secondID, "Ping", nil, nil) + if !errors.Is(err, ErrActivationLimit) { + t.Errorf("second Invoke error = %v, want ErrActivationLimit", err) + } + if got := factoryCalls.Load(); got != 1 { + t.Errorf("factory calls while limit is full = %d, want 1", got) + } + rt.mu.Lock() + lanes := len(rt.lanes) + reserved := rt.reservedActivations + rt.mu.Unlock() + if lanes != 1 { + t.Errorf("call lanes while limit is full = %d, want 1", lanes) + } + if reserved != 1 { + t.Errorf("reserved Activations while limit is full = %d, want 1", reserved) + } + + close(releaseFactory) + synctest.Wait() + if err := <-firstDone; err != nil { + t.Errorf("first Invoke error = %v", err) + } + }) +} + +func TestRuntime_ActivationLimitPreservesCanceledCall(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1, MaxActivations: 1}) + defer stopEngine(rt) + + releaseFactory := make(chan struct{}) + if err := rt.Register("account", Registration{ + Factory: func(context.Context, GrainId) (any, error) { + <-releaseFactory + return &testGrain{}, nil + }, + Dispatch: func(context.Context, any, string, any, any) error { return nil }, + }); err != nil { + t.Fatal(err) + } + + firstDone := make(chan error, 1) + go func() { + firstDone <- rt.Invoke(context.Background(), GrainId{GrainType: "account", GrainKey: "alice"}, "Ping", nil, nil) + }() + synctest.Wait() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := rt.Invoke(ctx, GrainId{GrainType: "account", GrainKey: "bob"}, "Ping", nil, nil) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled Call at full limit = %v, want context.Canceled", err) + } + + close(releaseFactory) + synctest.Wait() + if err := <-firstDone; err != nil { + t.Fatalf("first Invoke error = %v", err) + } + }) +} + +func TestRuntime_ActivationLimitKeepsActiveGrainAvailable(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1, MaxActivations: 1}) + defer stopEngine(rt) + + hookStarted := make(chan struct{}) + releaseHook := make(chan struct{}) + var factoryCalls atomic.Int32 + if err := rt.Register("account", Registration{ + Factory: func(context.Context, GrainId) (any, error) { + factoryCalls.Add(1) + return &testGrain{}, nil + }, + Dispatch: func(context.Context, any, string, any, any) error { return nil }, + OnDeactivate: func(context.Context, GrainId, DeactivationReason, any) { + close(hookStarted) + <-releaseHook + }, + }); err != nil { + t.Fatal(err) + } + + firstID := GrainId{GrainType: "account", GrainKey: "alice"} + if err := rt.Invoke(context.Background(), firstID, "Ping", nil, nil); err != nil { + t.Fatalf("first Invoke error = %v", err) + } + if err := rt.Invoke(context.Background(), firstID, "Ping", nil, nil); err != nil { + t.Fatalf("Call to active Grain error = %v", err) + } + if got := factoryCalls.Load(); got != 1 { + t.Fatalf("factory calls for active Grain = %d, want 1", got) + } + + rt.Deactivate(firstID) + synctest.Wait() + <-hookStarted + secondID := GrainId{GrainType: "account", GrainKey: "bob"} + if err := rt.Invoke(context.Background(), secondID, "Ping", nil, nil); !errors.Is(err, ErrActivationLimit) { + t.Fatalf("Invoke during deactivation error = %v, want ErrActivationLimit", err) + } + + close(releaseHook) + synctest.Wait() + if err := rt.Invoke(context.Background(), secondID, "Ping", nil, nil); err != nil { + t.Fatalf("Invoke after deactivation error = %v", err) + } + if got := factoryCalls.Load(); got != 2 { + t.Fatalf("factory calls after slot release = %d, want 2", got) + } + }) +} + +func TestRuntime_ActivationLimitReleasesFailedStart(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1, MaxActivations: 1}) + defer stopEngine(rt) + + var factoryCalls atomic.Int32 + startErr := errors.New("start failed") + if err := rt.Register("account", Registration{ + Factory: func(context.Context, GrainId) (any, error) { + if factoryCalls.Add(1) == 1 { + return nil, startErr + } + return &testGrain{}, nil + }, + Dispatch: func(context.Context, any, string, any, any) error { return nil }, + }); err != nil { + t.Fatal(err) + } + + firstID := GrainId{GrainType: "account", GrainKey: "alice"} + if err := rt.Invoke(context.Background(), firstID, "Ping", nil, nil); !errors.Is(err, startErr) { + t.Fatalf("failed start error = %v, want start error", err) + } + rt.mu.Lock() + lanes := len(rt.lanes) + reserved := rt.reservedActivations + rt.mu.Unlock() + if lanes != 0 || reserved != 0 { + t.Fatalf("after failed start: lanes=%d reserved=%d, want 0 and 0", lanes, reserved) + } + + secondID := GrainId{GrainType: "account", GrainKey: "bob"} + if err := rt.Invoke(context.Background(), secondID, "Ping", nil, nil); err != nil { + t.Fatalf("Invoke after failed start error = %v", err) + } + if got := factoryCalls.Load(); got != 2 { + t.Fatalf("factory calls after failed start = %d, want 2", got) + } + }) +} + +func TestRuntime_ActivationLimitReleasesStartInterruptedByClose(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1, MaxActivations: 1}) + defer stopEngine(rt) + + factoryStarted := make(chan struct{}) + releaseFactory := make(chan struct{}) + if err := rt.Register("account", Registration{ + Factory: func(context.Context, GrainId) (any, error) { + close(factoryStarted) + <-releaseFactory + return &testGrain{}, nil + }, + Dispatch: func(context.Context, any, string, any, any) error { return nil }, + }); err != nil { + t.Fatal(err) + } + + callDone := make(chan error, 1) + go func() { + callDone <- rt.Invoke(context.Background(), GrainId{GrainType: "account", GrainKey: "alice"}, "Ping", nil, nil) + }() + synctest.Wait() + <-factoryStarted + + rt.BeginClose() + synctest.Wait() + select { + case <-rt.Done(): + t.Fatal("Runtime closed before interrupted Factory returned") + default: + } + + close(releaseFactory) + synctest.Wait() + if err := <-callDone; !errors.Is(err, ErrRuntimeClosed) { + t.Fatalf("interrupted start error = %v, want ErrRuntimeClosed", err) + } + select { + case <-rt.Done(): + default: + t.Fatal("Runtime did not finish after interrupted Factory returned") + } + rt.mu.Lock() + lanes := len(rt.lanes) + reserved := rt.reservedActivations + rt.mu.Unlock() + if lanes != 0 || reserved != 0 { + t.Fatalf("after interrupted start: lanes=%d reserved=%d, want 0 and 0", lanes, reserved) + } + }) +} + func TestRuntime_DifferentKeysRunConcurrently(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 2}) @@ -92,12 +330,12 @@ func TestRuntime_DifferentKeysRunConcurrently(t *testing.T) { entered := make(chan struct{}, 2) release := make(chan struct{}) if err := rt.Register("account", Registration{ - Factory: func(context.Context, GrainId) (any, error) { return &testEntity{}, nil }, + Factory: func(context.Context, GrainId) (any, error) { return &testGrain{}, nil }, Dispatch: func(_ context.Context, instance any, method string, _ any, _ any) error { if method != "Block" { return errors.New("unknown method") } - _ = instance.(*testEntity) + _ = instance.(*testGrain) entered <- struct{}{} <-release return nil @@ -184,6 +422,121 @@ func TestRuntime_EvictsIdleActivationAndReactivates(t *testing.T) { }) } +func TestRuntime_ActivationChurnReleasesOwnedResources(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + const ( + seed = int64(1945) + grainCount = 16 + cycles = 8 + ) + fakeClock := clock.NewFake(time.Unix(0, 0).UTC()) + rt := New(Config{ + Clock: fakeClock, + MailboxCapacity: 2, + MaxActivations: grainCount, + IdleTimeout: time.Second, + EvictionInterval: time.Second, + }) + defer stopEngine(rt) + + var factoryCalls atomic.Int32 + if err := rt.Register("account", Registration{ + Factory: func(ctx context.Context, _ GrainId) (any, error) { + factoryCalls.Add(1) + if _, err := GrainTimerRegistrarFrom(ctx)(func(context.Context) error { return nil }, GrainTimerOptions{DueTime: time.Hour}); err != nil { + return nil, err + } + return &testGrain{}, nil + }, + Dispatch: func(context.Context, any, string, any, any) error { return nil }, + }); err != nil { + t.Fatal(err) + } + + ids := make([]GrainId, grainCount) + for index := range ids { + ids[index] = GrainId{GrainType: "account", GrainKey: fmt.Sprintf("grain-%02d", index)} + } + random := rand.New(rand.NewSource(seed)) + for cycle := 0; cycle < cycles; cycle++ { + random.Shuffle(len(ids), func(left, right int) { + ids[left], ids[right] = ids[right], ids[left] + }) + for _, id := range ids { + if err := rt.Invoke(context.Background(), id, "Ping", nil, nil); err != nil { + t.Fatalf("cycle %d Invoke %s: %v", cycle, id.GrainKey, err) + } + } + + rt.mu.Lock() + activationCount := len(rt.activations) + laneCount := len(rt.lanes) + reservationCount := rt.reservedActivations + mailboxDone := make([]<-chan struct{}, 0, len(rt.lanes)) + laneDone := make([]<-chan struct{}, 0, len(rt.lanes)) + activationDone := make([]<-chan struct{}, 0, len(rt.activations)) + grainTimerDone := make([]<-chan struct{}, 0, len(rt.timers)) + for _, lane := range rt.lanes { + mailboxDone = append(mailboxDone, lane.mailbox.Done()) + laneDone = append(laneDone, lane.done) + } + for _, activation := range rt.activations { + activationDone = append(activationDone, activation.done) + } + for timer := range rt.timers { + grainTimerDone = append(grainTimerDone, timer.done) + } + timerCount := len(rt.timers) + rt.mu.Unlock() + if activationCount != grainCount || laneCount != grainCount || reservationCount != grainCount || timerCount != grainCount { + t.Fatalf("cycle %d live resources = Activations %d, lanes %d, reservations %d, Grain Timers %d; want %d each", cycle, activationCount, laneCount, reservationCount, timerCount, grainCount) + } + + fakeClock.Advance(2 * time.Second) + synctest.Wait() + for index, done := range mailboxDone { + assertClosed(t, fmt.Sprintf("cycle %d mailbox %d", cycle, index), done) + } + for index, done := range laneDone { + assertClosed(t, fmt.Sprintf("cycle %d lane %d", cycle, index), done) + } + for index, done := range activationDone { + assertClosed(t, fmt.Sprintf("cycle %d Activation %d", cycle, index), done) + } + for index, done := range grainTimerDone { + assertClosed(t, fmt.Sprintf("cycle %d Grain Timer %d", cycle, index), done) + } + + rt.mu.Lock() + activationCount = len(rt.activations) + laneCount = len(rt.lanes) + reservationCount = rt.reservedActivations + timerCount = len(rt.timers) + pendingDeactivations := rt.pendingDeactivations + rt.mu.Unlock() + if activationCount != 0 || laneCount != 0 || reservationCount != 0 || timerCount != 0 || pendingDeactivations != 0 { + t.Fatalf("cycle %d released resources = Activations %d, lanes %d, reservations %d, Grain Timers %d, deactivations %d; want all zero", cycle, activationCount, laneCount, reservationCount, timerCount, pendingDeactivations) + } + } + + if got, want := factoryCalls.Load(), int32(grainCount*cycles); got != want { + t.Fatalf("factory calls = %d, want %d", got, want) + } + rt.BeginClose() + synctest.Wait() + assertClosed(t, "Runtime", rt.Done()) + }) +} + +func assertClosed(t *testing.T, name string, done <-chan struct{}) { + t.Helper() + select { + case <-done: + default: + t.Fatalf("%s is still running", name) + } +} + func TestRuntime_DeactivateStopsActivationAndReactivates(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := New(Config{ @@ -236,7 +589,7 @@ func TestRuntime_CloseWaitsForRunningCall(t *testing.T) { started := make(chan struct{}) release := make(chan struct{}) if err := rt.Register("account", Registration{ - Factory: func(context.Context, GrainId) (any, error) { return &testEntity{}, nil }, + Factory: func(context.Context, GrainId) (any, error) { return &testGrain{}, nil }, Dispatch: func(_ context.Context, _ any, _ string, _ any, _ any) error { close(started) <-release @@ -284,7 +637,7 @@ func TestRuntime_KillCancelsRunningCallAndRejectsQueuedCalls(t *testing.T) { queuedStarted := make(chan struct{}) var calls atomic.Int32 if err := rt.Register("account", Registration{ - Factory: func(context.Context, GrainId) (any, error) { return &testEntity{}, nil }, + Factory: func(context.Context, GrainId) (any, error) { return &testGrain{}, nil }, Dispatch: func(ctx context.Context, _ any, method string, _ any, _ any) error { if method != "Block" { return errors.New("unknown method") @@ -372,7 +725,7 @@ func TestRuntime_KillSkipsPendingDeactivationHook(t *testing.T) { defer stopEngine(rt) hookCalled := make(chan struct{}, 1) if err := rt.Register("account", Registration{ - Factory: func(context.Context, GrainId) (any, error) { return &testEntity{}, nil }, + Factory: func(context.Context, GrainId) (any, error) { return &testGrain{}, nil }, Dispatch: func(context.Context, any, string, any, any) error { return nil }, OnDeactivate: func(context.Context, GrainId, DeactivationReason, any) { hookCalled <- struct{}{} @@ -394,7 +747,7 @@ func TestRuntime_KillSkipsPendingDeactivationHook(t *testing.T) { rt.mu.Unlock() t.Fatal("activation did not enter deactivating state") } - rt.startDeactivationWaiterLocked(act) + rt.startDeactivationWaiterLocked(act, true) rt.mu.Unlock() rt.BeginKill() @@ -408,6 +761,67 @@ func TestRuntime_KillSkipsPendingDeactivationHook(t *testing.T) { }) } +func TestActivationStateTransitions(t *testing.T) { + act := &activation{ + state: ActivationActivating, + done: make(chan struct{}), + } + + if !finishActivation(act) { + t.Fatal("activating activation did not become active") + } + if act.state != ActivationActive { + t.Fatalf("activation state = %v, want active", act.state) + } + if !beginDeactivation(act, ApplicationRequested) { + t.Fatal("active activation did not begin deactivation") + } + if act.state != ActivationDeactivating { + t.Fatalf("activation state = %v, want deactivating", act.state) + } + if act.reason != ApplicationRequested { + t.Fatalf("deactivation reason = %v, want ApplicationRequested", act.reason) + } + if !finishDeactivation(act) { + t.Fatal("deactivating activation did not stop") + } + if act.state != ActivationStopped { + t.Fatalf("activation state = %v, want stopped", act.state) + } + select { + case <-act.done: + default: + t.Fatal("activation Done did not close") + } +} + +func TestCallLaneCloseModeUsesTerminalPrecedence(t *testing.T) { + lane := &callLane{accepting: true} + + closeCallLane(lane, Idle) + if lane.accepting || lane.closeMode != callLaneRetry { + t.Fatalf("Idle close mode = (%v, %v), want (false, retry)", lane.accepting, lane.closeMode) + } + closeCallLane(lane, OwnershipLost) + if lane.closeMode != callLaneOwnershipLost { + t.Fatalf("OwnershipLost close mode = %v, want ownership lost", lane.closeMode) + } + closeCallLane(lane, Faulted) + if lane.closeMode != callLaneReject { + t.Fatalf("Faulted close mode = %v, want reject", lane.closeMode) + } + closeCallLane(lane, OwnershipLost) + if lane.closeMode != callLaneReject { + t.Fatalf("OwnershipLost lowered reject mode to %v", lane.closeMode) + } + + runtimeClosed := &callLane{accepting: true} + closeCallLane(runtimeClosed, RuntimeClosed) + if runtimeClosed.accepting || runtimeClosed.closeMode != callLaneReject { + t.Fatalf("RuntimeClosed close mode = (%v, %v), want (false, reject)", runtimeClosed.accepting, runtimeClosed.closeMode) + } +} + // TestRuntime_DeactivationReasonIsNotRewritten covers the fixed-reason // contract at the transition level: the reason is written once when the // activation leaves active, and a later graceful stop that finds the @@ -430,7 +844,7 @@ func TestRuntime_DeactivationReasonIsNotRewritten(t *testing.T) { defer stopEngine(rt) defer release() if err := rt.Register("account", Registration{ - Factory: func(context.Context, GrainId) (any, error) { return &testEntity{}, nil }, + Factory: func(context.Context, GrainId) (any, error) { return &testGrain{}, nil }, Dispatch: func(context.Context, any, string, any, any) error { return nil }, OnDeactivate: func(context.Context, GrainId, DeactivationReason, any) { <-releaseHook @@ -490,7 +904,7 @@ func TestRuntime_KillEscalationSkipsPendingDeactivationHook(t *testing.T) { started := make(chan struct{}) release := make(chan struct{}) if err := rt.Register("account", Registration{ - Factory: func(context.Context, GrainId) (any, error) { return &testEntity{}, nil }, + Factory: func(context.Context, GrainId) (any, error) { return &testGrain{}, nil }, Dispatch: func(ctx context.Context, _ any, method string, _ any, _ any) error { if method != "Block" { return errors.New("unknown method") @@ -733,7 +1147,7 @@ func TestDiscard_ErrorHandlesNil(t *testing.T) { } } -func TestRuntime_ReactivatesCallsArrivingDuringDeactivation(t *testing.T) { +func TestRuntime_OwnershipLossRejectsQueuedCallsWithoutLocalReactivation(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) defer stopEngine(rt) @@ -768,46 +1182,80 @@ func TestRuntime_ReactivatesCallsArrivingDuringDeactivation(t *testing.T) { synctest.Wait() <-started - rt.mu.Lock() - old := rt.activations[id] - if !beginDeactivation(old, OwnershipLost) { - rt.mu.Unlock() - t.Fatal("activation did not enter deactivating state") - } - rt.mu.Unlock() - old.mailbox.Close() - go rt.waitForDeactivation(old) - - newDone := make(chan struct { + queuedDone := make(chan struct { value int err error }, 1) go func() { var value int err := rt.Invoke(context.Background(), id, "Value", nil, &value) - newDone <- struct { + queuedDone <- struct { value int err error }{value: value, err: err} }() synctest.Wait() select { - case <-newDone: - t.Fatal("call completed before old activation finished deactivating") + case <-queuedDone: + t.Fatal("queued call completed before ownership changed") default: } + rt.Deactivate(id) + close(release) synctest.Wait() if err := <-oldDone; err != nil { t.Fatalf("old call error = %v", err) } - result := <-newDone - if result.err != nil { - t.Fatalf("reactivated call error = %v", result.err) + result := <-queuedDone + if !errors.Is(result.err, errOwnershipLost) { + t.Fatalf("queued call error = %v, want ownership lost", result.err) + } + if result.value != 0 || factoryCalls.Load() != 1 { + t.Fatalf("queued value/factory calls = %d/%d, want 0/1", result.value, factoryCalls.Load()) + } + }) +} + +func TestRuntime_MethodErrorCannotRequestOwnershipReroute(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := New(Config{Clock: clock.Real{}, MailboxCapacity: 1}) + defer stopEngine(rt) + started := make(chan struct{}) + release := make(chan struct{}) + var entered atomic.Bool + if err := rt.Register("account", Registration{ + Factory: func(context.Context, GrainId) (any, error) { return &testGrain{}, nil }, + Dispatch: func(context.Context, any, string, any, any) error { + entered.Store(true) + close(started) + <-release + return mail.ErrClosed + }, + }); err != nil { + t.Fatal(err) + } + + id := GrainId{GrainType: "account", GrainKey: "alice"} + outcomeDone := make(chan InvokeOutcome, 1) + go func() { + outcomeDone <- rt.InvokeOutcome(context.Background(), id, "Value", nil, nil) + }() + synctest.Wait() + <-started + if !entered.Load() { + t.Fatal("Grain method did not enter") + } + rt.Deactivate(id) + close(release) + synctest.Wait() + outcome := <-outcomeDone + if outcome.OwnershipLost { + t.Fatal("started Grain method requested ownership reroute") } - if result.value != 2 || factoryCalls.Load() != 2 { - t.Fatalf("reactivated value/count = %d/%d, want 2/2", result.value, factoryCalls.Load()) + if !errors.Is(outcome.Err, mail.ErrClosed) { + t.Fatalf("method error = %v, want original mail.ErrClosed", outcome.Err) } }) } @@ -821,7 +1269,7 @@ func TestRuntime_SerializesConcurrentCallsPerKey(t *testing.T) { secondStarted := make(chan struct{}) release := make(chan struct{}) if err := rt.Register("account", Registration{ - Factory: func(context.Context, GrainId) (any, error) { return &testEntity{}, nil }, + Factory: func(context.Context, GrainId) (any, error) { return &testGrain{}, nil }, Dispatch: func(_ context.Context, _ any, method string, _ any, _ any) error { if method != "Block" { return errors.New("unknown method") @@ -873,7 +1321,7 @@ func TestRuntime_ActivationsReportsQueuedCallsAndSorts(t *testing.T) { release := make(chan struct{}) var blockCalls atomic.Int32 registration := Registration{ - Factory: func(context.Context, GrainId) (any, error) { return &testEntity{}, nil }, + Factory: func(context.Context, GrainId) (any, error) { return &testGrain{}, nil }, Dispatch: func(_ context.Context, _ any, method string, _ any, _ any) error { if method == "Block" && blockCalls.Add(1) == 1 { close(started) @@ -951,7 +1399,7 @@ func TestRuntime_ActivationsExcludesNonActiveStates(t *testing.T) { close(creating) <-releaseCreate } - return &testEntity{}, nil + return &testGrain{}, nil }, Dispatch: func(context.Context, any, string, any, any) error { return nil }, OnDeactivate: func(context.Context, GrainId, DeactivationReason, any) { diff --git a/internal/testcheck/main.go b/internal/testcheck/main.go new file mode 100644 index 0000000..3b86f33 --- /dev/null +++ b/internal/testcheck/main.go @@ -0,0 +1,91 @@ +// Command testcheck runs Go tests and rejects a test target that starts no +// tests. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" +) + +type testEvent struct { + Action string + Test string + Output string +} + +func main() { + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) +} + +func run(args []string, stdout io.Writer, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprintln(stderr, "testcheck: go test arguments are required") + return 2 + } + commandArgs := append([]string{"test", "-json"}, args...) + command := exec.Command("go", commandArgs...) + command.Stderr = stderr + pipe, err := command.StdoutPipe() + if err != nil { + fmt.Fprintf(stderr, "testcheck: open go test output: %v\n", err) + return 1 + } + if err := command.Start(); err != nil { + fmt.Fprintf(stderr, "testcheck: start go test: %v\n", err) + return 1 + } + + count, decodeErr := copyTestOutput(pipe, stdout) + if decodeErr != nil { + _ = command.Process.Kill() + } + waitErr := command.Wait() + if decodeErr != nil { + fmt.Fprintf(stderr, "testcheck: decode go test output: %v\n", decodeErr) + return 1 + } + if waitErr != nil { + var exitError *exec.ExitError + if errors.As(waitErr, &exitError) { + return exitError.ExitCode() + } + fmt.Fprintf(stderr, "testcheck: wait for go test: %v\n", waitErr) + return 1 + } + if count == 0 { + fmt.Fprintln(stderr, "testcheck: no tests started") + return 1 + } + text := "tests" + if count == 1 { + text = "test" + } + fmt.Fprintf(stderr, "testcheck: %d %s started\n", count, text) + return 0 +} + +func copyTestOutput(input io.Reader, output io.Writer) (int, error) { + decoder := json.NewDecoder(input) + count := 0 + for { + var event testEvent + if err := decoder.Decode(&event); err != nil { + if errors.Is(err, io.EOF) { + return count, nil + } + return count, err + } + if event.Output != "" { + if _, err := io.WriteString(output, event.Output); err != nil { + return count, err + } + } + if event.Action == "run" && event.Test != "" { + count++ + } + } +} diff --git a/internal/testcheck/main_test.go b/internal/testcheck/main_test.go new file mode 100644 index 0000000..4e0662e --- /dev/null +++ b/internal/testcheck/main_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "bytes" + "strings" + "testing" +) + +func TestRunPassesNonzeroSelection(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run([]string{"-run", "^TestPass$", "./testdata/pass"}, &stdout, &stderr) + if code != 0 { + t.Fatalf("run exit = %d, want 0\nstdout:\n%s\nstderr:\n%s", code, stdout.String(), stderr.String()) + } + if !strings.Contains(stderr.String(), "testcheck: 1 test started") { + t.Fatalf("stderr = %q, want nonzero test report", stderr.String()) + } +} + +func TestRunRejectsZeroSelection(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run([]string{"-run", "^TestMissing$", "./testdata/pass"}, &stdout, &stderr) + if code == 0 { + t.Fatalf("run exit = 0, want failure\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) + } + if !strings.Contains(stderr.String(), "testcheck: no tests started") { + t.Fatalf("stderr = %q, want zero-test error", stderr.String()) + } +} + +func TestRunKeepsGoTestFailure(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run([]string{"-run", "^TestFail$", "./testdata/pass"}, &stdout, &stderr) + if code != 1 { + t.Fatalf("run exit = %d, want 1\nstdout:\n%s\nstderr:\n%s", code, stdout.String(), stderr.String()) + } + if !strings.Contains(stdout.String(), "intentional failure") { + t.Fatalf("stdout = %q, want child test failure", stdout.String()) + } +} diff --git a/internal/testcheck/makefile_release_test.go b/internal/testcheck/makefile_release_test.go new file mode 100644 index 0000000..1a9ce51 --- /dev/null +++ b/internal/testcheck/makefile_release_test.go @@ -0,0 +1,157 @@ +//go:build release + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +const releaseCommandCheck = "go run ./internal/testcheck -tags release ./internal/testcheck " + + "-run '^TestMakefile_ReleaseCommandsKeepTestcheck$' -count=1" + +var releaseCommandContracts = []struct { + target string + commands []string +}{ + { + target: "ci", + commands: []string{ + "gofmt -l .", + "go mod tidy -diff", + "go run ./internal/constraintcheck", + "go vet ./...", + "go tool staticcheck ./...", + "go vet -tags sim ./sim/...", + "go tool staticcheck -tags sim ./sim/...", + releaseCommandCheck, + "go run ./internal/testcheck ./...", + "go run ./internal/testcheck -count=1 -race ./...", + "go run ./internal/testcheck -tags sim -run '^TestSim' ./sim/...", + "go run ./internal/testcheck -tags gen ./cmd/gorgen/...", + "go tool gorgen -pkg ./cmd/gorgen/testfixture/endtoend/domain -check", + "go tool gorgen -pkg ./examples/shadow/domain -check", + "go run ./internal/testcheck -tags net ./transport/...", + "go run ./internal/testcheck -tags net ./examples/shadow/...", + "go run ./internal/testcheck ./internal/runtime -run '^TestRuntime_ActivationChurnReleasesOwnedResources$' -count=1", + "go run ./internal/testcheck ./internal/timer -run '^TestPoller_LargeBacklogKeepsPagesAndWorkersBounded$' -count=1", + "go run ./internal/testcheck ./transport -run '^$' -fuzz '^FuzzReadFrame$' -fuzztime=10s", + "go run ./internal/testcheck . -run '^$' -fuzz '^FuzzDecodeRequestContext$' -fuzztime=10s", + "go run ./internal/testcheck ./internal/codegen -run '^$' -fuzz '^FuzzParseGrainMarker$' -fuzztime=10s", + }, + }, + { + target: "external", + commands: []string{ + releaseCommandCheck, + "go run ./internal/testcheck -tags release -run '^TestQuickStartHTTPProcess$' ./examples/shadow/cmd/shadow", + "go run ./internal/testcheck -tags release -run '^TestExternalModule(Configuration|RestartProof|UpgradeProof)$' ./examples/shadow/cmd/conformance", + }, + }, + { + target: "external-tagged", + commands: []string{ + releaseCommandCheck, + "go run ./internal/testcheck -tags release -run '^TestExternalModule(Configuration|RestartProof|UpgradeProof)$' ./examples/shadow/cmd/conformance -args -external-tagged", + }, + }, +} + +func TestMakefile_ReleaseCommandsKeepTestcheck(t *testing.T) { + repository := repositoryRoot(t) + for _, contract := range releaseCommandContracts { + t.Run(contract.target, func(t *testing.T) { + normal := makeDryRun(t, repository, contract.target) + requireCommandsInOrder(t, contract.target, normal, contract.commands) + for _, settings := range [][]string{ + {"GO_TEST=true"}, + {"MAKE=true"}, + {"GO_TEST=true", "MAKE=true"}, + {"-j8"}, + {"-j8", "GO_TEST=true", "MAKE=true"}, + } { + overridden := makeDryRun(t, repository, contract.target, settings...) + if normal != overridden { + line, want, got := firstDifferentLine(normal, overridden) + t.Fatalf("make -n %s changed with %s at line %d: got %q, want %q", contract.target, strings.Join(settings, " "), line, got, want) + } + } + }) + } +} + +func repositoryRoot(t *testing.T) string { + t.Helper() + workingDirectory, err := os.Getwd() + if err != nil { + t.Fatalf("get working directory: %v", err) + } + repository := filepath.Clean(filepath.Join(workingDirectory, "..", "..")) + if _, err := os.Stat(filepath.Join(repository, "Makefile")); err != nil { + t.Fatalf("find repository Makefile: %v", err) + } + return repository +} + +func makeDryRun(t *testing.T, repository string, target string, settings ...string) string { + t.Helper() + args := []string{"-n"} + args = append(args, settings...) + args = append(args, target) + command := exec.Command("make", args...) + command.Dir = repository + command.Env = cleanMakeEnvironment() + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("make -n %s: %v\n%s", target, err, output) + } + return string(output) +} + +func cleanMakeEnvironment() []string { + environment := make([]string, 0, len(os.Environ())) + for _, value := range os.Environ() { + name, _, _ := strings.Cut(value, "=") + switch strings.ToUpper(name) { + case "GO_TEST", "GNUMAKEFLAGS", "MAKE", "MAKEFLAGS", "MAKELEVEL", "MAKEOVERRIDES", "MFLAGS": + continue + } + environment = append(environment, value) + } + return environment +} + +func requireCommandsInOrder(t *testing.T, target string, plan string, commands []string) { + t.Helper() + offset := 0 + for _, command := range commands { + index := strings.Index(plan[offset:], command) + if index < 0 { + t.Fatalf("make -n %s does not run %q in the required order", target, command) + } + offset += index + len(command) + } +} + +func firstDifferentLine(want string, got string) (int, string, string) { + wantLines := strings.Split(want, "\n") + gotLines := strings.Split(got, "\n") + lineCount := max(len(wantLines), len(gotLines)) + for index := range lineCount { + var wantLine string + if index < len(wantLines) { + wantLine = wantLines[index] + } + var gotLine string + if index < len(gotLines) { + gotLine = gotLines[index] + } + if wantLine != gotLine { + return index + 1, wantLine, gotLine + } + } + return 0, "", "" +} diff --git a/internal/testcheck/testdata/pass/pass_test.go b/internal/testcheck/testdata/pass/pass_test.go new file mode 100644 index 0000000..0fb1d6f --- /dev/null +++ b/internal/testcheck/testdata/pass/pass_test.go @@ -0,0 +1,9 @@ +package pass + +import "testing" + +func TestPass(t *testing.T) {} + +func TestFail(t *testing.T) { + t.Fatal("intentional failure") +} diff --git a/internal/timer/timer.go b/internal/timer/timer.go new file mode 100644 index 0000000..228b397 --- /dev/null +++ b/internal/timer/timer.go @@ -0,0 +1,342 @@ +// Package timer polls persisted Reminders and delivers due Grain Calls for +// gor. +// +// It is an implementation package, not an application dependency. Create and +// manage Reminders through the root gor package's Reminder APIs instead of +// importing timer directly. +package timer + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/suraciii/gor/clock" + "github.com/suraciii/gor/store" +) + +type Table interface { + ListDue(context.Context, time.Time, *store.ReminderCursor, int) (store.ReminderPage, error) + Claim(context.Context, store.Reminder, time.Time) (bool, error) +} + +type Invoker interface { + Owns(store.GrainId) bool + Invoke(context.Context, store.GrainId, string, any, any) error +} + +// ReminderCallBuilder creates the typed request and reply values for one +// resolved Reminder method. +type ReminderCallBuilder func(string, time.Time, time.Duration, time.Time) (any, any) + +// ReminderCallFactory resolves one stored GrainType and method before Claim. +type ReminderCallFactory func(store.GrainId, string) (ReminderCallBuilder, error) + +// FailureKind identifies one failed Reminder poller operation. +type FailureKind uint8 + +const ( + // FailureScan reports a failed due-row scan. + FailureScan FailureKind = iota + 1 + // FailureDispatch reports a stored GrainType or method that cannot be + // resolved before Claim. + FailureDispatch + // FailureClaim reports a Store error from Claim. + FailureClaim + // FailureTerminal reports a Store error while an Invalid Reminder gets a + // Terminal Result. + FailureTerminal + // FailureInvoke reports a failed claimed Reminder Call. + FailureInvoke +) + +// Failure describes one Reminder failure for the root Runtime error sink. +type Failure struct { + Kind FailureKind + Reminder store.Reminder + CurrentTickTime time.Time + Err error +} + +// Config defines the fixed dependencies and work limits for one Poller. +type Config struct { + Table Table + Clock clock.Clock + Interval time.Duration + PageSize int + Workers int + Invoker Invoker + NewCall ReminderCallFactory + OnFailure func(Failure) +} + +// Poller has one coordinator and a fixed number of active Claim and Invoke +// workers. +type Poller struct { + table Table + clock clock.Clock + interval time.Duration + pageSize int + invoker Invoker + newCall ReminderCallFactory + onFailure func(Failure) + slots chan struct{} + workers sync.WaitGroup + + ctx context.Context + cancel context.CancelFunc + done chan struct{} +} + +// New starts one Poller. The root Runtime validates all Config values first. +func New(config Config) *Poller { + ctx, cancel := context.WithCancel(context.Background()) + poller := &Poller{ + table: config.Table, + clock: config.Clock, + interval: config.Interval, + pageSize: config.PageSize, + invoker: config.Invoker, + newCall: config.NewCall, + onFailure: config.OnFailure, + slots: make(chan struct{}, config.Workers), + ctx: ctx, + cancel: cancel, + done: make(chan struct{}), + } + go poller.run(poller.clock.NewTicker(poller.interval)) + return poller +} + +// Close stops scanning, cancels active delivery waits, and waits for all +// Poller goroutines. It is safe to call more than once. +func (p *Poller) Close() { + p.cancel() + <-p.done +} + +func (p *Poller) run(ticker clock.Ticker) { + defer func() { + ticker.Stop() + p.workers.Wait() + close(p.done) + }() + + for { + select { + case <-ticker.C(): + p.poll() + case <-p.ctx.Done(): + return + } + } +} + +func (p *Poller) poll() { + now := p.clock.Now() + var after *store.ReminderCursor + for { + page, err := p.table.ListDue(p.ctx, now, after, p.pageSize) + if err != nil { + p.report(Failure{Kind: FailureScan, Err: err}) + return + } + if err := validatePage(page, after, p.pageSize); err != nil { + p.report(Failure{Kind: FailureScan, Err: err}) + return + } + for _, reminder := range page.Rows { + if p.ctx.Err() != nil { + return + } + if !p.invoker.Owns(reminder.GrainId) { + continue + } + call, err := p.resolveCall(reminder) + if err != nil { + if !p.acquireWorker() { + return + } + p.workers.Add(1) + go p.terminateInvalid(reminder, err) + continue + } + if !p.acquireWorker() { + return + } + p.workers.Add(1) + go p.deliver(reminder, call) + } + if page.Next == nil { + return + } + next := *page.Next + after = &next + } +} + +func validatePage(page store.ReminderPage, after *store.ReminderCursor, limit int) error { + if len(page.Rows) > limit { + return errors.New("reminder store returned more rows than the page limit") + } + var previous store.ReminderCursor + hasPrevious := false + if after != nil { + previous = *after + hasPrevious = true + } + for _, reminder := range page.Rows { + current := store.ReminderCursor{DueAt: reminder.DueAt, GrainId: reminder.GrainId, Name: reminder.Name} + if hasPrevious && !cursorBefore(previous, current) { + return errors.New("reminder store rows are not in cursor order") + } + previous = current + hasPrevious = true + } + if page.Next == nil { + return nil + } + if len(page.Rows) == 0 { + return errors.New("reminder store returned a cursor without rows") + } + last := page.Rows[len(page.Rows)-1] + want := store.ReminderCursor{DueAt: last.DueAt, GrainId: last.GrainId, Name: last.Name} + if !sameCursor(*page.Next, want) { + return errors.New("reminder store cursor does not identify the last row") + } + return nil +} + +func sameCursor(left store.ReminderCursor, right store.ReminderCursor) bool { + return left.DueAt.Equal(right.DueAt) && left.GrainId == right.GrainId && left.Name == right.Name +} + +func cursorBefore(left store.ReminderCursor, right store.ReminderCursor) bool { + if !left.DueAt.Equal(right.DueAt) { + return left.DueAt.Before(right.DueAt) + } + if left.GrainId.GrainType != right.GrainId.GrainType { + return left.GrainId.GrainType < right.GrainId.GrainType + } + if left.GrainId.GrainKey != right.GrainId.GrainKey { + return left.GrainId.GrainKey < right.GrainId.GrainKey + } + return left.Name < right.Name +} + +func (p *Poller) resolveCall(reminder store.Reminder) (ReminderCallBuilder, error) { + if p.newCall == nil { + return nil, errors.New("reminder call factory is not configured") + } + return p.newCall(reminder.GrainId, reminder.Method) +} + +func (p *Poller) acquireWorker() bool { + if p.ctx.Err() != nil { + return false + } + select { + case p.slots <- struct{}{}: + if p.ctx.Err() != nil { + <-p.slots + return false + } + return true + case <-p.ctx.Done(): + return false + } +} + +func (p *Poller) deliver(reminder store.Reminder, call ReminderCallBuilder) { + defer p.releaseWorker() + if p.ctx.Err() != nil { + return + } + + claimTime := p.clock.Now() + claimed, err := p.table.Claim(p.ctx, reminder, nextDueAt(reminder, claimTime)) + if err != nil { + p.report(Failure{Kind: FailureClaim, Reminder: reminder, Err: err}) + return + } + if !claimed { + return + } + + currentTickTime := p.clock.Now() + args, reply := call(reminder.Name, reminder.FirstTickTime, reminder.Interval, currentTickTime) + if err := p.invoker.Invoke(p.ctx, reminder.GrainId, reminder.Method, args, reply); err != nil { + p.report(Failure{ + Kind: FailureInvoke, + Reminder: reminder, + CurrentTickTime: currentTickTime, + Err: err, + }) + } +} + +func (p *Poller) terminateInvalid(reminder store.Reminder, dispatchErr error) { + defer p.releaseWorker() + if p.ctx.Err() != nil { + return + } + + claimed, err := p.table.Claim(p.ctx, reminder, time.Time{}) + if err != nil { + p.report(Failure{Kind: FailureTerminal, Reminder: reminder, Err: err}) + return + } + if !claimed { + return + } + p.report(Failure{Kind: FailureDispatch, Reminder: reminder, Err: dispatchErr}) +} + +func (p *Poller) releaseWorker() { + <-p.slots + p.workers.Done() +} + +func (p *Poller) report(failure Failure) { + if p.onFailure == nil || p.ctx.Err() != nil { + return + } + p.onFailure(failure) +} + +const maxDuration = time.Duration(1<<63 - 1) + +func nextDueAt(reminder store.Reminder, now time.Time) time.Time { + period := reminder.Interval + if period <= 0 { + return time.Time{} + } + if reminder.DueAt.After(now) { + return reminder.DueAt + } + + elapsed := now.Sub(reminder.DueAt) + missed := elapsed / period + if missed == maxDuration { + return futureDueAt(now, period) + } + missed++ + if missed > maxDuration/period { + return futureDueAt(now, period) + } + + candidate := reminder.DueAt.Add(period * missed) + if !candidate.After(now) { + return futureDueAt(now, period) + } + return candidate +} + +func futureDueAt(now time.Time, period time.Duration) time.Time { + fallback := now.Add(period) + if fallback.After(now) { + return fallback + } + return now.Add(time.Nanosecond) +} diff --git a/internal/timer/timer_test.go b/internal/timer/timer_test.go new file mode 100644 index 0000000..dafa841 --- /dev/null +++ b/internal/timer/timer_test.go @@ -0,0 +1,1386 @@ +package timer + +import ( + "context" + "errors" + "fmt" + "math/rand" + "slices" + "sort" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/suraciii/gor/clock" + "github.com/suraciii/gor/store" +) + +type fakeTable struct { + rows []store.Reminder + claimWon bool + listErr error + claimErr error + + recorder *stepRecorder + nextDueAt []time.Time + claimCalls atomic.Int32 + afterClaim func() +} + +func (t *fakeTable) ListDue(_ context.Context, now time.Time, after *store.ReminderCursor, limit int) (store.ReminderPage, error) { + t.recorder.record("list") + if t.listErr != nil { + return store.ReminderPage{}, t.listErr + } + rows := make([]store.Reminder, 0, len(t.rows)) + for _, reminder := range t.rows { + if reminder.DueAt.After(now) || !testReminderAfter(reminder, after) { + continue + } + rows = append(rows, reminder) + } + sort.Slice(rows, func(left, right int) bool { + return testReminderBefore(rows[left], rows[right]) + }) + if len(rows) <= limit { + return store.ReminderPage{Rows: rows}, nil + } + rows = rows[:limit] + last := rows[len(rows)-1] + return store.ReminderPage{ + Rows: rows, + Next: &store.ReminderCursor{DueAt: last.DueAt, GrainId: last.GrainId, Name: last.Name}, + }, nil +} + +func testReminderBefore(left store.Reminder, right store.Reminder) bool { + if !left.DueAt.Equal(right.DueAt) { + return left.DueAt.Before(right.DueAt) + } + if left.GrainId.GrainType != right.GrainId.GrainType { + return left.GrainId.GrainType < right.GrainId.GrainType + } + if left.GrainId.GrainKey != right.GrainId.GrainKey { + return left.GrainId.GrainKey < right.GrainId.GrainKey + } + return left.Name < right.Name +} + +func testReminderAfter(reminder store.Reminder, cursor *store.ReminderCursor) bool { + if cursor == nil { + return true + } + return testReminderBefore(store.Reminder{DueAt: cursor.DueAt, GrainId: cursor.GrainId, Name: cursor.Name}, reminder) +} + +func (t *fakeTable) Claim(_ context.Context, schedule store.Reminder, nextDueAt time.Time) (bool, error) { + t.claimCalls.Add(1) + t.recorder.mu.Lock() + t.recorder.steps = append(t.recorder.steps, "claim") + t.nextDueAt = append(t.nextDueAt, nextDueAt) + t.recorder.mu.Unlock() + if t.claimErr != nil { + return false, t.claimErr + } + if t.claimWon && t.afterClaim != nil { + t.afterClaim() + } + return t.claimWon, nil +} + +type recordingInvoker struct { + recorder *stepRecorder + calls []store.GrainId +} + +func testReminderCall(store.GrainId, string, string, time.Time, time.Duration, time.Time) (any, any) { + return &struct{}{}, &struct{}{} +} + +type testCallFactory func(store.GrainId, string, string, time.Time, time.Duration, time.Time) (any, any) + +func newTestPoller(table Table, sourceClock clock.Clock, invoker Invoker, factory testCallFactory) *Poller { + return New(testPollerConfig(table, sourceClock, invoker, factory)) +} + +func testPollerConfig(table Table, sourceClock clock.Clock, invoker Invoker, factory testCallFactory) Config { + return Config{ + Table: table, + Clock: sourceClock, + Interval: time.Second, + PageSize: 256, + Workers: 16, + Invoker: invoker, + NewCall: func(id store.GrainId, method string) (ReminderCallBuilder, error) { + if factory == nil { + return nil, errors.New("unknown Reminder method") + } + return func(name string, first time.Time, period time.Duration, current time.Time) (any, any) { + return factory(id, method, name, first, period, current) + }, nil + }, + } +} + +func (i *recordingInvoker) Invoke(_ context.Context, id store.GrainId, method string, _, _ any) error { + i.recorder.mu.Lock() + i.recorder.steps = append(i.recorder.steps, "invoke") + i.calls = append(i.calls, id) + i.recorder.mu.Unlock() + return nil +} + +func (i *recordingInvoker) Owns(store.GrainId) bool { + return true +} + +type blockingInvoker struct { + started chan struct{} + finished chan struct{} +} + +type stepRecorder struct { + mu sync.Mutex + steps []string +} + +func (r *stepRecorder) record(step string) { + r.mu.Lock() + r.steps = append(r.steps, step) + r.mu.Unlock() +} + +func (i *blockingInvoker) Invoke(ctx context.Context, _ store.GrainId, _ string, _, _ any) error { + close(i.started) + <-ctx.Done() + close(i.finished) + return ctx.Err() +} + +func (i *blockingInvoker) Owns(store.GrainId) bool { + return true +} + +type ownershipInvoker struct { + owns bool + calls atomic.Int32 +} + +func (i *ownershipInvoker) Owns(store.GrainId) bool { + return i.owns +} + +func (i *ownershipInvoker) Invoke(context.Context, store.GrainId, string, any, any) error { + i.calls.Add(1) + return nil +} + +type afterClaimClock struct { + *clock.Fake + claimed *atomic.Bool + after time.Time +} + +func (c *afterClaimClock) Now() time.Time { + if c.claimed.Load() { + return c.after + } + return c.Fake.Now() +} + +type boundedInvoker struct { + started chan struct{} + active atomic.Int32 + maximum atomic.Int32 +} + +type backlogTable struct { + rows []store.Reminder + requestedLimits []int + maximumPage int + claimCalls atomic.Int32 +} + +type boundedInvalidTable struct { + rows []store.Reminder + started chan struct{} + active atomic.Int32 + maximum atomic.Int32 + calls atomic.Int32 +} + +type gatedTerminalTable struct { + *store.Memory + claimStarted chan struct{} + continueClaim chan struct{} +} + +func (t *gatedTerminalTable) Claim(ctx context.Context, reminder store.Reminder, nextDueAt time.Time) (bool, error) { + if nextDueAt.IsZero() { + close(t.claimStarted) + select { + case <-t.continueClaim: + case <-ctx.Done(): + return false, ctx.Err() + } + } + return t.Memory.Claim(ctx, reminder, nextDueAt) +} + +func (t *boundedInvalidTable) ListDue(_ context.Context, _ time.Time, after *store.ReminderCursor, limit int) (store.ReminderPage, error) { + if after != nil { + return store.ReminderPage{}, nil + } + return store.ReminderPage{Rows: slices.Clone(t.rows[:min(limit, len(t.rows))])}, nil +} + +func (t *boundedInvalidTable) Claim(ctx context.Context, _ store.Reminder, _ time.Time) (bool, error) { + t.calls.Add(1) + active := t.active.Add(1) + defer t.active.Add(-1) + for { + maximum := t.maximum.Load() + if active <= maximum || t.maximum.CompareAndSwap(maximum, active) { + break + } + } + t.started <- struct{}{} + <-ctx.Done() + return false, ctx.Err() +} + +func (t *backlogTable) ListDue(_ context.Context, _ time.Time, after *store.ReminderCursor, limit int) (store.ReminderPage, error) { + t.requestedLimits = append(t.requestedLimits, limit) + start := sort.Search(len(t.rows), func(index int) bool { + return testReminderAfter(t.rows[index], after) + }) + end := min(start+limit, len(t.rows)) + rows := slices.Clone(t.rows[start:end]) + if len(rows) > t.maximumPage { + t.maximumPage = len(rows) + } + page := store.ReminderPage{Rows: rows} + if end < len(t.rows) { + last := rows[len(rows)-1] + page.Next = &store.ReminderCursor{DueAt: last.DueAt, GrainId: last.GrainId, Name: last.Name} + } + return page, nil +} + +func (t *backlogTable) Claim(context.Context, store.Reminder, time.Time) (bool, error) { + t.claimCalls.Add(1) + return true, nil +} + +type controlledInvoker struct { + started chan struct{} + release chan struct{} + active atomic.Int32 + maximum atomic.Int32 + ended atomic.Int32 + mu sync.Mutex + delivered map[store.GrainId]int +} + +func (i *controlledInvoker) Owns(store.GrainId) bool { + return true +} + +func (i *controlledInvoker) Invoke(ctx context.Context, id store.GrainId, _ string, _, _ any) error { + active := i.active.Add(1) + defer i.active.Add(-1) + for { + maximum := i.maximum.Load() + if active <= maximum || i.maximum.CompareAndSwap(maximum, active) { + break + } + } + i.mu.Lock() + i.delivered[id]++ + i.mu.Unlock() + i.started <- struct{}{} + select { + case <-i.release: + i.ended.Add(1) + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (i *boundedInvoker) Owns(store.GrainId) bool { + return true +} + +func (i *boundedInvoker) Invoke(ctx context.Context, _ store.GrainId, _ string, _, _ any) error { + active := i.active.Add(1) + for { + maximum := i.maximum.Load() + if active <= maximum || i.maximum.CompareAndSwap(maximum, active) { + break + } + } + i.started <- struct{}{} + <-ctx.Done() + i.active.Add(-1) + return ctx.Err() +} + +type appliedClaimErrorTable struct { + *store.Memory + err error +} + +type stuckCursorTable struct { + row store.Reminder + listCalls atomic.Int32 + claimCalls atomic.Int32 +} + +type oversizedPageTable struct { + rows []store.Reminder + claimCalls atomic.Int32 +} + +func (t *oversizedPageTable) ListDue(context.Context, time.Time, *store.ReminderCursor, int) (store.ReminderPage, error) { + return store.ReminderPage{Rows: t.rows}, nil +} + +func (t *oversizedPageTable) Claim(context.Context, store.Reminder, time.Time) (bool, error) { + t.claimCalls.Add(1) + return false, nil +} + +func (t *stuckCursorTable) ListDue(context.Context, time.Time, *store.ReminderCursor, int) (store.ReminderPage, error) { + if t.listCalls.Add(1) > 2 { + return store.ReminderPage{}, nil + } + return store.ReminderPage{ + Rows: []store.Reminder{t.row}, + Next: &store.ReminderCursor{DueAt: t.row.DueAt, GrainId: t.row.GrainId, Name: t.row.Name}, + }, nil +} + +func (t *stuckCursorTable) Claim(context.Context, store.Reminder, time.Time) (bool, error) { + t.claimCalls.Add(1) + return false, nil +} + +func (t *appliedClaimErrorTable) Claim(ctx context.Context, reminder store.Reminder, nextDueAt time.Time) (bool, error) { + won, err := t.Memory.Claim(ctx, reminder, nextDueAt) + if err != nil || !won { + return won, err + } + return true, t.err +} + +func TestPoller_ClaimsBeforeInvoking(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(100, 0).UTC() + recorder := &stepRecorder{} + backend := &fakeTable{ + rows: []store.Reminder{{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + DueAt: start.Add(-time.Second), + Interval: time.Hour, + ETag: 1, + }}, + claimWon: true, + recorder: recorder, + } + fakeClock := clock.NewFake(start) + invoker := &recordingInvoker{recorder: recorder} + poller := newTestPoller(backend, fakeClock, invoker, testReminderCall) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + + if got, want := recorder.steps, []string{"list", "claim", "invoke"}; !slices.Equal(got, want) { + t.Fatalf("steps = %v, want %v", got, want) + } + }) +} + +func TestPoller_AdvancesToFirstFutureTime(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(200, 0).UTC() + interval := time.Hour + backend := &fakeTable{ + rows: []store.Reminder{{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + DueAt: start.Add(-3 * interval), + Interval: interval, + ETag: 1, + }}, + claimWon: true, + recorder: &stepRecorder{}, + } + fakeClock := clock.NewFake(start) + invoker := &recordingInvoker{recorder: backend.recorder} + poller := newTestPoller(backend, fakeClock, invoker, testReminderCall) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + + if len(backend.nextDueAt) != 1 { + t.Fatalf("next due times = %v, want one claim", backend.nextDueAt) + } + want := start.Add(interval) + if !backend.nextDueAt[0].Equal(want) { + t.Fatalf("next due time = %s, want %s", backend.nextDueAt[0], want) + } + }) +} + +func TestNextDueAt_LargeDowntimeReturnsPromptly(t *testing.T) { + period := time.Nanosecond + dueAt := time.Unix(0, 0).UTC() + now := dueAt.Add(time.Hour) + reminder := store.Reminder{DueAt: dueAt, Interval: period} + + got := nextDueAt(reminder, now) + want := now.Add(period) + if !got.After(now) || !got.Equal(want) { + t.Fatalf("next due time = %s, want %s strictly after now", got, want) + } + + future := now.Add(time.Hour) + reminder.DueAt = future + if got := nextDueAt(reminder, now); !got.Equal(future) { + t.Fatalf("future due time = %s, want unchanged %s", got, future) + } + + reminder.DueAt = now + reminder.Interval = 2 * time.Nanosecond + if got := nextDueAt(reminder, now); !got.Equal(now.Add(reminder.Interval)) { + t.Fatalf("due-now next time = %s, want %s", got, now.Add(reminder.Interval)) + } + + for _, interval := range []time.Duration{0, -time.Nanosecond} { + reminder.Interval = interval + if got := nextDueAt(reminder, now); !got.IsZero() { + t.Fatalf("interval %s next time = %s, want zero", interval, got) + } + } + + const maxElapsed = time.Duration(1<<63 - 1) + overflowDueAt := time.Unix(0, 0).UTC() + overflowNow := overflowDueAt.Add(maxElapsed) + reminder.DueAt = overflowDueAt + reminder.Interval = 2 * time.Nanosecond + if got := nextDueAt(reminder, overflowNow); !got.After(overflowNow) { + t.Fatalf("overflow fallback = %s, want strictly after %s", got, overflowNow) + } +} + +func TestPoller_PassesPeriodicTickStatus(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(250, 0).UTC() + period := time.Hour + first := start.Add(-3 * period) + due := start.Add(-period) + claimed := new(atomic.Bool) + backend := &fakeTable{ + rows: []store.Reminder{{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + FirstTickTime: first, + DueAt: due, + Interval: period, + ETag: 1, + }}, + claimWon: true, + recorder: &stepRecorder{}, + afterClaim: func() { + claimed.Store(true) + }, + } + fakeClock := clock.NewFake(start) + deliveryStart := start.Add(7 * time.Second) + observedClock := &afterClaimClock{Fake: fakeClock, claimed: claimed, after: deliveryStart} + var gotName string + var gotFirst, gotCurrent time.Time + var gotPeriod time.Duration + factory := func(_ store.GrainId, _ string, name string, firstTick time.Time, tickPeriod time.Duration, current time.Time) (any, any) { + gotName = name + gotFirst = firstTick + gotPeriod = tickPeriod + gotCurrent = current + return &struct{}{}, &struct{}{} + } + poller := newTestPoller(backend, observedClock, &recordingInvoker{recorder: backend.recorder}, factory) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + + wantCurrent := deliveryStart + if gotName != "wake" || !gotFirst.Equal(first) || gotPeriod != period || !gotCurrent.Equal(wantCurrent) { + t.Fatalf("TickStatus = name %q first %s period %s current %s, want wake %s %s %s", gotName, gotFirst, gotPeriod, gotCurrent, first, period, wantCurrent) + } + }) +} + +func TestPoller_PassesZeroPeriodForOneShot(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(275, 0).UTC() + backend := &fakeTable{ + rows: []store.Reminder{{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + FirstTickTime: start, + DueAt: start, + ETag: 1, + }}, + claimWon: true, + recorder: &stepRecorder{}, + } + fakeClock := clock.NewFake(start) + var gotPeriod time.Duration + factory := func(_ store.GrainId, _ string, _ string, _ time.Time, period time.Duration, _ time.Time) (any, any) { + gotPeriod = period + return &struct{}{}, &struct{}{} + } + poller := newTestPoller(backend, fakeClock, &recordingInvoker{recorder: backend.recorder}, factory) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + + if gotPeriod != 0 { + t.Fatalf("one-shot Period = %s, want 0", gotPeriod) + } + }) +} + +func TestPoller_ClaimFailureDoesNotInvoke(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(300, 0).UTC() + backend := &fakeTable{ + rows: []store.Reminder{{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + DueAt: start.Add(-time.Second), + }}, + recorder: &stepRecorder{}, + } + fakeClock := clock.NewFake(start) + invoker := &recordingInvoker{recorder: backend.recorder} + poller := newTestPoller(backend, fakeClock, invoker, testReminderCall) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + + if len(invoker.calls) != 0 { + t.Fatalf("invocations = %v, want none", invoker.calls) + } + }) +} + +func TestPoller_SkipsSchedulesNotOwnedByInvoker(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(350, 0).UTC() + backend := store.NewMemory() + schedule := store.Reminder{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + DueAt: start.Add(-time.Second), + } + if err := backend.Put(context.Background(), schedule); err != nil { + t.Fatalf("Put: %v", err) + } + + nonOwner := &ownershipInvoker{} + owner := &ownershipInvoker{owns: true} + nonOwnerClock := clock.NewFake(start) + ownerClock := clock.NewFake(start) + nonOwnerPoller := newTestPoller(backend, nonOwnerClock, nonOwner, testReminderCall) + ownerPoller := newTestPoller(backend, ownerClock, owner, testReminderCall) + synctest.Wait() + + nonOwnerClock.Advance(time.Second) + synctest.Wait() + if got := nonOwner.calls.Load(); got != 0 { + t.Fatalf("non-owner invocations = %d, want 0", got) + } + + ownerClock.Advance(time.Second) + synctest.Wait() + if got := owner.calls.Load(); got != 1 { + t.Fatalf("owner invocations = %d, want 1", got) + } + if got := nonOwner.calls.Load(); got != 0 { + t.Fatalf("non-owner invocations after owner poll = %d, want 0", got) + } + nonOwnerPoller.Close() + ownerPoller.Close() + }) +} + +func TestPoller_ReportsDispatchFailureAfterTerminalCAS(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(360, 0).UTC() + resolveErr := errors.New("unknown Reminder method") + backend := &fakeTable{ + rows: []store.Reminder{{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Missing", + DueAt: start, + ETag: 1, + }}, + claimWon: true, + recorder: &stepRecorder{}, + } + fakeClock := clock.NewFake(start) + invoker := &recordingInvoker{recorder: backend.recorder} + failures := make(chan Failure, 1) + config := testPollerConfig(backend, fakeClock, invoker, testReminderCall) + config.NewCall = func(store.GrainId, string) (ReminderCallBuilder, error) { + return nil, resolveErr + } + config.OnFailure = func(failure Failure) { failures <- failure } + poller := New(config) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + + if got := backend.claimCalls.Load(); got != 1 { + t.Fatalf("terminal Claim calls = %d, want 1", got) + } + if len(invoker.calls) != 0 { + t.Fatalf("Invoke calls = %v, want none", invoker.calls) + } + select { + case failure := <-failures: + if failure.Kind != FailureDispatch || failure.Reminder.Name != "wake" || !errors.Is(failure.Err, resolveErr) { + t.Fatalf("failure = %#v, want dispatch failure for wake", failure) + } + default: + t.Fatal("dispatch failure was not reported") + } + }) +} + +func TestPoller_InvalidReminderIsTerminalAcrossPolls(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(365, 0).UTC() + backend := store.NewMemory() + row := store.Reminder{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Missing", + FirstTickTime: start, + DueAt: start, + Interval: time.Second, + } + if err := backend.Put(context.Background(), row); err != nil { + t.Fatal(err) + } + + resolveErr := errors.New("unknown Reminder method") + fakeClock := clock.NewFake(start) + failures := make(chan Failure, 4) + config := testPollerConfig(backend, fakeClock, &ownershipInvoker{owns: true}, testReminderCall) + config.NewCall = func(store.GrainId, string) (ReminderCallBuilder, error) { + return nil, resolveErr + } + config.OnFailure = func(failure Failure) { failures <- failure } + poller := New(config) + synctest.Wait() + + for range 3 { + fakeClock.Advance(time.Second) + synctest.Wait() + } + poller.Close() + + if got := len(failures); got != 1 { + t.Fatalf("dispatch failures after three polls = %d, want 1", got) + } + failure := <-failures + if failure.Kind != FailureDispatch || failure.Reminder.Name != row.Name || !errors.Is(failure.Err, resolveErr) { + t.Fatalf("failure = %#v, want one dispatch failure for wake", failure) + } + page, err := backend.ListDue(context.Background(), fakeClock.Now(), nil, 1) + if err != nil || len(page.Rows) != 0 { + t.Fatalf("due rows after terminal result = (%#v, %v), want none", page.Rows, err) + } + + restartFailures := make(chan Failure, 1) + config.OnFailure = func(failure Failure) { restartFailures <- failure } + restarted := New(config) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + restarted.Close() + if got := len(restartFailures); got != 0 { + t.Fatalf("dispatch failures after restart = %d, want 0", got) + } + }) +} + +func TestPoller_StaleInvalidReminderDoesNotReport(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(366, 0).UTC() + resolveErr := errors.New("unknown Reminder method") + backend := &fakeTable{ + rows: []store.Reminder{{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Missing", + DueAt: start, + ETag: 1, + }}, + recorder: &stepRecorder{}, + } + fakeClock := clock.NewFake(start) + failures := make(chan Failure, 1) + config := testPollerConfig(backend, fakeClock, &ownershipInvoker{owns: true}, testReminderCall) + config.NewCall = func(store.GrainId, string) (ReminderCallBuilder, error) { + return nil, resolveErr + } + config.OnFailure = func(failure Failure) { failures <- failure } + poller := New(config) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + + if got := backend.claimCalls.Load(); got != 1 { + t.Fatalf("terminal Claim calls = %d, want 1", got) + } + if got := len(failures); got != 0 { + t.Fatalf("failures after stale terminal CAS = %d, want 0", got) + } + }) +} + +func TestPoller_SetOrCancelCanWinAgainstTerminalClaim(t *testing.T) { + tests := []struct { + name string + change func(context.Context, *store.Memory, store.Reminder) error + want int + }{ + { + name: "Set", + change: func(ctx context.Context, backend *store.Memory, reminder store.Reminder) error { + reminder.Method = "Wake" + return backend.Put(ctx, reminder) + }, + want: 1, + }, + { + name: "Cancel", + change: func(ctx context.Context, backend *store.Memory, reminder store.Reminder) error { + return backend.Delete(ctx, reminder.GrainId, reminder.Name) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(366, 0).UTC() + memory := store.NewMemory() + row := store.Reminder{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Missing", + DueAt: start, + } + if err := memory.Put(context.Background(), row); err != nil { + t.Fatal(err) + } + backend := &gatedTerminalTable{ + Memory: memory, + claimStarted: make(chan struct{}), + continueClaim: make(chan struct{}), + } + failures := make(chan Failure, 1) + fakeClock := clock.NewFake(start) + config := testPollerConfig(backend, fakeClock, &ownershipInvoker{owns: true}, testReminderCall) + config.NewCall = func(store.GrainId, string) (ReminderCallBuilder, error) { + return nil, errors.New("unknown Reminder method") + } + config.OnFailure = func(failure Failure) { failures <- failure } + poller := New(config) + defer poller.Close() + synctest.Wait() + + fakeClock.Advance(time.Second) + <-backend.claimStarted + if err := test.change(context.Background(), memory, row); err != nil { + t.Fatalf("%s Reminder: %v", test.name, err) + } + close(backend.continueClaim) + synctest.Wait() + + if got := len(failures); got != 0 { + t.Fatalf("failures after %s won = %d, want 0", test.name, got) + } + page, err := memory.ListDue(context.Background(), start.Add(time.Second), nil, 1) + if err != nil || len(page.Rows) != test.want { + t.Fatalf("rows after %s = (%#v, %v), want %d", test.name, page.Rows, err, test.want) + } + if test.want == 1 && page.Rows[0].Method != "Wake" { + t.Fatalf("Reminder after Set = %#v, want method Wake", page.Rows[0]) + } + }) + }) + } +} + +func TestPoller_TwoPollersReportOneInvalidReminderOnce(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(367, 0).UTC() + backend := store.NewMemory() + if err := backend.Put(context.Background(), store.Reminder{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Missing", + DueAt: start, + }); err != nil { + t.Fatal(err) + } + + resolveErr := errors.New("unknown Reminder method") + failures := make(chan Failure, 2) + newConfig := func(sourceClock clock.Clock) Config { + config := testPollerConfig(backend, sourceClock, &ownershipInvoker{owns: true}, testReminderCall) + config.NewCall = func(store.GrainId, string) (ReminderCallBuilder, error) { + return nil, resolveErr + } + config.OnFailure = func(failure Failure) { failures <- failure } + return config + } + firstClock := clock.NewFake(start) + secondClock := clock.NewFake(start) + first := New(newConfig(firstClock)) + second := New(newConfig(secondClock)) + synctest.Wait() + firstClock.Advance(time.Second) + secondClock.Advance(time.Second) + synctest.Wait() + first.Close() + second.Close() + + if got := len(failures); got != 1 { + t.Fatalf("dispatch failures from two pollers = %d, want 1", got) + } + failure := <-failures + if failure.Kind != FailureDispatch || !errors.Is(failure.Err, resolveErr) { + t.Fatalf("failure = %#v, want one dispatch failure", failure) + } + }) +} + +func TestPoller_InvalidReminderTerminalErrorDoesNotReportDispatch(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(368, 0).UTC() + terminalErr := errors.New("terminal Store write failed") + backend := &fakeTable{ + rows: []store.Reminder{{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Missing", + DueAt: start, + ETag: 1, + }}, + claimErr: terminalErr, + recorder: &stepRecorder{}, + } + fakeClock := clock.NewFake(start) + failures := make(chan Failure, 1) + config := testPollerConfig(backend, fakeClock, &ownershipInvoker{owns: true}, testReminderCall) + config.NewCall = func(store.GrainId, string) (ReminderCallBuilder, error) { + return nil, errors.New("unknown Reminder method") + } + config.OnFailure = func(failure Failure) { failures <- failure } + poller := New(config) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + + if got := len(failures); got != 1 { + t.Fatalf("failures = %d, want 1 terminal failure", got) + } + failure := <-failures + if failure.Kind != FailureTerminal || !errors.Is(failure.Err, terminalErr) { + t.Fatalf("failure = %#v, want terminal Store failure", failure) + } + }) +} + +func TestPoller_AppliedTerminalErrorDoesNotRepeatAfterRestart(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(369, 0).UTC() + terminalErr := errors.New("terminal result was lost") + backend := &appliedClaimErrorTable{Memory: store.NewMemory(), err: terminalErr} + if err := backend.Put(context.Background(), store.Reminder{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Missing", + DueAt: start, + }); err != nil { + t.Fatal(err) + } + + resolveErr := errors.New("unknown Reminder method") + fakeClock := clock.NewFake(start) + failures := make(chan Failure, 2) + config := testPollerConfig(backend, fakeClock, &ownershipInvoker{owns: true}, testReminderCall) + config.NewCall = func(store.GrainId, string) (ReminderCallBuilder, error) { + return nil, resolveErr + } + config.OnFailure = func(failure Failure) { failures <- failure } + first := New(config) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + first.Close() + + if got := len(failures); got != 1 { + t.Fatalf("first-process failures = %d, want 1 terminal Store failure", got) + } + failure := <-failures + if failure.Kind != FailureTerminal || !errors.Is(failure.Err, terminalErr) { + t.Fatalf("failure = %#v, want terminal Unknown Result", failure) + } + + restarted := New(config) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + restarted.Close() + if got := len(failures); got != 0 { + t.Fatalf("failures after restart = %d, want 0", got) + } + }) +} + +func TestPoller_WorkerCapacityBoundsClaims(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(370, 0).UTC() + rows := make([]store.Reminder, 5) + for index := range rows { + rows[index] = store.Reminder{ + GrainId: store.GrainId{GrainType: "account", GrainKey: string(rune('a' + index))}, + Name: "wake", + Method: "Wake", + DueAt: start, + ETag: 1, + } + } + backend := &fakeTable{rows: rows, claimWon: true, recorder: &stepRecorder{}} + fakeClock := clock.NewFake(start) + invoker := &boundedInvoker{started: make(chan struct{}, len(rows))} + config := testPollerConfig(backend, fakeClock, invoker, testReminderCall) + config.PageSize = 2 + config.Workers = 2 + poller := New(config) + defer poller.Close() + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + + if got := backend.claimCalls.Load(); got != 2 { + t.Fatalf("Claim calls while workers are full = %d, want 2", got) + } + if got := len(invoker.started); got != 2 { + t.Fatalf("started Calls = %d, want 2", got) + } + if got := invoker.maximum.Load(); got != 2 { + t.Fatalf("maximum active workers = %d, want 2", got) + } + }) +} + +func TestPoller_WorkerCapacityBoundsTerminalWrites(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(372, 0).UTC() + rows := make([]store.Reminder, 5) + for index := range rows { + rows[index] = store.Reminder{ + GrainId: store.GrainId{GrainType: "account", GrainKey: string(rune('a' + index))}, + Name: "wake", + Method: "Missing", + DueAt: start, + ETag: 1, + } + } + backend := &boundedInvalidTable{rows: rows, started: make(chan struct{}, len(rows))} + fakeClock := clock.NewFake(start) + config := testPollerConfig(backend, fakeClock, &ownershipInvoker{owns: true}, nil) + config.PageSize = len(rows) + config.Workers = 2 + poller := New(config) + defer poller.Close() + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + + if got := backend.calls.Load(); got != 2 { + t.Fatalf("terminal Claim calls while workers are full = %d, want 2", got) + } + if got := len(backend.started); got != 2 { + t.Fatalf("started terminal writes = %d, want 2", got) + } + if got := backend.maximum.Load(); got != 2 { + t.Fatalf("maximum active terminal writes = %d, want 2", got) + } + + poller.Close() + if got := backend.active.Load(); got != 0 { + t.Fatalf("active terminal writes after Close = %d, want 0", got) + } + }) +} + +func TestPoller_LargeBacklogKeepsPagesAndWorkersBounded(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + const ( + seed = int64(1945) + backlogSize = 257 + pageSize = 17 + workerCount = 5 + ) + start := time.Unix(375, 0).UTC() + random := rand.New(rand.NewSource(seed)) + rows := make([]store.Reminder, backlogSize) + for index, key := range random.Perm(backlogSize) { + rows[index] = store.Reminder{ + GrainId: store.GrainId{GrainType: "account", GrainKey: fmt.Sprintf("grain-%03d", key)}, + Name: "wake", + Method: "Wake", + DueAt: start.Add(-time.Duration(random.Intn(5)) * time.Second), + ETag: 1, + } + } + sort.Slice(rows, func(left, right int) bool { + return testReminderBefore(rows[left], rows[right]) + }) + backend := &backlogTable{rows: rows} + fakeClock := clock.NewFake(start) + invoker := &controlledInvoker{ + started: make(chan struct{}, backlogSize), + release: make(chan struct{}), + delivered: make(map[store.GrainId]int, backlogSize), + } + failures := make(chan Failure, backlogSize) + config := testPollerConfig(backend, fakeClock, invoker, testReminderCall) + config.PageSize = pageSize + config.Workers = workerCount + config.OnFailure = func(failure Failure) { failures <- failure } + poller := New(config) + defer poller.Close() + + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + if got := len(invoker.started); got != workerCount { + t.Fatalf("started Calls with full workers = %d, want %d", got, workerCount) + } + for started := workerCount; started < backlogSize; started++ { + invoker.release <- struct{}{} + synctest.Wait() + if got := len(invoker.started); got != started+1 { + t.Fatalf("started Calls after release %d = %d, want %d", started-workerCount+1, got, started+1) + } + } + for invoker.active.Load() > 0 { + invoker.release <- struct{}{} + synctest.Wait() + } + poller.Close() + + if got := invoker.maximum.Load(); got != workerCount { + t.Errorf("maximum active workers = %d, want %d", got, workerCount) + } + if got := invoker.ended.Load(); got != backlogSize { + t.Errorf("completed Calls = %d, want %d", got, backlogSize) + } + if got := backend.claimCalls.Load(); got != backlogSize { + t.Errorf("Claim calls = %d, want %d", got, backlogSize) + } + invoker.mu.Lock() + if got := len(invoker.delivered); got != backlogSize { + t.Errorf("delivered GrainIds = %d, want %d", got, backlogSize) + } + for _, reminder := range rows { + if got := invoker.delivered[reminder.GrainId]; got != 1 { + t.Errorf("deliveries for %s = %d, want 1", reminder.GrainId.GrainKey, got) + } + } + invoker.mu.Unlock() + if got := backend.maximumPage; got != pageSize { + t.Errorf("maximum Store page = %d, want %d", got, pageSize) + } + wantPages := (backlogSize + pageSize - 1) / pageSize + if got := len(backend.requestedLimits); got != wantPages { + t.Errorf("Store pages = %d, want %d", got, wantPages) + } + for index, limit := range backend.requestedLimits { + if limit != pageSize { + t.Errorf("Store page %d limit = %d, want %d", index, limit, pageSize) + } + } + if got := len(poller.slots); got != 0 { + t.Errorf("worker slots after Close = %d, want 0", got) + } + if got := invoker.active.Load(); got != 0 { + t.Errorf("active workers after Close = %d, want 0", got) + } + select { + case <-poller.done: + default: + t.Error("Poller is still running after Close") + } + if got := len(failures); got != 0 { + t.Errorf("Reminder failures = %d, want 0", got) + } + }) +} + +func TestPoller_TwoPollersDeliverOneClaimOnce(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(380, 0).UTC() + backend := store.NewMemory() + if err := backend.Put(context.Background(), store.Reminder{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + DueAt: start, + }); err != nil { + t.Fatal(err) + } + firstClock := clock.NewFake(start) + secondClock := clock.NewFake(start) + first := &ownershipInvoker{owns: true} + second := &ownershipInvoker{owns: true} + firstPoller := newTestPoller(backend, firstClock, first, testReminderCall) + secondPoller := newTestPoller(backend, secondClock, second, testReminderCall) + synctest.Wait() + firstClock.Advance(time.Second) + secondClock.Advance(time.Second) + synctest.Wait() + firstPoller.Close() + secondPoller.Close() + + if got := first.calls.Load() + second.calls.Load(); got != 1 { + t.Fatalf("total Calls = %d, want one Claim winner", got) + } + }) +} + +func TestPoller_AppliedClaimErrorMissesDelivery(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(390, 0).UTC() + claimErr := errors.New("Claim result was lost") + backend := &appliedClaimErrorTable{Memory: store.NewMemory(), err: claimErr} + if err := backend.Put(context.Background(), store.Reminder{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + DueAt: start, + }); err != nil { + t.Fatal(err) + } + fakeClock := clock.NewFake(start) + invoker := &ownershipInvoker{owns: true} + failures := make(chan Failure, 1) + config := testPollerConfig(backend, fakeClock, invoker, testReminderCall) + config.OnFailure = func(failure Failure) { failures <- failure } + poller := New(config) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + + if got := invoker.calls.Load(); got != 0 { + t.Fatalf("Calls after an Unknown Claim result = %d, want 0", got) + } + page, err := backend.ListDue(context.Background(), start.Add(time.Second), nil, 1) + if err != nil || len(page.Rows) != 0 { + t.Fatalf("due rows after applied Claim = (%#v, %v), want none", page.Rows, err) + } + select { + case failure := <-failures: + if failure.Kind != FailureClaim || !errors.Is(failure.Err, claimErr) { + t.Fatalf("failure = %#v, want Claim result error", failure) + } + default: + t.Fatal("applied Claim error was not reported") + } + }) +} + +func TestPoller_ReportsStoreErrorsButNotLostClaimCAS(t *testing.T) { + tests := []struct { + name string + listErr error + claimErr error + claimWon bool + want FailureKind + }{ + {name: "scan", listErr: errors.New("scan failed"), want: FailureScan}, + {name: "claim", claimErr: errors.New("claim failed"), claimWon: true, want: FailureClaim}, + {name: "lost CAS", claimWon: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(395, 0).UTC() + backend := &fakeTable{ + rows: []store.Reminder{{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + DueAt: start, + }}, + claimWon: test.claimWon, + listErr: test.listErr, + claimErr: test.claimErr, + recorder: &stepRecorder{}, + } + fakeClock := clock.NewFake(start) + failures := make(chan Failure, 1) + config := testPollerConfig(backend, fakeClock, &ownershipInvoker{owns: true}, testReminderCall) + config.OnFailure = func(failure Failure) { failures <- failure } + poller := New(config) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + + select { + case failure := <-failures: + if test.want == 0 || failure.Kind != test.want { + t.Fatalf("failure = %#v, want kind %d", failure, test.want) + } + default: + if test.want != 0 { + t.Fatalf("failure kind %d was not reported", test.want) + } + } + }) + }) + } +} + +func TestPoller_ReportsAndStopsOnCursorThatDoesNotAdvance(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(398, 0).UTC() + backend := &stuckCursorTable{row: store.Reminder{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + DueAt: start, + ETag: 1, + }} + fakeClock := clock.NewFake(start) + failures := make(chan Failure, 1) + config := testPollerConfig(backend, fakeClock, &ownershipInvoker{owns: true}, testReminderCall) + config.PageSize = 1 + config.OnFailure = func(failure Failure) { failures <- failure } + poller := New(config) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + + if got := backend.listCalls.Load(); got != 2 { + t.Fatalf("ListDue calls = %d, want two pages before cursor rejection", got) + } + if got := backend.claimCalls.Load(); got != 1 { + t.Fatalf("Claim calls = %d, want only the first page row", got) + } + select { + case failure := <-failures: + if failure.Kind != FailureScan { + t.Fatalf("failure = %#v, want scan failure", failure) + } + default: + t.Fatal("non-advancing cursor was not reported") + } + }) +} + +func TestPoller_RejectsRowsAbovePageLimitBeforeClaim(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(399, 0).UTC() + backend := &oversizedPageTable{rows: []store.Reminder{ + {GrainId: store.GrainId{GrainType: "account", GrainKey: "a"}, Name: "wake", Method: "Wake", DueAt: start}, + {GrainId: store.GrainId{GrainType: "account", GrainKey: "b"}, Name: "wake", Method: "Wake", DueAt: start}, + }} + fakeClock := clock.NewFake(start) + failures := make(chan Failure, 1) + config := testPollerConfig(backend, fakeClock, &ownershipInvoker{owns: true}, testReminderCall) + config.PageSize = 1 + config.OnFailure = func(failure Failure) { failures <- failure } + poller := New(config) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + + if got := backend.claimCalls.Load(); got != 0 { + t.Fatalf("Claim calls = %d, want none for an oversized page", got) + } + select { + case failure := <-failures: + if failure.Kind != FailureScan { + t.Fatalf("failure = %#v, want scan failure", failure) + } + default: + t.Fatal("oversized page was not reported") + } + }) +} + +func TestPoller_CloseStopsTheGoroutine(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(400, 0).UTC() + backend := &fakeTable{ + rows: []store.Reminder{{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + DueAt: start.Add(-time.Second), + }}, + claimWon: true, + recorder: &stepRecorder{}, + } + fakeClock := clock.NewFake(start) + invoker := &blockingInvoker{started: make(chan struct{}), finished: make(chan struct{})} + poller := newTestPoller(backend, fakeClock, invoker, testReminderCall) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + poller.Close() + poller.Close() + select { + case <-invoker.finished: + default: + t.Fatal("invocation is still running after Close") + } + }) +} + +var _ Table = (*fakeTable)(nil) +var _ Invoker = (*recordingInvoker)(nil) +var _ Invoker = (*blockingInvoker)(nil) diff --git a/lifecycle_test.go b/lifecycle_test.go index 1802969..8f64214 100644 --- a/lifecycle_test.go +++ b/lifecycle_test.go @@ -15,9 +15,10 @@ import ( func TestRootLifecycle_RejectsNewCallsAfterClose(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) registerAccount(t, rt) - rt.Close() + mustStart(t, rt) + closeRuntime(rt) - id := GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"} err := rt.Invoke(context.Background(), id, "Balance", &accountBalanceRequest{}, &accountBalanceReply{}) if !errors.Is(err, ErrRuntimeClosed) { t.Fatalf("invoke after close = %v, want ErrRuntimeClosed", err) @@ -44,13 +45,14 @@ func TestRootLifecycle_AdmittedCallFinishesDuringClose(t *testing.T) { } return dispatchAccount(ctx, instance, method, args, reply) }) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) - id := GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"} admittedDone := make(chan error, 1) go func() { admittedDone <- rt.Invoke(context.Background(), id, "Block", nil, nil) @@ -60,7 +62,7 @@ func TestRootLifecycle_AdmittedCallFinishesDuringClose(t *testing.T) { closeDone := make(chan struct{}) go func() { - rt.Close() + closeRuntime(rt) close(closeDone) }() synctest.Wait() @@ -106,13 +108,14 @@ func TestRootLifecycle_QueuedCallRejectedWithoutRunning(t *testing.T) { return dispatchAccount(ctx, instance, method, args, reply) } }) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) - id := GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"} blockDone := make(chan error, 1) go func() { blockDone <- rt.Invoke(context.Background(), id, "Block", nil, nil) @@ -129,7 +132,7 @@ func TestRootLifecycle_QueuedCallRejectedWithoutRunning(t *testing.T) { closeDone := make(chan struct{}) go func() { - rt.Close() + closeRuntime(rt) close(closeDone) }() synctest.Wait() @@ -157,10 +160,11 @@ func TestRootLifecycle_StopIsIdempotent(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) registerAccount(t, rt) + mustStart(t, rt) - rt.Close() - rt.Close() - rt.Kill() + closeRuntime(rt) + closeRuntime(rt) + killRuntime(rt) select { case <-rt.Done(): @@ -172,10 +176,11 @@ func TestRootLifecycle_StopIsIdempotent(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) registerAccount(t, rt) + mustStart(t, rt) - rt.Kill() - rt.Kill() - rt.Close() + killRuntime(rt) + killRuntime(rt) + closeRuntime(rt) select { case <-rt.Done(): @@ -192,6 +197,7 @@ func TestRootLifecycle_StopIsIdempotent(t *testing.T) { // branch on this return value without a check-then-act window. func TestRootLifecycle_BecomeDeadOnlyTransitionsRunning(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) + mustStart(t, rt) if !rt.becomeDead() { t.Fatal("becomeDead from running = false, want true") } @@ -204,12 +210,15 @@ func TestRootLifecycle_BecomeDeadOnlyTransitionsRunning(t *testing.T) { if rt.beginKill() { t.Fatal("beginKill after dead = true, want false") } + rt.closeImmediately() closed := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) + mustStart(t, closed) closed.beginClose() if closed.becomeDead() { t.Fatal("becomeDead after beginClose = true, want false (root stays closing)") } + closed.closeGracefully() } // TestRootLifecycle_AdmitGatesBeforeEngineClose proves the root admission gate, @@ -220,7 +229,8 @@ func TestRootLifecycle_AdmitGatesBeforeEngineClose(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) registerAccount(t, rt) - id := GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + mustStart(t, rt) + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"} if err := rt.Invoke(context.Background(), id, "Deposit", &accountDepositRequest{A0: 5}, &accountDepositReply{}); err != nil { t.Fatal(err) } @@ -233,7 +243,7 @@ func TestRootLifecycle_AdmitGatesBeforeEngineClose(t *testing.T) { if !errors.Is(err, ErrRuntimeClosed) { t.Fatalf("invoke after beginClose with engine still open = %v, want ErrRuntimeClosed", err) } - rt.Close() + rt.closeGracefully() }) } @@ -245,13 +255,14 @@ func TestRootLifecycle_HandleInvokeGatedBeforeEngineClose(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) registerAccount(t, rt) + mustStart(t, rt) // Enter closing without closing the engine. rt.beginClose() payload, err := rt.handleInvoke(context.Background(), callRequest{ Kind: requestKindInvoke, - GrainType: TypeName[Account](), + GrainType: "gor.Account", GrainKey: "alice", Method: "Balance", Args: json.RawMessage(`{}`), @@ -266,7 +277,7 @@ func TestRootLifecycle_HandleInvokeGatedBeforeEngineClose(t *testing.T) { if response.Error == nil || response.Error.Code != string(ErrRuntimeClosed) { t.Fatalf("inbound invoke after beginClose = %#v, want runtime-closed code", response.Error) } - rt.Close() + rt.closeGracefully() }) } @@ -278,11 +289,12 @@ func TestRootLifecycle_KillAfterStoppedIsIdempotent(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) registerAccount(t, rt) + mustStart(t, rt) - rt.Close() - rt.Kill() + closeRuntime(rt) + killRuntime(rt) - id := GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"} if err := rt.Invoke(context.Background(), id, "Balance", &accountBalanceRequest{}, &accountBalanceReply{}); !errors.Is(err, ErrRuntimeClosed) { t.Fatalf("invoke after close-then-kill = %v, want ErrRuntimeClosed", err) } diff --git a/mail/mail.go b/mail/mail.go deleted file mode 100644 index 63bd2df..0000000 --- a/mail/mail.go +++ /dev/null @@ -1,128 +0,0 @@ -// Package mail provides the per-entity mailboxes gor uses to serialize calls. -// -// It is an implementation package, not an application dependency. Invoke -// entities through the root gor package instead of importing mail directly. -package mail - -import ( - "context" - "errors" - "sync" -) - -var ( - ErrOverloaded = errors.New("mailbox overloaded") - ErrClosed = errors.New("mailbox closed") -) - -type Call func(context.Context) (any, error) - -type Result struct { - Value any - Err error -} - -type Box struct { - in chan *call - done chan struct{} - - mu sync.Mutex - closed bool -} - -type call struct { - fn Call - reply chan Result - ctx context.Context -} - -func New(capacity int) *Box { - b := &Box{ - in: make(chan *call, capacity), - done: make(chan struct{}), - } - go b.run() - return b -} - -func (b *Box) Call(ctx context.Context, fn Call) (any, error) { - c := &call{fn: fn, reply: make(chan Result, 1), ctx: ctx} - - b.mu.Lock() - if b.closed { - b.mu.Unlock() - return nil, ErrClosed - } - select { - case b.in <- c: - b.mu.Unlock() - default: - b.mu.Unlock() - return nil, ErrOverloaded - } - - select { - case result := <-c.reply: - return result.Value, result.Err - case <-ctx.Done(): - return nil, ctx.Err() - } -} - -func (b *Box) Close() { - b.mu.Lock() - defer b.mu.Unlock() - if b.closed { - return - } - b.closed = true - close(b.in) -} - -func (b *Box) Done() <-chan struct{} { - return b.done -} - -func (b *Box) Len() int { - return len(b.in) -} - -func (b *Box) run() { - defer close(b.done) - - for c := range b.in { - if b.isClosed() { - c.reply <- Result{Err: ErrClosed} - b.rejectQueued() - return - } - value, err := c.fn(c.ctx) - c.reply <- Result{Value: value, Err: err} - if b.isClosed() { - b.rejectQueued() - return - } - } - - b.rejectQueued() -} - -func (b *Box) isClosed() bool { - b.mu.Lock() - defer b.mu.Unlock() - return b.closed -} - -func (b *Box) rejectQueued() { - for { - select { - case c, ok := <-b.in: - if !ok { - return - } - c.reply <- Result{Err: ErrClosed} - default: - return - } - } -} diff --git a/mail/mail_test.go b/mail/mail_test.go deleted file mode 100644 index 698331c..0000000 --- a/mail/mail_test.go +++ /dev/null @@ -1,218 +0,0 @@ -package mail - -import ( - "context" - "errors" - "testing" - "testing/synctest" -) - -type callResult struct { - value any - err error -} - -func TestMailbox_SerializesCalls(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - box := New(2) - defer box.Close() - - started := make(chan struct{}) - release := make(chan struct{}) - secondStarted := make(chan struct{}) - firstDone := make(chan callResult, 1) - secondDone := make(chan callResult, 1) - - go func() { - value, err := box.Call(context.Background(), func(context.Context) (any, error) { - close(started) - <-release - return "first", nil - }) - firstDone <- callResult{value: value, err: err} - }() - synctest.Wait() - <-started - - go func() { - value, err := box.Call(context.Background(), func(context.Context) (any, error) { - close(secondStarted) - return "second", nil - }) - secondDone <- callResult{value: value, err: err} - }() - synctest.Wait() - select { - case <-secondStarted: - t.Fatal("second call ran before first call completed") - default: - } - - close(release) - synctest.Wait() - if result := <-firstDone; result.value != "first" || result.err != nil { - t.Fatalf("first result = %#v, want first result", result) - } - if result := <-secondDone; result.value != "second" || result.err != nil { - t.Fatalf("second result = %#v, want second result", result) - } - }) -} - -func TestMailbox_RejectsCallsWhenQueueIsFull(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - box := New(1) - defer box.Close() - - started := make(chan struct{}) - release := make(chan struct{}) - firstDone := make(chan error, 1) - secondDone := make(chan error, 1) - - go func() { - _, err := box.Call(context.Background(), func(context.Context) (any, error) { - close(started) - <-release - return nil, nil - }) - firstDone <- err - }() - synctest.Wait() - <-started - - go func() { - _, err := box.Call(context.Background(), func(context.Context) (any, error) { return nil, nil }) - secondDone <- err - }() - synctest.Wait() - - if _, err := box.Call(context.Background(), func(context.Context) (any, error) { return nil, nil }); !errors.Is(err, ErrOverloaded) { - t.Fatalf("third call error = %v, want ErrOverloaded", err) - } - - close(release) - synctest.Wait() - if err := <-firstDone; err != nil { - t.Fatalf("first call error = %v", err) - } - if err := <-secondDone; err != nil { - t.Fatalf("second call error = %v", err) - } - }) -} - -func TestMailbox_ContinuesAfterCallerTimeout(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - box := New(1) - defer box.Close() - - started := make(chan struct{}) - release := make(chan struct{}) - ctx, cancel := context.WithCancel(context.Background()) - callDone := make(chan error, 1) - go func() { - _, err := box.Call(ctx, func(context.Context) (any, error) { - close(started) - <-release - return "timed out", nil - }) - callDone <- err - }() - - synctest.Wait() - <-started - cancel() - synctest.Wait() - if err := <-callDone; !errors.Is(err, context.Canceled) { - t.Fatalf("timed out call error = %v, want context.Canceled", err) - } - - close(release) - synctest.Wait() - value, err := box.Call(context.Background(), func(context.Context) (any, error) { - return "after timeout", nil - }) - if err != nil || value != "after timeout" { - t.Fatalf("follow-up call = %#v, %v", value, err) - } - }) -} - -func TestMailbox_RejectsQueuedCallsOnClose(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - box := New(1) - - started := make(chan struct{}) - release := make(chan struct{}) - firstDone := make(chan callResult, 1) - queuedDone := make(chan callResult, 1) - go func() { - value, err := box.Call(context.Background(), func(context.Context) (any, error) { - close(started) - <-release - return "first", nil - }) - firstDone <- callResult{value: value, err: err} - }() - synctest.Wait() - <-started - - go func() { - value, err := box.Call(context.Background(), func(context.Context) (any, error) { - return "queued", nil - }) - queuedDone <- callResult{value: value, err: err} - }() - synctest.Wait() - box.Close() - close(release) - synctest.Wait() - - if result := <-firstDone; result.value != "first" || result.err != nil { - t.Fatalf("first result = %#v, want first result", result) - } - if result := <-queuedDone; !errors.Is(result.err, ErrClosed) { - t.Fatalf("queued result error = %v, want ErrClosed", result.err) - } - }) -} - -func TestMailbox_LenReportsQueuedCalls(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - box := New(1) - defer box.Close() - - started := make(chan struct{}) - release := make(chan struct{}) - firstDone := make(chan error, 1) - queuedDone := make(chan error, 1) - go func() { - _, err := box.Call(context.Background(), func(context.Context) (any, error) { - close(started) - <-release - return nil, nil - }) - firstDone <- err - }() - synctest.Wait() - <-started - - go func() { - _, err := box.Call(context.Background(), func(context.Context) (any, error) { return nil, nil }) - queuedDone <- err - }() - synctest.Wait() - if got := box.Len(); got != 1 { - t.Fatalf("Len = %d, want 1", got) - } - - close(release) - synctest.Wait() - if err := <-firstDone; err != nil { - t.Fatalf("first call error = %v", err) - } - if err := <-queuedDone; err != nil { - t.Fatalf("queued call error = %v", err) - } - }) -} diff --git a/method_handle_test.go b/method_handle_test.go index 97b7a94..02c0ac5 100644 --- a/method_handle_test.go +++ b/method_handle_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -// handleProbe is the entity interface the extraction tripwire locks its map +// handleProbe is the grain interface the extraction tripwire locks its map // on. Both methods must have the Reminder shape func(handleProbe, // context.Context, TickStatus) error, which is what Handle admits. type handleProbe interface { diff --git a/observability_test.go b/observability_test.go index 8afda21..ad5e9c0 100644 --- a/observability_test.go +++ b/observability_test.go @@ -18,7 +18,7 @@ type observedAccount interface { Block(context.Context) error } -type observedAccountEntity struct { +type observedAccountGrain struct { clock *clock.Fake activationErr error methodErr error @@ -28,29 +28,29 @@ type observedAccountEntity struct { deactivated chan<- struct{} } -func (e *observedAccountEntity) OnActivate(context.Context) error { +func (e *observedAccountGrain) OnActivate(context.Context) error { return e.activationErr } -func (e *observedAccountEntity) OnDeactivate(_ context.Context, _ DeactivationReason) error { +func (e *observedAccountGrain) OnDeactivate(_ context.Context, _ DeactivationReason) error { if e.deactivated != nil { e.deactivated <- struct{}{} } return nil } -func (e *observedAccountEntity) Success(context.Context) error { +func (e *observedAccountGrain) Success(context.Context) error { if e.clock != nil { e.clock.Advance(3 * time.Second) } return nil } -func (e *observedAccountEntity) Failure(context.Context) error { +func (e *observedAccountGrain) Failure(context.Context) error { return e.methodErr } -func (e *observedAccountEntity) Block(context.Context) error { +func (e *observedAccountGrain) Block(context.Context) error { if e.started != nil { e.started <- struct{}{} } @@ -74,7 +74,7 @@ func dispatchObservedAccount(ctx context.Context, instance observedAccount, meth } } -func newObservedRuntime(t *testing.T, sourceClock clock.Clock, events chan<- CallObservation, factory func(*Binder) observedAccount, options ...Option) *Runtime { +func newObservedRuntime(t *testing.T, sourceClock clock.Clock, events chan<- CallObservation, factory func(*GrainContext) observedAccount, options ...Option) *Runtime { t.Helper() base := []Option{ WithClock(sourceClock), @@ -88,23 +88,24 @@ func newObservedRuntime(t *testing.T, sourceClock clock.Clock, events chan<- Cal })) } rt := mustNew(t, append(base, options...)...) - if err := InstallType[observedAccount](rt, dispatchObservedAccount, func(Invoker, GrainId) observedAccount { + if err := InstallType[observedAccount](rt, GeneratedCodeVersion, "gor.observedAccount", dispatchObservedAccount, func(Invoker, GrainId) observedAccount { return nil }, func(string) (any, any) { return nil, nil - }, nil); err != nil { + }, noReminderCall); err != nil { t.Fatal(err) } if err := Register[observedAccount](rt, factory); err != nil { t.Fatal(err) } + mustStart(t, rt) return rt } func assertObservation(t *testing.T, got CallObservation, method string, wantErr error) { t.Helper() - if got.GrainType != TypeName[observedAccount]() { - t.Fatalf("GrainType = %q, want %q", got.GrainType, TypeName[observedAccount]()) + if got.GrainType != GrainType("gor.observedAccount") { + t.Fatalf("GrainType = %q, want %q", got.GrainType, GrainType("gor.observedAccount")) } if got.Method != method { t.Fatalf("Method = %q, want %q", got.Method, method) @@ -119,12 +120,12 @@ func TestOnCallReportsSuccessfulInvocationWithInjectedDuration(t *testing.T) { start := time.Unix(0, 0).UTC() fakeClock := clock.NewFake(start) events := make(chan CallObservation, 1) - rt := newObservedRuntime(t, fakeClock, events, func(*Binder) observedAccount { - return &observedAccountEntity{clock: fakeClock} + rt := newObservedRuntime(t, fakeClock, events, func(*GrainContext) observedAccount { + return &observedAccountGrain{clock: fakeClock} }) - defer rt.Close() + defer closeRuntime(rt) - if err := rt.Invoke(context.Background(), GrainId{GrainType: TypeName[observedAccount](), GrainKey: "alice"}, "Success", nil, nil); err != nil { + if err := rt.Invoke(context.Background(), GrainId{GrainType: GrainType("gor.observedAccount"), GrainKey: "alice"}, "Success", nil, nil); err != nil { t.Fatalf("Invoke error = %v", err) } got := <-events @@ -140,11 +141,11 @@ func TestOnCallDoesNotReportDeactivation(t *testing.T) { events := make(chan CallObservation, 1) deactivated := make(chan struct{}, 1) fakeClock := clock.NewFake(time.Unix(0, 0).UTC()) - rt := newObservedRuntime(t, fakeClock, events, func(*Binder) observedAccount { - return &observedAccountEntity{deactivated: deactivated} + rt := newObservedRuntime(t, fakeClock, events, func(*GrainContext) observedAccount { + return &observedAccountGrain{deactivated: deactivated} }, WithIdleTimeout(time.Second), WithEvictionInterval(time.Second)) - defer rt.Close() - id := GrainId{GrainType: TypeName[observedAccount](), GrainKey: "alice"} + defer closeRuntime(rt) + id := GrainId{GrainType: GrainType("gor.observedAccount"), GrainKey: "alice"} if err := rt.Invoke(context.Background(), id, "Success", nil, nil); err != nil { t.Fatalf("Invoke error = %v", err) @@ -180,11 +181,12 @@ func TestOnCallReportsReminderInvocation(t *testing.T) { events <- observation }), ) - defer rt.Close() + defer closeRuntime(rt) installScheduledAccount(t, rt, new(atomic.Int32)) - id := GrainId{GrainType: TypeName[scheduledAccount](), GrainKey: "alice"} + mustStart(t, rt) + id := GrainId{GrainType: GrainType("gor.scheduledAccount"), GrainKey: "alice"} - if err := Ref[scheduledAccount](rt, id.GrainKey).Arm(context.Background()); err != nil { + if err := Ref[scheduledAccount](rt, id.GrainKey).Arm(context.Background(), "wake"); err != nil { t.Fatalf("Arm error = %v", err) } arm := <-events @@ -210,12 +212,12 @@ func TestOnCallReportsMethodError(t *testing.T) { synctest.Test(t, func(t *testing.T) { methodErr := errors.New("method failed") events := make(chan CallObservation, 1) - rt := newObservedRuntime(t, clock.NewFake(time.Unix(0, 0).UTC()), events, func(*Binder) observedAccount { - return &observedAccountEntity{methodErr: methodErr} + rt := newObservedRuntime(t, clock.NewFake(time.Unix(0, 0).UTC()), events, func(*GrainContext) observedAccount { + return &observedAccountGrain{methodErr: methodErr} }) - defer rt.Close() + defer closeRuntime(rt) - err := rt.Invoke(context.Background(), GrainId{GrainType: TypeName[observedAccount](), GrainKey: "alice"}, "Failure", nil, nil) + err := rt.Invoke(context.Background(), GrainId{GrainType: GrainType("gor.observedAccount"), GrainKey: "alice"}, "Failure", nil, nil) if !errors.Is(err, methodErr) { t.Fatalf("Invoke error = %v, want %v", err, methodErr) } @@ -224,16 +226,39 @@ func TestOnCallReportsMethodError(t *testing.T) { }) } +func TestOnCallPanicDoesNotChangeCallResult(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var observations atomic.Int32 + rt := newObservedRuntime(t, clock.NewFake(time.Unix(0, 0).UTC()), nil, func(*GrainContext) observedAccount { + return &observedAccountGrain{} + }, OnCall(func(CallObservation) { + observations.Add(1) + panic("observer failure") + })) + defer closeRuntime(rt) + + id := GrainId{GrainType: GrainType("gor.observedAccount"), GrainKey: "alice"} + for range 2 { + if err := rt.Invoke(context.Background(), id, "Success", nil, nil); err != nil { + t.Fatalf("Invoke after OnCall panic = %v", err) + } + } + if got := observations.Load(); got != 2 { + t.Fatalf("OnCall calls = %d, want 2", got) + } + }) +} + func TestOnCallReportsActivationFailure(t *testing.T) { synctest.Test(t, func(t *testing.T) { activationErr := errors.New("activation failed") events := make(chan CallObservation, 1) - rt := newObservedRuntime(t, clock.NewFake(time.Unix(0, 0).UTC()), events, func(*Binder) observedAccount { - return &observedAccountEntity{activationErr: activationErr} + rt := newObservedRuntime(t, clock.NewFake(time.Unix(0, 0).UTC()), events, func(*GrainContext) observedAccount { + return &observedAccountGrain{activationErr: activationErr} }) - defer rt.Close() + defer closeRuntime(rt) - err := rt.Invoke(context.Background(), GrainId{GrainType: TypeName[observedAccount](), GrainKey: "alice"}, "Success", nil, nil) + err := rt.Invoke(context.Background(), GrainId{GrainType: GrainType("gor.observedAccount"), GrainKey: "alice"}, "Success", nil, nil) if !errors.Is(err, activationErr) { t.Fatalf("Invoke error = %v, want %v", err, activationErr) } @@ -247,11 +272,11 @@ func TestOnCallReportsOverload(t *testing.T) { started := make(chan struct{}, 2) release := make(chan struct{}) events := make(chan CallObservation, 3) - rt := newObservedRuntime(t, clock.NewFake(time.Unix(0, 0).UTC()), events, func(*Binder) observedAccount { - return &observedAccountEntity{started: started, release: release} + rt := newObservedRuntime(t, clock.NewFake(time.Unix(0, 0).UTC()), events, func(*GrainContext) observedAccount { + return &observedAccountGrain{started: started, release: release} }, WithMailboxCapacity(1)) - defer rt.Close() - id := GrainId{GrainType: TypeName[observedAccount](), GrainKey: "alice"} + defer closeRuntime(rt) + id := GrainId{GrainType: GrainType("gor.observedAccount"), GrainKey: "alice"} firstDone := make(chan error, 1) go func() { firstDone <- rt.Invoke(context.Background(), id, "Block", nil, nil) }() @@ -291,11 +316,11 @@ func TestOnCallReportsCancellationOnceAfterMethodFinishes(t *testing.T) { release := make(chan struct{}) finished := make(chan struct{}, 1) events := make(chan CallObservation, 2) - rt := newObservedRuntime(t, clock.NewFake(time.Unix(0, 0).UTC()), events, func(*Binder) observedAccount { - return &observedAccountEntity{started: started, release: release, finished: finished} + rt := newObservedRuntime(t, clock.NewFake(time.Unix(0, 0).UTC()), events, func(*GrainContext) observedAccount { + return &observedAccountGrain{started: started, release: release, finished: finished} }) - defer rt.Close() - id := GrainId{GrainType: TypeName[observedAccount](), GrainKey: "alice"} + defer closeRuntime(rt) + id := GrainId{GrainType: GrainType("gor.observedAccount"), GrainKey: "alice"} ctx, cancel := context.WithCancel(context.Background()) callDone := make(chan error, 1) go func() { callDone <- rt.Invoke(ctx, id, "Block", nil, nil) }() @@ -339,12 +364,12 @@ func (c *countingClock) Now() time.Time { func TestOnCallDisabledDoesNotReadAnExtraClock(t *testing.T) { synctest.Test(t, func(t *testing.T) { counting := &countingClock{Clock: clock.NewFake(time.Unix(0, 0).UTC())} - rt := newObservedRuntime(t, counting, nil, func(*Binder) observedAccount { - return &observedAccountEntity{} + rt := newObservedRuntime(t, counting, nil, func(*GrainContext) observedAccount { + return &observedAccountGrain{} }) - defer rt.Close() + defer closeRuntime(rt) - if err := rt.Invoke(context.Background(), GrainId{GrainType: TypeName[observedAccount](), GrainKey: "alice"}, "Success", nil, nil); err != nil { + if err := rt.Invoke(context.Background(), GrainId{GrainType: GrainType("gor.observedAccount"), GrainKey: "alice"}, "Success", nil, nil); err != nil { t.Fatalf("Invoke error = %v", err) } if got := counting.nowCalls.Load(); got != 2 { diff --git a/probe_test.go b/probe_test.go index b2de55d..8bff9f8 100644 --- a/probe_test.go +++ b/probe_test.go @@ -130,7 +130,8 @@ func TestRuntime_HandleProbeReturnsCurrentMemberIDWithoutTableAccess(t *testing. table := &probeMemberStore{backend: store.NewMemory()} network := newTestTransportNetwork() rt := mustNew(t, clusterRuntimeOptions(store.NewMemory(), table, clock.NewFake(start), "node-a", "generation-new", network.add("node-a"))...) - defer rt.Close() + defer closeRuntime(rt) + mustStart(t, rt) operations := table.operations payload, err := rt.handle(context.Background(), []byte(`{"kind":"probe"}`)) @@ -174,7 +175,7 @@ func (s *probeMemberStore) ListMembers(ctx context.Context) (store.MemberSnapsho func TestRuntime_HandleProbeRejectsUnknownKind(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) payload, err := rt.handle(context.Background(), []byte(`{"kind":"unknown"}`)) if err != nil { @@ -195,7 +196,8 @@ func TestRuntime_HandleProbeRejectsStoppedNode(t *testing.T) { members := store.NewMemory() network := newTestTransportNetwork() rt := mustNew(t, clusterRuntimeOptions(store.NewMemory(), members, clock.NewFake(start), "node-a", "generation-a", network.add("node-a"))...) - rt.Close() + mustStart(t, rt) + closeRuntime(rt) payload, err := rt.handleProbe() if err != nil { @@ -221,7 +223,7 @@ func TestRuntime_HandleProbeReadsRootStateDuringClosing(t *testing.T) { members := store.NewMemory() network := newTestTransportNetwork() rt := mustNew(t, clusterRuntimeOptions(store.NewMemory(), members, clock.NewFake(start), "node-a", "generation-a", network.add("node-a"))...) - defer rt.Close() + mustStart(t, rt) // While running the probe replies with the current member id. runningPayload, err := rt.handleProbe() @@ -250,6 +252,7 @@ func TestRuntime_HandleProbeReadsRootStateDuringClosing(t *testing.T) { if response.Error == nil || response.Error.Code != string(ErrRuntimeClosed) { t.Fatalf("closing probe response = %#v, want runtime-closed code", response.Error) } + rt.closeGracefully() }) } @@ -263,7 +266,8 @@ func TestRuntime_HandleProbeRejectsAfterClusterDeath(t *testing.T) { members := store.NewMemory() network := newTestTransportNetwork() rt := mustNew(t, clusterRuntimeOptions(store.NewMemory(), members, fakeClock, "node-a", "generation-a", network.add("node-a"))...) - defer rt.Close() + defer closeRuntime(rt) + mustStart(t, rt) self := findClusterMember(t, members, "node-a", "generation-a") self.Status = store.MemberDead diff --git a/public_boundary_test.go b/public_boundary_test.go new file mode 100644 index 0000000..7447031 --- /dev/null +++ b/public_boundary_test.go @@ -0,0 +1,55 @@ +package gor + +import ( + "reflect" + "strings" + "testing" + + runtimepkg "github.com/suraciii/gor/internal/runtime" +) + +func TestPublicBoundary_OwnsPublicContractTypes(t *testing.T) { + const packagePath = "github.com/suraciii/gor" + if got := reflect.TypeFor[DeactivationReason]().PkgPath(); got != packagePath { + t.Fatalf("DeactivationReason package = %q, want %q", got, packagePath) + } + + configType := reflect.TypeFor[Config]() + for i := 0; i < configType.NumField(); i++ { + field := configType.Field(i) + if field.Anonymous { + t.Errorf("Config embeds %s; public configuration fields must be owned by gor", field.Type) + } + if path := namedPackagePath(field.Type); strings.Contains(path, "/internal/") { + t.Errorf("Config.%s exposes internal type %s", field.Name, field.Type) + } + } +} + +func TestPublicBoundary_DeactivationReasonsMatchExecutionRuntime(t *testing.T) { + tests := []struct { + name string + internal runtimepkg.DeactivationReason + public DeactivationReason + }{ + {name: "idle", internal: runtimepkg.Idle, public: Idle}, + {name: "application requested", internal: runtimepkg.ApplicationRequested, public: ApplicationRequested}, + {name: "ownership lost", internal: runtimepkg.OwnershipLost, public: OwnershipLost}, + {name: "runtime closed", internal: runtimepkg.RuntimeClosed, public: RuntimeClosed}, + {name: "faulted", internal: runtimepkg.Faulted, public: Faulted}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := DeactivationReason(test.internal); got != test.public { + t.Fatalf("public reason = %d, want %d", got, test.public) + } + }) + } +} + +func namedPackagePath(value reflect.Type) string { + for value.Kind() == reflect.Pointer || value.Kind() == reflect.Slice || value.Kind() == reflect.Array || value.Kind() == reflect.Map || value.Kind() == reflect.Chan { + value = value.Elem() + } + return value.PkgPath() +} diff --git a/request_context.go b/request_context.go index 9028751..e803c0f 100644 --- a/request_context.go +++ b/request_context.go @@ -223,6 +223,9 @@ func decodeRequestContext(raw json.RawMessage) (requestContextSnapshot, error) { if len(trimmed) == 0 || trimmed[0] != '{' { return requestContextSnapshot{}, fmt.Errorf("request_context must be a JSON object") } + if len(trimmed) > maxRequestContextBytes { + return requestContextSnapshot{}, fmt.Errorf("request_context is %d bytes, maximum is %d", len(trimmed), maxRequestContextBytes) + } var wire map[string]json.RawMessage if err := json.Unmarshal(trimmed, &wire); err != nil { return requestContextSnapshot{}, fmt.Errorf("decode request_context: %w", err) diff --git a/request_context_test.go b/request_context_test.go index 7c381cc..e76116d 100644 --- a/request_context_test.go +++ b/request_context_test.go @@ -1,11 +1,13 @@ package gor import ( + "bytes" "context" "encoding/json" "errors" "fmt" "math" + "reflect" "strings" "sync/atomic" "testing" @@ -243,14 +245,69 @@ func TestRequestContext_WireRoundTripKeepsExactIntegers(t *testing.T) { } } +func TestDecodeRequestContextRejectsWireSizeBeforeJSONDecode(t *testing.T) { + raw := append([]byte(`{"value":"`), bytes.Repeat([]byte("x"), maxRequestContextBytes)...) + _, err := decodeRequestContext(raw) + if err == nil || !strings.Contains(err.Error(), "maximum") { + t.Fatalf("oversized incomplete Request Context error = %v, want size error", err) + } +} + +func FuzzDecodeRequestContext(f *testing.F) { + for _, raw := range [][]byte{ + {}, + []byte(`{}`), + []byte(`{"trace_id":{"type":"string","value":"call-1"}}`), + []byte(`{"enabled":{"type":"bool","value":true}}`), + []byte(`{"signed":{"type":"int64","value":-9223372036854775808}}`), + []byte(`{"unsigned":{"type":"uint64","value":18446744073709551615}}`), + []byte(`{"ratio":{"type":"float64","value":1.5}}`), + []byte(`{"nil":{"type":"null"}}`), + []byte(`null`), + []byte(`[]`), + []byte(`{"bad":{"type":"unknown","value":true}}`), + append([]byte(`{"large":{"type":"string","value":"`), bytes.Repeat([]byte("x"), maxRequestContextBytes)...), + {0xff}, + } { + f.Add(raw) + } + + f.Fuzz(func(t *testing.T, raw []byte) { + snapshot, err := decodeRequestContext(json.RawMessage(raw)) + if err != nil { + return + } + encoded, err := marshalRequestContext(snapshot) + if err != nil { + t.Fatalf("marshal successful decode: %v", err) + } + if len(encoded) > maxRequestContextBytes { + t.Fatalf("encoded Request Context = %d bytes, maximum is %d", len(encoded), maxRequestContextBytes) + } + roundTrip, err := decodeRequestContext(encoded) + if err != nil { + t.Fatalf("decode canonical Request Context: %v", err) + } + if len(roundTrip.values) != len(snapshot.values) { + t.Fatalf("Request Context round trip has %d entries, want %d", len(roundTrip.values), len(snapshot.values)) + } + for key, want := range snapshot.values { + if got, ok := roundTrip.values[key]; !ok || !reflect.DeepEqual(got, want) { + t.Fatalf("Request Context round trip value %q = (%#v, %v), want %#v", key, got, ok, want) + } + } + }) +} + func TestRequestContext_RejectsMalformedPeerBeforeActivation(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) var factoryCalls atomic.Int32 - installRequestContextProbe(t, rt, func(*Binder) requestContextProbe { + installRequestContextProbe(t, rt, func(*GrainContext) requestContextProbe { factoryCalls.Add(1) - return &requestContextProbeEntity{} + return &requestContextProbeGrain{} }) + mustStart(t, rt) oversizedEntries := make(map[string]requestContextWireEntry) for index := 0; index < maxRequestContextEntries+1; index++ { @@ -286,7 +343,7 @@ func TestRequestContext_RejectsMalformedPeerBeforeActivation(t *testing.T) { for _, requestContext := range cases { payload, err := rt.handleInvoke(context.Background(), callRequest{ Kind: requestKindInvoke, - GrainType: TypeName[requestContextProbe](), + GrainType: "gor.requestContextProbe", GrainKey: "malformed", Method: "Snapshot", Args: json.RawMessage(`{}`), @@ -310,7 +367,7 @@ func TestRequestContext_RejectsMalformedPeerBeforeActivation(t *testing.T) { rt.beginClose() payload, err := rt.handleInvoke(context.Background(), callRequest{ Kind: requestKindInvoke, - GrainType: TypeName[requestContextProbe](), + GrainType: "gor.requestContextProbe", GrainKey: "closed", Method: "Snapshot", Args: json.RawMessage(`{}`), @@ -326,6 +383,7 @@ func TestRequestContext_RejectsMalformedPeerBeforeActivation(t *testing.T) { if response.Error == nil || response.Error.Code != string(ErrRuntimeClosed) { t.Fatalf("closed malformed request response = %#v, want runtime-closed", response.Error) } + rt.closeGracefully() } type requestContextSnapshotReply struct { @@ -344,7 +402,7 @@ type requestContextProbe interface { Snapshot(context.Context) (requestContextSnapshotReply, error) } -type requestContextProbeEntity struct{} +type requestContextProbeGrain struct{} type requestContextSnapshotRequest struct{} type requestContextSnapshotResponse struct { @@ -356,7 +414,7 @@ type requestContextProbeProxy struct { id GrainId } -func (e *requestContextProbeEntity) Snapshot(ctx context.Context) (requestContextSnapshotReply, error) { +func (e *requestContextProbeGrain) Snapshot(ctx context.Context) (requestContextSnapshotReply, error) { if _, err := WithRequestContext(ctx, "server", "server-only"); err != nil { return requestContextSnapshotReply{}, err } @@ -404,11 +462,11 @@ func newRequestContextProbeCall(method string) (args any, reply any) { return &requestContextSnapshotRequest{}, &requestContextSnapshotResponse{} } -func installRequestContextProbe(t *testing.T, rt *Runtime, factory func(*Binder) requestContextProbe) { +func installRequestContextProbe(t *testing.T, rt *Runtime, factory func(*GrainContext) requestContextProbe) { t.Helper() - if err := InstallType[requestContextProbe](rt, dispatchRequestContextProbe, func(invoker Invoker, id GrainId) requestContextProbe { + if err := InstallType[requestContextProbe](rt, GeneratedCodeVersion, "gor.requestContextProbe", dispatchRequestContextProbe, func(invoker Invoker, id GrainId) requestContextProbe { return &requestContextProbeProxy{invoker: invoker, id: id} - }, newRequestContextProbeCall, nil); err != nil { + }, newRequestContextProbeCall, noReminderCall); err != nil { t.Fatal(err) } if err := Register[requestContextProbe](rt, factory); err != nil { @@ -446,18 +504,19 @@ func requestContextCallContext(t *testing.T) context.Context { func TestRequestContext_StateRecordHasNoContext(t *testing.T) { backend := store.NewMemory() rt := mustNew(t, WithStore(backend), WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) registerAccount(t, rt) + mustStart(t, rt) ctx, err := WithRequestContext(context.Background(), "trace_id", "state-call") if err != nil { t.Fatal(err) } - id := GrainId{GrainType: TypeName[Account](), GrainKey: "state-isolation"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "state-isolation"} var reply accountDepositReply if err := rt.Invoke(ctx, id, "Deposit", &accountDepositRequest{A0: 7}, &reply); err != nil { t.Fatalf("Deposit: %v", err) } - record, err := backend.Read(context.Background(), store.GrainId(id)) + record, err := backend.Read(context.Background(), toStoreGrainID(id)) if err != nil { t.Fatalf("Read: %v", err) } @@ -468,20 +527,21 @@ func TestRequestContext_StateRecordHasNoContext(t *testing.T) { func TestRequestContext_LocalNestedAndResponseIsolation(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() - installRequestContextProbe(t, rt, func(*Binder) requestContextProbe { - return &requestContextProbeEntity{} + defer closeRuntime(rt) + installRequestContextProbe(t, rt, func(*GrainContext) requestContextProbe { + return &requestContextProbeGrain{} }) - if err := InstallType[requestContextParent](rt, dispatchRequestContextParent, func(invoker Invoker, id GrainId) requestContextParent { + if err := InstallType[requestContextParent](rt, GeneratedCodeVersion, "gor.requestContextParent", dispatchRequestContextParent, func(invoker Invoker, id GrainId) requestContextParent { return &requestContextParentProxy{invoker: invoker, id: id} - }, newRequestContextParentCall, nil); err != nil { + }, newRequestContextParentCall, noReminderCall); err != nil { t.Fatal(err) } - if err := Register[requestContextParent](rt, func(b *Binder) requestContextParent { - return &requestContextParentEntity{target: Ref[requestContextProbe](b, "nested-target")} + if err := Register[requestContextParent](rt, func(b *GrainContext) requestContextParent { + return &requestContextParentGrain{target: Ref[requestContextProbe](b, "nested-target")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) ctx := requestContextCallContext(t) result, err := Ref[requestContextProbe](rt, "local").Snapshot(ctx) @@ -513,7 +573,7 @@ type requestContextParent interface { Nested(context.Context) (requestContextNestedReply, error) } -type requestContextParentEntity struct { +type requestContextParentGrain struct { target requestContextProbe } @@ -527,7 +587,7 @@ type requestContextParentProxy struct { id GrainId } -func (e *requestContextParentEntity) Nested(ctx context.Context) (requestContextNestedReply, error) { +func (e *requestContextParentGrain) Nested(ctx context.Context) (requestContextNestedReply, error) { first, err := e.target.Snapshot(ctx) if err != nil { return requestContextNestedReply{}, err @@ -580,10 +640,12 @@ func TestRequestContext_ForwardedAndIndependentCalls(t *testing.T) { secondOptions := clusterRuntimeOptions(backend, members, fakeClock, "node-b", "generation-b", secondTransport) first := mustNew(t, firstOptions...) second := mustNew(t, secondOptions...) - defer first.Close() - defer second.Close() - installRequestContextProbe(t, first, func(*Binder) requestContextProbe { return &requestContextProbeEntity{} }) - installRequestContextProbe(t, second, func(*Binder) requestContextProbe { return &requestContextProbeEntity{} }) + defer closeRuntime(first) + defer closeRuntime(second) + installRequestContextProbe(t, first, func(*GrainContext) requestContextProbe { return &requestContextProbeGrain{} }) + installRequestContextProbe(t, second, func(*GrainContext) requestContextProbe { return &requestContextProbeGrain{} }) + mustStart(t, first) + mustStart(t, second) synctest.Wait() <-firstTransport.served <-secondTransport.served @@ -625,8 +687,8 @@ func findRequestContextTarget(t *testing.T, rt *Runtime, owner string) GrainId { t.Helper() view := rt.clusterView.Load() for index := 0; index < 4096; index++ { - candidate := GrainId{GrainType: TypeName[requestContextProbe](), GrainKey: fmt.Sprintf("context-%d", index)} - candidateOwner, ok := cluster.Owner(*view, store.GrainId(candidate)) + candidate := GrainId{GrainType: GrainType("gor.requestContextProbe"), GrainKey: fmt.Sprintf("context-%d", index)} + candidateOwner, ok := cluster.Owner(*view, toStoreGrainID(candidate)) if ok && candidateOwner == owner { return candidate } @@ -638,10 +700,11 @@ func findRequestContextTarget(t *testing.T, rt *Runtime, owner string) GrainId { func TestRequestContext_ConcurrentDerivationsPreserveParentSnapshot(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() - installRequestContextProbe(t, rt, func(*Binder) requestContextProbe { - return &requestContextProbeEntity{} + defer closeRuntime(rt) + installRequestContextProbe(t, rt, func(*GrainContext) requestContextProbe { + return &requestContextProbeGrain{} }) + mustStart(t, rt) parent, err := WithRequestContext(context.Background(), "trace_id", "parent") if err != nil { t.Fatal(err) @@ -689,9 +752,10 @@ func TestRequestContext_ConcurrentDerivationsPreserveParentSnapshot(t *testing.T func TestRequestContext_LocalCancellationKeepsValuesReadable(t *testing.T) { synctest.Test(t, func(t *testing.T) { rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) - defer rt.Close() + defer closeRuntime(rt) started := make(chan context.Context, 1) installRequestContextCancellation(t, rt, started) + mustStart(t, rt) ctx, err := WithRequestContext(context.Background(), "trace_id", "cancelled-call") if err != nil { @@ -717,7 +781,7 @@ type requestContextCancellation interface { Block(context.Context) error } -type requestContextCancellationEntity struct { +type requestContextCancellationGrain struct { started chan context.Context } @@ -728,7 +792,7 @@ type requestContextCancellationProxy struct { id GrainId } -func (e *requestContextCancellationEntity) Block(ctx context.Context) error { +func (e *requestContextCancellationGrain) Block(ctx context.Context) error { e.started <- ctx <-ctx.Done() return ctx.Err() @@ -750,13 +814,13 @@ func newRequestContextCancellationCall(method string) (args any, reply any) { func installRequestContextCancellation(t *testing.T, rt *Runtime, started chan context.Context) { t.Helper() - if err := InstallType[requestContextCancellation](rt, dispatchRequestContextCancellation, func(invoker Invoker, id GrainId) requestContextCancellation { + if err := InstallType[requestContextCancellation](rt, GeneratedCodeVersion, "gor.requestContextCancellation", dispatchRequestContextCancellation, func(invoker Invoker, id GrainId) requestContextCancellation { return &requestContextCancellationProxy{invoker: invoker, id: id} - }, newRequestContextCancellationCall, nil); err != nil { + }, newRequestContextCancellationCall, noReminderCall); err != nil { t.Fatal(err) } - if err := Register[requestContextCancellation](rt, func(*Binder) requestContextCancellation { - return &requestContextCancellationEntity{started: started} + if err := Register[requestContextCancellation](rt, func(*GrainContext) requestContextCancellation { + return &requestContextCancellationGrain{started: started} }); err != nil { t.Fatal(err) } @@ -766,14 +830,102 @@ func (p *requestContextCancellationProxy) Block(ctx context.Context) error { return p.invoker.Invoke(ctx, p.id, "Block", &requestContextCancellationRequest{}, &requestContextCancellationResponse{}) } +type requestContextDeadline struct { + context.Context + deadline time.Time +} + +func (c requestContextDeadline) Deadline() (time.Time, bool) { + return c.deadline, true +} + +func TestRequestContext_ActivationUsesRuntimeOwnedContext(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + type privateKey struct{} + activateStarted := make(chan struct{}) + releaseActivate := make(chan struct{}) + grains := make(chan *requestContextLifecycleGrain, 1) + rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + defer func() { + select { + case <-releaseActivate: + default: + close(releaseActivate) + } + }() + installRequestContextLifecycleConfigured(t, rt, grains, func(grain *requestContextLifecycleGrain) { + grain.activateStarted = activateStarted + grain.releaseActivate = releaseActivate + }) + mustStart(t, rt) + + base := requestContextDeadline{Context: context.Background(), deadline: time.Unix(9_999_999, 0).UTC()} + base = requestContextDeadline{Context: context.WithValue(base, privateKey{}, "private"), deadline: base.deadline} + first, err := WithRequestContext(base, "trace_id", "activation") + if err != nil { + t.Fatal(err) + } + first, cancelFirst := context.WithCancel(first) + ref := Ref[requestContextLifecycle](rt, "lifecycle") + firstDone := make(chan error, 1) + go func() { + _, err := ref.Snapshot(first) + firstDone <- err + }() + synctest.Wait() + <-activateStarted + grain := <-grains + + second, err := WithRequestContext(context.Background(), "trace_id", "method") + if err != nil { + t.Fatal(err) + } + secondDone := make(chan error, 1) + go func() { + result, err := ref.Snapshot(second) + if err == nil && result.Trace != "method" { + err = fmt.Errorf("method Request Context = %q, want method", result.Trace) + } + secondDone <- err + }() + synctest.Wait() + + cancelFirst() + synctest.Wait() + if err := <-firstDone; !errors.Is(err, context.Canceled) { + t.Fatalf("triggering Call error = %v, want context.Canceled", err) + } + if err := grain.activateContext.Err(); err != nil { + t.Fatalf("OnActivate context after caller cancellation = %v, want nil", err) + } + if _, ok := grain.activateContext.Deadline(); ok { + t.Fatal("OnActivate copied the caller deadline") + } + if got, ok := RequestContextValue(grain.activateContext, "trace_id"); !ok || got != "activation" { + t.Fatalf("OnActivate Request Context = (%v, %v), want activation", got, ok) + } + if got := grain.activateContext.Value(privateKey{}); got != nil { + t.Fatalf("OnActivate copied an unrelated context value: %v", got) + } + + close(releaseActivate) + synctest.Wait() + if err := <-secondDone; err != nil { + t.Fatalf("waiting Call error = %v", err) + } + }) +} + func TestRequestContext_LifecycleAndReminderUseFreshContexts(t *testing.T) { synctest.Test(t, func(t *testing.T) { start := time.Unix(1900, 0).UTC() fakeClock := clock.NewFake(start) backend := store.NewMemory() - entities := make(chan *requestContextLifecycleEntity, 1) - rt := mustNew(t, WithStore(backend), WithClock(fakeClock), WithReminderInterval(time.Second), WithIdleTimeout(0), WithEvictionInterval(0)) - installRequestContextLifecycle(t, rt, entities) + grains := make(chan *requestContextLifecycleGrain, 1) + rt := mustNew(t, WithStore(backend), WithReminderStore(backend), WithClock(fakeClock), WithReminderInterval(time.Second), WithIdleTimeout(0), WithEvictionInterval(0)) + installRequestContextLifecycle(t, rt, grains) + mustStart(t, rt) first, err := WithRequestContext(context.Background(), "trace_id", "activation") if err != nil { @@ -783,7 +935,7 @@ func TestRequestContext_LifecycleAndReminderUseFreshContexts(t *testing.T) { if _, err := ref.Snapshot(first); err != nil { t.Fatalf("first lifecycle call: %v", err) } - entity := <-entities + grain := <-grains second, err := WithRequestContext(context.Background(), "trace_id", "method") if err != nil { t.Fatal(err) @@ -795,11 +947,11 @@ func TestRequestContext_LifecycleAndReminderUseFreshContexts(t *testing.T) { if result.Trace != "method" { t.Fatalf("method Request Context = %q, want method", result.Trace) } - if got, _ := RequestContextValue(entity.activateContext, "trace_id"); got != "activation" { + if got, _ := RequestContextValue(grain.activateContext, "trace_id"); got != "activation" { t.Fatalf("OnActivate Request Context = %v, want activation", got) } - rt.Close() - if _, ok := RequestContextValue(entity.deactivateContext, "trace_id"); ok { + closeRuntime(rt) + if _, ok := RequestContextValue(grain.deactivateContext, "trace_id"); ok { t.Fatal("OnDeactivate received Request Context") } @@ -807,9 +959,10 @@ func TestRequestContext_LifecycleAndReminderUseFreshContexts(t *testing.T) { // Reminder delivery below starts with the poller's empty context. reminderBackend := store.NewMemory() reminderClock := clock.NewFake(start) - reminderRuntime := mustNew(t, WithStore(reminderBackend), WithClock(reminderClock), WithReminderInterval(time.Second), WithIdleTimeout(0), WithEvictionInterval(0)) + reminderRuntime := mustNew(t, WithStore(reminderBackend), WithReminderStore(reminderBackend), WithClock(reminderClock), WithReminderInterval(time.Second), WithIdleTimeout(0), WithEvictionInterval(0)) wakeContext := make(chan context.Context, 1) installRequestContextReminder(t, reminderRuntime, wakeContext) + mustStart(t, reminderRuntime) armContext, err := WithRequestContext(context.Background(), "trace_id", "caller") if err != nil { t.Fatal(err) @@ -817,9 +970,9 @@ func TestRequestContext_LifecycleAndReminderUseFreshContexts(t *testing.T) { if err := Ref[requestContextReminder](reminderRuntime, "reminder").Arm(armContext); err != nil { t.Fatalf("Arm: %v", err) } - rows, err := reminderBackend.ListDue(context.Background(), start) - if err != nil || len(rows) != 1 { - t.Fatalf("stored Reminders = (%#v, %v), want one row", rows, err) + rows := readDueReminders(t, reminderBackend, start) + if len(rows) != 1 { + t.Fatalf("stored Reminders = %#v, want one row", rows) } stored, _ := json.Marshal(rows[0]) if strings.Contains(string(stored), "request_context") { @@ -835,7 +988,7 @@ func TestRequestContext_LifecycleAndReminderUseFreshContexts(t *testing.T) { default: t.Fatal("Reminder did not run") } - reminderRuntime.Close() + closeRuntime(reminderRuntime) }) } @@ -843,9 +996,11 @@ type requestContextLifecycle interface { Snapshot(context.Context) (requestContextSnapshotReply, error) } -type requestContextLifecycleEntity struct { +type requestContextLifecycleGrain struct { activateContext context.Context deactivateContext context.Context + activateStarted chan struct{} + releaseActivate chan struct{} } type requestContextLifecycleProxy struct { @@ -853,17 +1008,23 @@ type requestContextLifecycleProxy struct { id GrainId } -func (e *requestContextLifecycleEntity) OnActivate(ctx context.Context) error { +func (e *requestContextLifecycleGrain) OnActivate(ctx context.Context) error { e.activateContext = ctx + if e.activateStarted != nil { + close(e.activateStarted) + } + if e.releaseActivate != nil { + <-e.releaseActivate + } return nil } -func (e *requestContextLifecycleEntity) OnDeactivate(ctx context.Context, _ DeactivationReason) error { +func (e *requestContextLifecycleGrain) OnDeactivate(ctx context.Context, _ DeactivationReason) error { e.deactivateContext = ctx return nil } -func (e *requestContextLifecycleEntity) Snapshot(ctx context.Context) (requestContextSnapshotReply, error) { +func (e *requestContextLifecycleGrain) Snapshot(ctx context.Context) (requestContextSnapshotReply, error) { trace, _ := RequestContextValue(ctx, "trace_id") return requestContextSnapshotReply{Trace: trace.(string)}, nil } @@ -886,17 +1047,24 @@ func newRequestContextLifecycleCall(method string) (args any, reply any) { return &requestContextSnapshotRequest{}, &requestContextSnapshotResponse{} } -func installRequestContextLifecycle(t *testing.T, rt *Runtime, entities chan *requestContextLifecycleEntity) { +func installRequestContextLifecycle(t *testing.T, rt *Runtime, grains chan *requestContextLifecycleGrain) { + installRequestContextLifecycleConfigured(t, rt, grains, nil) +} + +func installRequestContextLifecycleConfigured(t *testing.T, rt *Runtime, grains chan *requestContextLifecycleGrain, configure func(*requestContextLifecycleGrain)) { t.Helper() - if err := InstallType[requestContextLifecycle](rt, dispatchRequestContextLifecycle, func(invoker Invoker, id GrainId) requestContextLifecycle { + if err := InstallType[requestContextLifecycle](rt, GeneratedCodeVersion, "gor.requestContextLifecycle", dispatchRequestContextLifecycle, func(invoker Invoker, id GrainId) requestContextLifecycle { return &requestContextLifecycleProxy{invoker: invoker, id: id} - }, newRequestContextLifecycleCall, nil); err != nil { + }, newRequestContextLifecycleCall, noReminderCall); err != nil { t.Fatal(err) } - if err := Register[requestContextLifecycle](rt, func(*Binder) requestContextLifecycle { - entity := &requestContextLifecycleEntity{} - entities <- entity - return entity + if err := Register[requestContextLifecycle](rt, func(*GrainContext) requestContextLifecycle { + grain := &requestContextLifecycleGrain{} + if configure != nil { + configure(grain) + } + grains <- grain + return grain }); err != nil { t.Fatal(err) } @@ -913,7 +1081,7 @@ type requestContextReminder interface { Wake(context.Context, TickStatus) error } -type requestContextReminderEntity struct { +type requestContextReminderGrain struct { schedule Reminder[requestContextReminder] wake chan context.Context } @@ -928,11 +1096,11 @@ type requestContextReminderArmResponse struct{} type requestContextReminderWakeRequest struct{ A0 TickStatus } type requestContextReminderWakeResponse struct{} -func (e *requestContextReminderEntity) Arm(ctx context.Context) error { +func (e *requestContextReminderGrain) Arm(ctx context.Context) error { return e.schedule.Set(ctx, "wake", After(0), Handle(requestContextReminder.Wake)) } -func (e *requestContextReminderEntity) Wake(ctx context.Context, _ TickStatus) error { +func (e *requestContextReminderGrain) Wake(ctx context.Context, _ TickStatus) error { e.wake <- ctx return nil } @@ -968,13 +1136,13 @@ func newRequestContextReminderCallFromStatus(method string, status TickStatus) ( func installRequestContextReminder(t *testing.T, rt *Runtime, wake chan context.Context) { t.Helper() - if err := InstallType[requestContextReminder](rt, dispatchRequestContextReminder, func(invoker Invoker, id GrainId) requestContextReminder { + if err := InstallType[requestContextReminder](rt, GeneratedCodeVersion, "gor.requestContextReminder", dispatchRequestContextReminder, func(invoker Invoker, id GrainId) requestContextReminder { return &requestContextReminderProxy{invoker: invoker, id: id} }, newRequestContextReminderCall, newRequestContextReminderCallFromStatus); err != nil { t.Fatal(err) } - if err := Register[requestContextReminder](rt, func(b *Binder) requestContextReminder { - return &requestContextReminderEntity{schedule: NewReminder[requestContextReminder](b), wake: wake} + if err := Register[requestContextReminder](rt, func(b *GrainContext) requestContextReminder { + return &requestContextReminderGrain{schedule: NewReminder[requestContextReminder](b), wake: wake} }); err != nil { t.Fatal(err) } diff --git a/research/README.md b/research/README.md index 40ed4e3..5d56232 100644 --- a/research/README.md +++ b/research/README.md @@ -1,22 +1,36 @@ -# research —— 证据基线 - -这一层是 [`design/`](../design/README.md) 里各项决策的**实测依据**。 - -与 `design/` 的区别：`design/` 写「我们要怎么做」，`research/` 写「我们看到了什么」。两者分开是为了让决策可以被重新审视——如果将来某个决策要翻案，先看它依赖的事实是否还成立。 - -## 记录纪律 - -- **只写实测到的东西。** 行数是数出来的，star 数是查出来的，源码结论是读出来的。推断与判断要标明。 -- **带测量日期。** 生态数据会过期，过期的数据不删，标注日期让读者自己判断。 -- **说清测量方法。** 「Orleans 有 27 万行」这种数字没有方法就没有意义。 - -## 篇目 - -- [orleans-internals.md](orleans-internals.md) —— Orleans 10.1 源码实测：规模构成、单激活仲裁、membership 无共识、调度器。 -- [landscape.md](landscape.md) —— 同类方案现状：Go 侧的虚拟 actor 实现、durable execution 阵营、已死的先例。 -- [go-capabilities.md](go-capabilities.md) —— Go 平台相对 .NET 的优势与劣势，以及每一条对设计的影响。 -- [embedded-store-bench.md](embedded-store-bench.md) —— 嵌入式存储后端在统一条件下的读写与冷启动实测，是持久化选型可回看的证据。 - -## 测量时间 - -全部数据采集于 **2026-07-30**，对象为 Orleans 10.1（`dotnet/orleans` 稀疏 clone）与各项目当时的 GitHub 状态。 +# Research - Evidence Baseline + +This layer contains measured evidence for the decisions in [`design/`](../design/README.md). + +`design/` says what we plan to do. `research/` says what we observed. We keep +them separate so that we can review each decision. If a decision changes, we +first check whether its evidence still holds. + +## Record Rules + +- **Record only measured facts.** We count lines, check star counts, and read + source code. We mark inferences and judgments. +- **Add the measurement date.** Ecosystem data becomes old. We keep old data + and show its date so that readers can judge its age. +- **State the measurement method.** A number such as "Orleans has 270,000 + lines" has no clear meaning without a method. + +## Contents + +- [orleans-internals.md](orleans-internals.md) - Measured Orleans 10.1 source + data: size, single-Activation arbitration, membership without consensus, and + scheduling. +- [landscape.md](landscape.md) - The current state of related solutions: Go + virtual actor implementations, durable execution products, and discontinued + examples. +- [go-capabilities.md](go-capabilities.md) - Go capabilities that differ from + .NET, and the design impact of each difference. +- [embedded-store-bench.md](embedded-store-bench.md) - Measured read, write, + and cold-start results for embedded stores under the same conditions. These + results support later persistence reviews. + +## Measurement Date + +We collected all data on **2026-07-30**. The subjects were Orleans 10.1 (a +sparse clone of `dotnet/orleans`) and the GitHub state of each project at that +time. diff --git a/research/embedded-store-bench.md b/research/embedded-store-bench.md index 049271d..c04feec 100644 --- a/research/embedded-store-bench.md +++ b/research/embedded-store-bench.md @@ -1,20 +1,25 @@ -# 嵌入式存储实测 +# Embedded Store Measurements -本文件记录批次 0 的实测事实和采集方法，不包含选型结论或推荐。 +This file records measured facts and the collection method for Batch 0. It +does not contain a selection conclusion or recommendation. -## 条件 +## Conditions - Machine: AMD Ryzen 9 9955HX 16-Core Processor, 1 socket, 16 cores, 32 threads. - OS: Linux `pluto`, kernel `7.0.0-27-generic`, `x86_64`. - Storage path: `/home/szf/repos/gor/.claude/worktrees/step2-persistence/.bench-data`. -- Filesystem: `ext4`, mounted `rw` from `/dev/mapper/ubuntu--vg-ubuntu--lv`; the backing device is LVM on `nvme0n1` (`CT2000P310SSD8`, `ROTA=0`). +- Filesystem: `ext4`, mounted `rw` from + `/dev/mapper/ubuntu--vg-ubuntu--lv`. The backing device is LVM on `nvme0n1` + (`CT2000P310SSD8`, `ROTA=0`). - Go: `go1.26.5`, `linux/amd64`; `GOMAXPROCS=32`. - Candidates: `modernc.org/sqlite v1.55.0`, `go.etcd.io/bbolt v1.5.0`, `github.com/cockroachdb/pebble/v2 v2.1.6`. - Logical record data sizes: 256 B and 4 KiB. - Seed size: 50,000 records per database. - Timed samples: 5 per point-read, CAS-write, and concurrent-write result; 10 per cold-open result. -- Workload operations: 100,000 point reads and 4,096 writes per timed sample. The write count is divisible by 1, 8, and 64. -- Timed runs use `time.Now()` around the complete operation loop. Reported `ns/op` is elapsed sample time divided by operation count. +- Workload operations: 100,000 point reads and 4,096 writes per timed sample. + The write count is divisible by 1, 8, and 64. +- Timed runs use `time.Now()` around the complete operation loop. Reported + `ns/op` is elapsed sample time divided by the operation count. - Each workload uses a newly seeded database. The OS page cache was not cleared between samples or candidates. Backend settings: @@ -25,33 +30,63 @@ Backend settings: | bbolt | No separate WAL; default `NoSync=false` | Default mmap and transaction settings; open timeout 1 minute | | Pebble | Default WAL; `WriteOptions{Sync:true}` for every write batch | Default Pebble options; logging disabled for benchmark output | -The SQLite store uses a table with `id BLOB PRIMARY KEY`, `data BLOB`, and `etag INTEGER`. bbolt stores records in a `records` bucket. Pebble uses the record identity as the key. The bbolt and Pebble values contain an 8-byte big-endian ETag followed by the data payload. +The SQLite store uses a table with `id BLOB PRIMARY KEY`, `data BLOB`, and +`etag INTEGER`. bbolt stores records in a `records` bucket. Pebble uses the +record key as its key. The bbolt and Pebble values contain an 8-byte +big-endian ETag followed by the data payload. ## Method -Point read reads the pre-existing `entity-00000` key. Each CAS-write operation reads the current record, then writes the same-size payload with the current ETag as the expected ETag and the ETag incremented by one. Each write is one committed transaction or batch. +Point read uses the existing key whose literal value is `entity-00000`. Each +CAS-write operation reads the current record. It then writes a payload of the +same size. It uses the current ETag as the expected ETag and adds one to the +new ETag. Each write is one committed transaction or batch. -The concurrent-write workload starts exactly 8 or 64 workers. Each worker repeatedly performs the same read-then-CAS sequence on its own pre-existing key, so workers do not contend on the same identity. The CAS implementations validate the expected ETag before committing. +The concurrent-write workload starts exactly 8 or 64 goroutines. Each +goroutine performs the same read-then-CAS sequence on its own existing key. +Thus, the goroutines do not contend on one key. The CAS implementations +validate the expected ETag before they commit. ### Pebble CAS behavior -Pebble has no conditional-update primitive matching SQLite's `UPDATE ... WHERE etag` or bbolt's writable transaction. Its benchmark path creates an indexed batch, reads the current value through that batch, checks the ETag, sets the value, and commits the batch with `Sync:true`. The batch commit is atomic for the operations in that batch, but the ETag check is not atomic with respect to the current database value. If two batches write the same key concurrently, both can read the same ETag and both can commit successfully. The benchmark uses a distinct key per worker, so this same-key behavior is not exercised by the concurrent-write numbers. +Pebble has no conditional-update operation that matches SQLite's +`UPDATE ... WHERE etag` or a bbolt write transaction. Its benchmark path +creates an indexed batch. It reads the current value through the batch, +checks the ETag, sets the value, and commits with `Sync:true`. -Each sequential workload has 100 warm-up operations. Concurrent workloads warm one key per worker before timing. The cold-open workload seeds and closes a database, then measures `Open` plus the backend initialization needed by the benchmark: SQLite includes `Ping` and prepared statement creation; bbolt and Pebble include their normal open path. It closes the database after every cold-open sample. +The commit is atomic for operations in one batch. The ETag check is not atomic +with respect to the current database value. Two concurrent batches can read +the same ETag and both can commit. The benchmark uses a different key for each +goroutine. Thus, the concurrent-write results do not test this same-key case. -The benchmark source and module are outside this repository at `/tmp/gor-storebench/`. The formal command was: +Each sequential workload has 100 warm-up operations. Concurrent workloads +warm one key for each goroutine before timing. The cold-open workload seeds +and closes a database. It then measures `Open` and the required backend +initialization. SQLite includes `Ping` and prepared statement creation. bbolt +and Pebble include their normal open path. The workload closes the database +after each cold-open sample. + +The one-time benchmark source and module were outside this repository at +`/tmp/gor-storebench/`. They were not retained. The command below records the +original method, but it cannot reproduce the results now: ```text GOCACHE=/tmp/gocache GOPROXY=off go run . -data-dir /home/szf/repos/gor/.claude/worktrees/step2-persistence/.bench-data -runs 5 -point-ops 100000 -write-ops 4096 -cold-samples 10 ``` -The seed operation used one transaction for SQLite, one writable transaction for bbolt, and one synchronous batch commit for Pebble. The benchmark source passed `go test ./...` before the formal run. +The seed operation used one transaction for SQLite and one write transaction +for bbolt. It used one synchronous batch commit for Pebble. The benchmark +source passed `go test ./...` before the formal run. The `.bench-data` directory was removed after the formal run. +The current SQLite performance checks are in +[`store/benchmark_test.go`](../store/benchmark_test.go). + ## Results -All values are elapsed nanoseconds per operation. `range` is the minimum-to-maximum sample range. `median ops/s` is derived from the median value. +All values are elapsed nanoseconds for each operation. `range` shows the +minimum and maximum sample values. `median ops/s` comes from the median value. ### Point read @@ -75,7 +110,7 @@ All values are elapsed nanoseconds per operation. `range` is the minimum-to-maxi | Pebble | 256 B | 1 | 3,969,279 | 3,806,799-4,454,872 | 251.93 | | Pebble | 4 KiB | 1 | 871,199 | 852,571-1,602,912 | 1,147.84 | -### Concurrent write, 8 workers +### Concurrent write, 8 goroutines | Backend | Data | Concurrency | Median ns/op | Range ns/op | Median ops/s | |---|---:|---:|---:|---:|---:| @@ -86,7 +121,7 @@ All values are elapsed nanoseconds per operation. `range` is the minimum-to-maxi | Pebble | 256 B | 8 | 417,302 | 410,623-1,014,412 | 2,396.35 | | Pebble | 4 KiB | 8 | 225,535 | 212,398-429,257 | 4,433.90 | -### Concurrent write, 64 workers +### Concurrent write, 64 goroutines | Backend | Data | Concurrency | Median ns/op | Range ns/op | Median ops/s | |---|---:|---:|---:|---:|---:| diff --git a/research/go-capabilities.md b/research/go-capabilities.md index c63dffe..d573fc3 100644 --- a/research/go-capabilities.md +++ b/research/go-capabilities.md @@ -1,63 +1,110 @@ -# Go 平台能力边界 +# Go Platform Capability Boundaries -把 Orleans 的做法搬到 Go，哪些变简单、哪些变难。每条都标出对设计的影响。 +This document compares what becomes easier or harder when we adapt the +Orleans runtime model to Go. Each item states its design impact. -## Go 更省的地方 +## What Go Makes Easier -**没有 async/await 的颜色问题。** Orleans 需要 823 行自定义 `TaskScheduler`（`WorkItemGroup` 336 行 + `ActivationTaskScheduler` 171 行 + ……），目的是保证 `await` 之后回到同一逻辑执行上下文。Go 里 goroutine 本身就是执行上下文，同样语义约 100 行——一个 goroutine 从一个 channel 读，循环执行。 +**No async/await color problem.** Orleans needs 823 lines of custom +`TaskScheduler` code (`WorkItemGroup` 336 lines + `ActivationTaskScheduler` +171 lines + ...). The purpose is to return to the same logical execution +context after `await`. The `gor` design uses one goroutine to read one channel +and run a loop. It does not need a custom platform scheduler. -→ [../design/scheduling.md](../design/scheduling.md) +-> [../design/scheduling.md](../design/scheduling.md) -**不需要序列化子系统。** Orleans 的序列化实测 30,761 行，主要复杂度在版本容忍（滚动升级时新旧节点互通）。`gor` 放弃这个能力，用标准库能覆盖的编码就够。 +**No separate serialization subsystem.** The measured Orleans serialization +code has 30,761 lines. Most of its complexity supports version tolerance so +that old and new Silos can communicate during a rolling upgrade. `gor` does +not provide this capability. Standard library encoding is sufficient for the +current product contract. -→ [../design/architecture.md](../design/architecture.md)（含代价说明：不支持不兼容变更的不停机升级） +-> [../design/architecture.md](../design/architecture.md) (including the cost: +no rolling upgrade for incompatible changes) -**单二进制分发。** 这不只是打包方便。它决定了产品形态：不需要装 server、不需要跑 sidecar、`import` 进去就能用。Temporal / Restate / Rivet / Dapr 都做不到这一点，这是 `gor` 存在的主要理由。 +**Single-binary distribution.** This is more than a packaging benefit. It +defines the product form: users do not install a server, run a sidecar, or +start a separate process. Users can use `gor` through `import`. Temporal, +Restate, Rivet, and Dapr do not provide this form. This is a main reason for +`gor`. -**嵌入式存储可选项多且都是纯 Go。** `modernc.org/sqlite`、bbolt、pebble 都无需 CGO，单节点持久化不引入外部依赖。 +**Many embedded store options are pure Go.** `modernc.org/sqlite`, bbolt, and +pebble do not need CGO. Single-Silo persistence can therefore avoid an +external dependency. -→ [../design/persistence.md](../design/persistence.md) +-> [../design/persistence.md](../design/persistence.md) -**`testing/synctest`（Go 1.25 GA）。** 假时钟 + 「durably blocked」静止判定，直接消灭了「`time.Sleep` 然后祈祷」这个最大的 flaky 来源。.NET 侧没有等价物——Orleans 的 136 个含 `Sleep` 的测试文件就是这个缺口的证据。 +**`testing/synctest` (Go 1.25 GA).** A fake clock and the "durably blocked" +quiescence check remove the largest source of flaky tests: "run `time.Sleep` +and hope." .NET has no equivalent feature. The 136 Orleans test files that +contain `Sleep` are evidence of this gap. -## Go 更难的地方 +## What Go Makes Harder -**没有 `AsyncLocal`。** .NET 里 `RequestContext` 靠 `AsyncLocal` 隐式沿异步调用链传播。Go 里只能显式塞进 `context.Context` 并层层传递。 +**No `AsyncLocal`.** .NET uses `AsyncLocal` to pass Request Context through an +implicit asynchronous call path. Go must pass it explicitly through +`context.Context` and each layer. -影响：调用环检测（需要沿链传「已占用的实体」集合）必须显式穿透所有中间层。这是实打实的劣势，没有优雅解法。 +Impact: Call-cycle detection must explicitly pass a set of Grains already in +the Call path through every intermediate layer. This is a real limitation. It +has no simple alternative. -→ [../design/runtime.md](../design/runtime.md) +-> [../design/runtime.md](../design/runtime.md) -**反射能力弱。** `reflect.MakeFunc` 能构造函数值，但**不能构造实现任意接口的类型**。所以「给我一个 `Account`，调用转发到远端」这件事无法在运行时完成。 +**Limited reflection.** `reflect.MakeFunc` can create a function value. It +cannot create a type that implements an arbitrary interface at run time. The +runtime therefore cannot receive an `Account` value and forward a Call to a +remote Grain at run time. -影响：必须做代码生成。这是 `gor` 相对 goakt 多出来的一个构建步骤，换来编译期类型安全。 +Impact: `gor` must use code generation. This adds a build step compared with +goakt, but it provides compile-time type safety. -→ [../design/codegen.md](../design/codegen.md) +-> [../design/codegen.md](../design/codegen.md) -**没有版本容忍序列化。** 上面算作「省了 3 万行」的同一件事，从另一面看是能力缺失。Orleans 能不停机滚动升级到不兼容的方法签名，`gor` 不能。 +**No version-tolerant serialization.** The same fact that saves about 30,000 +lines above is also a missing capability. Orleans can perform a rolling +upgrade with incompatible method signatures. `gor` cannot. -**没有可用的 DST 框架。** gosim 停更、Antithesis 每年 16.8 万美元、porcupine 只检查历史不控制调度。Go 运行时不提供 goroutine 调度控制。 +**No usable DST framework.** gosim is no longer maintained, Antithesis costs +168 thousand dollars per year, and porcupine checks histories but does not +control scheduling. The Go runtime does not control goroutine scheduling. -影响：`sim` 包必须自建，而且它对生产代码提出四条硬约束（I/O 在接口后、时间可注入、组件是显式状态机、等待用 channel 不用 mutex）。这是全项目最大的持续性成本。 +Impact: The `sim` package must be built in-house. It requires four production +code rules: put I/O behind interfaces, inject time, use explicit state +machines, and use channels instead of mutexes for waits. This is a long-term +design cost for the project. -→ [../design/testing.md](../design/testing.md) +-> [../design/testing.md](../design/testing.md) -## 两个容易踩的具体坑 +## Two Specific Traps -**mutex 阻塞不算 durably blocking。** 在 `synctest` bubble 里，goroutine 阻塞在 `sync.Mutex` 上不会被判为静止，`synctest.Wait()` 会挂或误判。 +**Mutex blocking is not durably blocked.** In a `synctest` bubble, a goroutine +blocked on a `sync.Mutex` is not marked as quiescent. `synctest.Wait()` can +then hang or give a wrong result. -后果：任何用 mutex 做「等待」的实现都会让测试策略失效。所以连 `golang.org/x/sync/singleflight` 都不能用（它内部是 mutex），激活去重要自己用 channel 实现。 +Result: Any implementation that uses a mutex for a wait breaks the test +strategy. For this reason, `golang.org/x/sync/singleflight` is not suitable: +it uses a mutex internally. Activation deduplication must use channels. -**`go/types` 要求包能通过类型检查。** 如果生成物和用户接口同包，会形成死锁：生成物不存在 → 用户代码引用它 → 包类型检查失败 → 生成器加载不了包 → 生成不出来。 +**`go/types` requires a type-checkable package.** If generated code and the +user interface are in the same package, a deadlock occurs: -解法：生成物落子包，用户包不直接引用生成物。 +generated code is missing -> user code refers to it -> package type checking +fails -> the generator cannot load the package -> the generator cannot create +the generated code. -## Go 侧可参考的代码生成先例 +Solution: Put generated code in a subpackage. The user package must not refer +to generated code directly. -| 项目 | 可借鉴的点 | -|---|---| -| `alecthomas/go-rpcgen` | 输入契约的形状：interface + 命名返回值 + 末位 error | -| `segmentio/glue` | 用一个窄 `Call` 接口让生成物与传输实现解耦 | -| Encore | metadata 驱动生成，而非纯语法树驱动 | +## Go Code Generation Examples -三者合起来给出了 [../design/codegen.md](../design/codegen.md) 的设计：从 Go interface 读契约，生成物只依赖窄 `Invoker` 接口，运行时内部变更不需要重新生成。 +| Project | Reusable point | +|---|---| +| `alecthomas/go-rpcgen` | Input contract shape: interface + named returns + final error | +| `segmentio/glue` | A narrow `Call` interface decouples generated code from transport | +| Encore | Metadata-driven generation instead of pure syntax-tree generation | + +Together, these three examples support the design in +[../design/codegen.md](../design/codegen.md): read the contract from a Go +interface, make generated code depend only on a narrow `Invoker` interface, +and allow internal runtime changes without regeneration. diff --git a/research/landscape.md b/research/landscape.md index e3b9b9c..2fa578e 100644 --- a/research/landscape.md +++ b/research/landscape.md @@ -1,22 +1,25 @@ -# 生态现状 +# Ecosystem State -测量日期 2026-07-30。star 数会变，日期在这里是为了让读者判断数据是否还新鲜。 +Measurement date: 2026-07-30. Star counts change. The date lets readers judge +whether the data is still current. -## 市场共识已经从「虚拟 actor」漂移到「持久化执行」 +## Durable Execution Products -| 项目 | stars | 语言 | 形态 | +| Project | stars | language | form | |---|---|---|---| -| Temporal | 21,947 | Go | 独立 server + 数据库 + worker | -| Rivet | 5,762 | Rust | 单二进制 server | -| Restate | 4,233 | Rust | 单二进制 server | +| Temporal | 21,947 | Go | Separate server + database + worker | +| Rivet | 5,762 | Rust | Single-binary server | +| Restate | 4,233 | Rust | Single-binary server | -Restate 的 "virtual objects" 就是 grain 换了个名字：有 key 的对象 + 持久 K/V + 串行执行。它把 Orleans 的模型重新包装成 durable execution 卖出去了。 +Restate's "virtual objects" use a different name for a Grain. Each object has a +key, durable K/V storage, and serial execution. This evidence shows that +restart recovery has user value. `gor` keeps the Orleans Grain model and +terms. -这是 `gor` 定位跟着 durable execution 走的原因（[../docs/vision.md](../docs/vision.md)）：卖点是「崩溃后接着跑」，不是「actor 模型」。用 actor 的词汇讲同一件事，市场已经不接了。 +## Go Virtual Actor Implementations -## Go 侧的虚拟 actor 实现 - -**goakt**（`Tochemey/goakt`）—— Go 里最接近的库。实测它的 grain API： +**goakt** (`Tochemey/goakt`) is the closest Go library. We measured its Grain +API: ```go // actor/grain.go @@ -33,52 +36,80 @@ func GrainOf[T Grain](ctx context.Context, system ActorSystem, name string, opts AskGrain(ctx context.Context, identity *GrainIdentity, message any, timeout time.Duration) (response any, err error) ``` -两处关键观察： +Two key observations: -1. **`any` 进 `any` 出。** `GrainContext.Message() any`（grain_context.go:162）、`Response(resp any)`（:258）。所有类型错误推迟到运行时。这是 `gor` 做代码生成的直接动因（[../design/codegen.md](../design/codegen.md)）。 -2. **没有持久化状态。** 通读 `actor/grain_option.go`，没有 `WithGrainStateStore` 或任何状态存储选项——**不存在 `[PersistentState]` 的对等物**。用户得自己接存储。 +1. **`any` in and `any` out.** `GrainContext.Message() any` + (`grain_context.go:162`) and `Response(resp any)` (`:258`) use `any`. + Type errors are therefore delayed until run time. This directly motivates + code generation in `gor` ([../design/codegen.md](../design/codegen.md)). +2. **No durable State.** A review of `actor/grain_option.go` found no + `WithGrainStateStore` or other State store option. There is no equivalent + of `[PersistentState]`. Users must connect their own store. -另外 `GrainFactory` 已标 Deprecated，转向 `GrainOf` + `WithGrainDependencies`——API 还在动。 +`GrainFactory` is marked Deprecated. The API moves to `GrainOf` and +`WithGrainDependencies`. The API is still changing. -维护集中在单人。这对一个要被放进生产系统的基础库是实打实的可信度问题。 +At the measurement date, one person made most maintenance changes. This +concentration is a maintenance risk for a base library. -**proto.actor**（Go 版）—— 是 Akka 式 actor 加上 "virtual actor"（cluster grain）。它的 grain 走 protobuf IDL 生成，类型是有的，但代价是引入一套 IDL 和 protobuf 依赖，而且核心仍然是 Akka 式监督模型，不是持久化对象模型。 +**proto.actor** (Go) combines Akka-style actors with virtual actors (cluster +Grains). Its Grain API uses protobuf IDL generation and provides types. The +cost is a new IDL and a protobuf dependency. Its core still uses Akka-style +supervision. It is not a durable object model. -**Dapr** —— 有虚拟 actor，Go 写的。但形态是 sidecar：多一个部署单元、多一跳网络、多一套配置。不是库。 +**Dapr** provides virtual actors in Go. Its form is a sidecar: one more +deployment unit, one more network hop, and one more configuration system. It +is not a library. -## 已死的先例 +## Discontinued Examples -**Orbit**（`orbit/orbit`）—— EA / BioWare 做的 JVM 虚拟 actor 实现，Orleans 启发。 +**Orbit** (`orbit/orbit`) is a JVM virtual actor implementation from EA / BioWare. +It was inspired by Orleans. - 1,724 stars -- 用 Kotlin 重写过一轮 -- **2021-06-15 后停更** -- `orbit-legacy/Orbit1` 只有 9 stars - -技术上它没输给谁。它死于生态——没有形成用户群，公司内部需求变了，就没人推了。 +- Rewritten in Kotlin +- No updates after **2021-06-15** +- `orbit-legacy/Orbit1` has only 9 stars -这是 [../ROADMAP.md](../ROADMAP.md) 风险小节里「生态风险」的实证：这个位置上有过一个资源充足、技术过关的项目，然后它死了。所以差异化定位（单二进制库形态 + DST + 类型安全）比功能对齐 Orleans 更重要。 +The repository does not establish why work stopped. It does show that a +project with 1,724 stars can become inactive. This is evidence for the +"ecosystem risk" section in [../ROADMAP.md](../ROADMAP.md). -## DST 工具链 +## DST Toolchain -`gor` 的测试策略依赖确定性模拟测试，所以专门查了 Go 侧有什么可用： +The `gor` test strategy depends on deterministic simulation testing. We +therefore checked the available Go tools: -| 工具 | 状态 | 能不能用 | +| Tool | status | usable | |---|---|---| -| `testing/synctest` | **Go 1.25 GA** | 能，但只覆盖单元测试层 | -| gosim（`jellevandenhooff/gosim`） | 80 stars，**2024-12 后停更** | 形状最对（多机 + 确定性 goroutine 调度），但不能依赖一个停更的地基 | -| Antithesis | 商业产品 | 能用，2025-09 报价 **16.8 万美元/年** | -| porcupine（`anishathalye/porcupine`） | 1,230 stars，活跃 | 能用，但它只检查历史是否可线性化，不控制调度 | - -Resonate 团队的公开结论与此一致：Go 里无法控制 goroutine 调度，DST 只能靠侵入式地约束整个代码库。 - -**结论**：没有可依赖的现成 DST 框架，`sim` 包必须自建，而自建的前提是代码库从第一天就满足那几条约束。这就是为什么 DST 在 `gor` 里是架构约束而不是测试任务（[../design/testing.md](../design/testing.md)）。 - -## 跨语言移植的成功案例长什么样 - -tsgo（TypeScript 编译器的 Go 移植）是这类项目里最成功的一个。它成功的两个条件值得记录，因为它们**对 Orleans 都不成立**： - -1. **有确定性的预言机** —— TypeScript 有一整套 conformance 测试，输入输出都是确定的文本，移植后逐个对比就知道对不对。Orleans 的正确性藏在 258 个 `TestCluster` 测试和 136 个 `Sleep` 里，没有这样的预言机。 -2. **纪律是 port 而不是 rewrite** —— 逐文件、逐函数对照，不重新设计。而 Orleans 的模型本身在漂移（创始人已走、Orleans 10 在转 durable execution），照搬 2014 年的设计是移植一个正在被作者放弃的目标。 - -所以 `gor` 明确**不是 port**。这个判断是本仓库存在形态的根据：重新设计，只继承那个仍然成立的核心想法（用 key 引用、运行时负责激活、按 key 串行）。 +| `testing/synctest` | **Go 1.25 GA** | Yes, but only at the unit-test layer | +| gosim (`jellevandenhooff/gosim`) | 80 stars, **no updates after 2024-12** | Its shape fits best (multiple machines + deterministic goroutine scheduling), but we must not depend on an unmaintained base | +| Antithesis | Commercial product | Yes; the 2025-09 price was **168 thousand dollars per year** | +| porcupine (`anishathalye/porcupine`) | 1,230 stars, active | Yes, but it checks only whether a history is linearizable. It does not control scheduling | + +The public conclusion from the Resonate team is the same: Go cannot control +goroutine scheduling. DST must therefore impose rules on the whole codebase. + +**Conclusion:** No existing DST framework can be relied on. The `sim` package +must be built in-house. Its prerequisite is that the codebase follows those +rules from the first day. This is why DST is an architecture rule in `gor`, +not only a test task ([../design/testing.md](../design/testing.md)). + +## What Successful Cross-Language Ports Need + +tsgo, a Go port of the TypeScript compiler, is one of the most successful +projects of this type. Its two success conditions are useful to record. They +do not map directly to the Orleans runtime-model port: + +1. **A deterministic oracle.** TypeScript has a conformance suite with + deterministic text input and output. A port can compare each result. The + correctness of Orleans is spread across 258 `TestCluster` tests and 136 + tests with `Sleep`. Orleans has no equivalent single oracle. +2. **Port discipline instead of a rewrite.** The project follows files and + functions without redesign. A source-file port needs a stable source + structure. `gor` instead ports the Orleans runtime model and verifies its + behavior with deterministic tests. + +Therefore, `gor` is a Go port of the Orleans runtime model. It adapts the +model to Go instead of copying each Orleans source file. It keeps Grain +References, runtime Activations, serialized Calls, State, and Reminders. diff --git a/research/orleans-internals.md b/research/orleans-internals.md index 9ec178b..6447fcd 100644 --- a/research/orleans-internals.md +++ b/research/orleans-internals.md @@ -1,35 +1,43 @@ -# Orleans 源码实测 +# Orleans Source Measurements -对象：Orleans 10.1，`dotnet/orleans` clone 后直接读源码。测量日期 2026-07-30。 +Scope: Orleans 10.1. We cloned `dotnet/orleans` and read the source code. +Measurement date: 2026-07-30. -## 规模构成 +## Size Breakdown -`src/` 共 **27.4 万行 / 1897 文件**。但这个数字会误导，拆开看： +`src/` has **274,000 lines / 1897 files**. This number can mislead. The table +gives the measured parts: -| 部分 | 行数 | 对 gor 的意义 | +| Part | Lines | Meaning for gor | |---|---|---| -| `src/api/` API 基线快照 | 35,384 | **不是实现代码**，是 API 审批用的签名快照 | -| 序列化 | 30,761 | 不做（不追求版本容忍） | -| 流 / 事务 / 事件溯源 / journaling | 35,247 | 不在范围内 | -| 云厂商 provider | ~26,000 | 换成嵌入式 + Postgres 两个后端 | -| **真正需要重建的核心** | **~26,000** | 见下 | - -核心的构成： - -| 子系统 | 行数 | -|---|---| -| GrainDirectory | 7,136 | -| Catalog（激活生命周期） | 5,817 | -| MembershipService | 5,811 | -| Placement | 5,123 | -| ConsistentRing | 884 | -| Scheduler | 823 | - -**结论：要写的是 2 到 3 万行，不是 24 万行。** 而且 Go 侧还能再省（见 [go-capabilities.md](go-capabilities.md)）。 - -## 单激活仲裁只有约 25 行 - -`src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.Interface.cs` 的 `RegisterCore`——整个「保证同一个 grain 只有一个激活」的仲裁逻辑： +| `src/api/` API baseline snapshot | 35,384 | **Not implementation code**. It is a signature snapshot for API approval. | +| Serialization | 30,761 | Not included. The product does not seek version tolerance. | +| Other out-of-scope subsystems | 35,247 | Out of scope. | +| Cloud provider code | ~26,000 | Not required for Single Silo. | +| **Relevant runtime areas** | **~26,000** | See the scope split below. | + +The relevant runtime areas contain: + +| Subsystem | Lines | Scope for gor | +|---|---:|---| +| Catalog (Activation lifecycle) | 5,817 | Single Silo lifecycle evidence | +| Scheduler | 823 | Single Silo Call serialization evidence | +| GrainDirectory | 7,136 | Future Cluster evidence | +| MembershipService | 5,811 | Future Cluster evidence | +| Placement | 5,123 | Future Cluster evidence | +| ConsistentRing | 884 | Future Cluster evidence | + +These areas total about 26,000 lines. This is not a 0.1.0 scope estimate. +Catalog and Scheduler behavior inform Single Silo. The other areas inform a +future Cluster. The full `src/` count does not measure the work needed for the +`gor` port. Go removes some .NET-specific work (see +[go-capabilities.md](go-capabilities.md)). + +## Single-Activation Arbitration Takes About 25 Lines + +`RegisterCore` in +`src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.Interface.cs` +contains the arbitration logic for one Activation for one Grain: ```csharp private GrainAddress RegisterCore(GrainAddress newAddress, GrainAddress? existingAddress, MembershipVersion currentVersion) @@ -41,7 +49,7 @@ private GrainAddress RegisterCore(GrainAddress newAddress, GrainAddress? existin newAddress = new() { ... MembershipVersion = currentVersion }; existing = newAddress; } - return existing; // 输者拿回权威地址，去那边转发 + return existing; // The loser gets the authoritative address and forwards there. } private bool IsSiloDead(GrainAddress existing) @@ -49,36 +57,55 @@ private bool IsSiloDead(GrainAddress existing) || _owner.ClusterMembershipSnapshot.GetSiloStatus(existing.SiloAddress, existing.MembershipVersion) == SiloStatus.Dead; ``` -对一个内存 dict 做 CAS，三个条件之一成立就写入：没人占、占的正是我预期的那个、占的那个节点已经死了。返回值总是权威地址——赢了拿到自己的，输了拿到对方的然后转发过去。 +The single-threaded partition makes a conditional dictionary update. It writes +when no owner exists, the owner is the expected owner, or the current owner is +dead. The return value is always the authoritative address. The winner gets +its own address. The loser gets the other address and forwards the Call there. -调用点在 `DhtGrainLocator.cs:37`：`_localGrainDirectory.RegisterAsync(address, currentRegistration: previousAddress)`。 +The call site is `DhtGrainLocator.cs:37`: +`_localGrainDirectory.RegisterAsync(address, currentRegistration: previousAddress)`. -**这个发现改变了对项目难度的判断。** 最让人望而生畏的那个保证，实现是单线程 dict 上的一次条件写。 +**This finding changes the project difficulty estimate.** The guarantee uses +one conditional write in a single-threaded dictionary. -## 单激活是明确的弱保证 +## Single Activation Is a Clear Weak Guarantee -Orleans 官方文档明说：默认 grain directory 是**最终一致的**，集群不稳定期间**允许出现重复激活**。推荐的缓解手段是存储层的 ETag 乐观并发。 +The Orleans documentation states that the default Grain Directory is +**eventually consistent**. During cluster instability, **duplicate +Activations are allowed**. The recommended control is optimistic concurrency +with a storage ETag. -这不是文档的免责声明，是设计立场。`gor` 采纳同样立场并在用户可见处写明（[../design/cluster.md](../design/cluster.md)、[../design/persistence.md](../design/persistence.md)）。 +This is a design position, not a documentation note. `gor` adopts the same +position. The design states it in +[cluster.md](../design/cluster.md) and +[persistence.md](../design/persistence.md). -## Membership 不含共识 +## Membership Has No Consensus -在 `src/Orleans.Runtime/MembershipService/` 下 grep `quorum|Quorum|consensus|Consensus|Raft|Paxos`：**零命中**。 +In `src/Orleans.Runtime/MembershipService/`, the source search returned no +matches for `quorum|Quorum|consensus|Consensus|Raft|Paxos`. -它的做法是： +The design uses: -- 一张共享表，用 ETag/CAS 做原子更新（`MembershipTableManager.cs`）。 -- 节点互相探测（`ProbingSiloHealthMonitor.cs:47,93`）。 -- 探测失败投「死亡票」，**票会过期**——`ClusterHealthMonitor.cs:195` 的 `entry.GetFreshVotes(now, options.DeathVoteExpirationTimeout)` 只统计新鲜票。 -- 节点还监控自身健康（`LocalSiloHealthMonitor.cs:168-176`），自己不健康时不该有资格判别人死。 +- One shared table uses ETag/CAS for atomic updates (`MembershipTableManager.cs`). +- Silos probe each other (`ProbingSiloHealthMonitor.cs:47,93`). +- A failed probe casts a "death vote". **The vote expires.** + `entry.GetFreshVotes(now, options.DeathVoteExpirationTimeout)` at + `ClusterHealthMonitor.cs:195` counts only fresh votes. +- Each Silo also monitors its own health + (`LocalSiloHealthMonitor.cs:168-176`). An unhealthy Silo must not decide that + another Silo is dead. -线性一致性完全外包给存储。这意味着 `gor` 也不需要实现共识——需要的是一个支持条件更新的表。 +The Orleans membership design relies on atomic conditional updates in shared +storage. It does not implement a separate consensus system in +`MembershipService`. A future `gor` Cluster needs a table that supports these +conditional updates. -## 调度器 823 行，Go 里不需要 +## The Custom Scheduler Has 823 Lines -`src/Orleans.Runtime/Scheduler/` 全部文件： +All files in `src/Orleans.Runtime/Scheduler/`: -| 文件 | 行数 | +| File | Lines | |---|---| | WorkItemGroup.cs | 336 | | ActivationTaskScheduler.cs | 171 | @@ -87,28 +114,32 @@ Orleans 官方文档明说：默认 grain directory 是**最终一致的**，集 | TaskSchedulerUtils.cs | 41 | | WorkItemBase.cs | 22 | | IWorkItem.cs | 13 | -| **合计** | **823** | - -`WorkItemGroup : IThreadPoolWorkItem, IWorkItemScheduler` 内部是 `lock (_lockObj)` 加一个队列。 - -这 823 行存在的原因是 .NET 需要自定义 `TaskScheduler` 来保证 `await` 之后回到同一逻辑上下文。Go 里 goroutine 本身就是执行上下文，同样语义约 100 行（[../design/scheduling.md](../design/scheduling.md)）。 +| **Total** | **823** | -## 测试套件不能当预言机 +`WorkItemGroup : IThreadPoolWorkItem, IWorkItemScheduler` uses +`lock (_lockObj)` and one queue. -测试 **17.7 万行 / 1168 文件**。其中： +These 823 lines exist because .NET needs a custom `TaskScheduler`. It returns +work to the same logical context after `await`. The `gor` design uses a +goroutine and a channel. It does not need a custom platform scheduler +([scheduling.md](../design/scheduling.md)). -- **258 个文件**依赖 `TestCluster`（进程内起真实集群） -- **136 个文件**含 `Task.Delay` 或 `Thread.Sleep` +## The Test Suite Is Not an Oracle -正确性藏在时序里。这条观察直接导出了 `gor` 的测试策略（[../design/testing.md](../design/testing.md)）：不可能移植这套测试，也不该模仿它。 +The test suite has **177,000 lines / 1168 files**. It includes: -## Orleans 自己的方向 +- **258 files** that depend on `TestCluster` (a real cluster in one process) +- **136 files** that contain `Task.Delay` or `Thread.Sleep` -Orleans 10 新增： +Correctness depends on timing. This observation led to the `gor` testing +strategy ([testing.md](../design/testing.md)). The test suite cannot be ported +as-is. `gor` must verify the same runtime rules with deterministic tests. -- `Orleans.Journaling` —— 8,148 行 -- `Orleans.DurableJobs` —— 5,278 行，即 "Reminders v2"，**仍是 preview/alpha** +## Orleans Reminder Evidence -两个都指向持久化执行，不指向 actor 模型。`DurableJobs` 的存在说明 Orleans 自己也认为 Reminders v1 的设计需要换掉——这是 [../design/scheduling.md](../design/scheduling.md) 里选「表 + 轮询」而非复刻 v1 的依据。 +Orleans 10 adds `Orleans.DurableJobs`. It has 5,278 lines and calls itself +"Reminders v2". It was still preview software at the measurement date. -创始人 Sergey Bykov（领导 Orleans 十余年）现在在 **Temporal Cloud 做架构**，并公开写过不再使用 "actors" 一词。 +This is evidence that the Reminders design is still changing. It supports the +`gor` choice of a table and polling instead of a source-file copy of Reminders +v1 ([scheduling.md](../design/scheduling.md)). diff --git a/research/release-0.1.0-batch-3-mutations.md b/research/release-0.1.0-batch-3-mutations.md new file mode 100644 index 0000000..ae9fdf5 --- /dev/null +++ b/research/release-0.1.0-batch-3-mutations.md @@ -0,0 +1,38 @@ +# Batch 3 Mutation Results + +Date: 2026-08-10 + +## Scope + +These checks cover the critical Activation and Call rules in Batch 3. Each +mutation compiled. Each focused test reached the behavior assertion that owns +the changed rule. + +## Method + +For each check: + +1. Apply one source mutation to the Batch 3 release candidate. +2. Run the exact focused test command. +3. Confirm that the command fails at the intended assertion. +4. Restore the source change before the next check. + +## Results + +| Rule | Source mutation | Focused command | Result | +| --- | --- | --- | --- | +| Activation becomes active after setup. | Remove `act.state = ActivationActive`. | `go test ./internal/runtime -run '^TestActivationStateTransitions$' -count=1` | Exit 1: `activation state = 0, want active`. | +| Active Activation starts deactivation. | Remove `act.state = ActivationDeactivating`. | `go test ./internal/runtime -run '^TestActivationStateTransitions$' -count=1` | Exit 1: `activation state = 1, want deactivating`. | +| Deactivating Activation becomes stopped. | Remove `act.state = ActivationStopped`. | `go test ./internal/runtime -run '^TestActivationStateTransitions$' -count=1` | Exit 1: `activation state = 2, want stopped`. | +| A canceled queued Call does not enter its method. | Remove the context check after mailbox dequeue. | `go test ./internal/mail -run '^TestMailbox_CanceledQueuedCallDoesNotEnter$' -count=1` | Exit 1: `canceled queued call entered its method`. | +| Requested deactivation keeps the Call lane. | Close the Call lane for `ApplicationRequested`. | `go test ./internal/runtime -run '^TestRuntime_RequestedDeactivationPreservesQueuedCallOrder$' -count=1` | Exit 1: `queued Call order = [2 1], want [1 2]`. | +| Ownership loss does not reactivate on the old Silo. | Change the `OwnershipLost` lane result to a local retry. | `go test . -run '^TestRuntime_OwnershipChangeReroutesQueuedCall$' -count=1` | Exit 1: `rerouted Call = ("node-a", ), want (node-b, nil)`. | +| A started method error cannot request ownership reroute. | Treat a method's `mail.ErrClosed` as a closed-lane control result. | `go test ./internal/runtime -run '^TestRuntime_MethodErrorCannotRequestOwnershipReroute$' -count=1` | Exit 1: `started Grain method requested ownership reroute`. | +| Fault wins over a deactivation request in the same Call. | Process the request before the fault. | `go test ./internal/runtime -run '^TestRuntime_FaultWinsOverRequestedDeactivation$' -count=1` | Exit 1: `queued Call error = , want mailbox closed`. | +| Activation uses a Runtime-owned context. | Use the caller context as the Activation root. | `go test . -run '^TestRequestContext_ActivationUsesRuntimeOwnedContext$' -count=1` | Exit 1: `OnActivate context after caller cancellation = context canceled, want nil`. | +| An already canceled shutdown uses only the abrupt stop path. | Remove the early canceled-context branch from `Shutdown`. | `go test . -run '^TestRuntime_StopModeRouting$' -count=1` | Exit 1: `abrupt Shutdown also started Transport.Close`. | + +## Restoration Check + +All source mutations were restored. The Batch 3 handoff records the focused +tests and the full `make ci` result on the restored candidate. diff --git a/research/release-0.1.0-batch-4-mutations.md b/research/release-0.1.0-batch-4-mutations.md new file mode 100644 index 0000000..698d926 --- /dev/null +++ b/research/release-0.1.0-batch-4-mutations.md @@ -0,0 +1,43 @@ +# Batch 4 Mutation Results + +Date: 2026-08-10 + +## Scope + +These checks cover the critical Grain Timer rules in Batch 4. Each listed +mutation compiled. Each focused test reached the behavior assertion that owns +the changed rule. + +## Method + +For each check: + +1. Apply one source mutation to the Batch 4 release candidate. +2. Run the exact focused test command. +3. Confirm that the command fails at the intended assertion. +4. Restore the source change before the next check. + +## Results + +| Rule | Source mutation | Focused command | Result | +| --- | --- | --- | --- | +| A Timer turn waits for mailbox capacity. | Use non-waiting mailbox admission. | `go test ./internal/mail -run '^TestMailbox_CallWaitResultWaitsForCapacity$' -count=1` | Exit 1: `waiting turn: mailbox overloaded`. | +| Stop discards a queued tick. | Do not cancel the queued tick on Stop. | `go test . -run '^TestGrainTimerStopDiscardsQueuedTick$' -count=1` | Exit 1: `stopped queued tick entered callback`. | +| Stop does not cancel a running callback. | Run the callback with the queue context. | `go test . -run '^TestGrainTimerStopDoesNotCancelRunningCallback$' -count=1` | Exit 1: `Stop canceled running callback: context canceled`. | +| Change during a callback sets the next due time. | Ignore the changed due time after callback completion. | `go test . -run '^TestGrainTimerChangeDuringCallbackControlsNextDueTime$' -count=1` | Exit 1: `changed Grain Timer tick was not ready`. | +| A callback panic faults the Activation. | Report the panic without setting the fault result. | `go test . -run '^TestGrainTimerErrorsContinueAndPanicFaultsActivation$/panic_faults$' -count=1` | Exit 1: `Activation after timer panic = 1, want 2`. | +| KeepAlive refreshes idle time after callback completion. | Remove the completion refresh. | `go test . -run '^TestGrainTimerKeepAliveRefreshesAfterBlockedCallbackCompletes$' -count=1` | Exit 1: `Activations four seconds after callback completion = 0, want 1`. | +| Runtime cancellation does not enter OnError. | Report a canceled shutdown callback. | `go test . -run '^TestGrainTimerUsesRuntimeContextAndShutdownCancellationIsSilent$' -count=1` | Exit 1: `Runtime cancellation reached OnError`. | +| An error handler panic cannot skip Timer cleanup. | Remove panic containment around the Timer error handler. | `go test ./internal/runtime -run '^TestGrainTimerErrorHandlerPanicDoesNotBlockCleanup$' -count=1` | Exit 1: `panic: error handler failed`. | +| An ignored State failure faults the Activation. | Ignore the Grain Context discard result after the callback. | `go test . -run '^TestGrainTimerStateFailureFaultsActivation$' -count=1` | Exit 1: `Grain Timer State error event was not ready`. | +| Cancellation after enqueue waits for mailbox disposition. | End the wait when the queue context ends. | `go test ./internal/mail -run '^TestMailbox_CallWaitResultCancellationAfterEnqueueWaitsForDisposition$' -count=1` | Exit 1: `queued Runtime turn returned before mailbox disposition`. | +| Kill stops the Timer owner without waiting for user callback code. | Remove the abrupt-stop signal from the mailbox wait. | `go test ./internal/runtime -run '^TestGrainTimerKillWaitsForOwnerButNotCallback$' -count=1` | Exit 1: `Kill did not stop Runtime infrastructure`. | +| Requested deactivation lets the next Call pass an old queued Timer turn. | Keep waiting for the old Timer mailbox result. | `go test . -run '^TestGrainTimerRequestedDeactivationDoesNotBlockNextCallBehindQueuedTick$' -count=1` | Exit 1: `Call behind requested deactivation was not ready`. | +| A Timer without KeepAlive does not refresh idle time. | Refresh idle time at every Timer callback start. | `go test . -run '^TestGrainTimerWithoutKeepAliveDoesNotPreventIdleDeactivation$' -count=1` | Exit 1: `Activations with a non-keep-alive Grain Timer = 1, want 0`. | +| A one-shot Timer runs once. | Schedule another tick after one hour. | `go test . -run '^TestGrainTimerOneShotRunsOnce$' -count=1` | Exit 1: `one-shot timer repeated at tick 2`. | +| A normal callback error does not stop a repeating Timer. | Stop the Timer after its first callback result. | `go test . -run '^TestGrainTimerErrorsContinueAndPanicFaultsActivation$/error_continues$' -count=1` | Exit 1: `Grain Timer error event was not ready`. | + +## Restoration Check + +All source mutations were restored. The Batch 4 handoff records the focused +tests and the full `make ci` result on the restored candidate. diff --git a/research/release-0.1.0-batch-5-mutations.md b/research/release-0.1.0-batch-5-mutations.md new file mode 100644 index 0000000..53b3210 --- /dev/null +++ b/research/release-0.1.0-batch-5-mutations.md @@ -0,0 +1,48 @@ +# Batch 5 Mutation Results + +Date: 2026-08-10 + +## Scope + +These checks cover the critical State and SQLite rules in Batch 5. Each listed +mutation compiled. Each focused test reached the behavior assertion that owns +the changed rule. + +## Method + +For each check: + +1. Apply one source mutation to the Batch 5 release candidate. +2. Run the exact focused test command. +3. Confirm that the command fails at the intended assertion. +4. Restore the source change before the next check. + +## Results + +| Rule | Source mutation | Focused command | Result | +| --- | --- | --- | --- | +| An Unknown Result discards the Activation. | Do not mark the Activation after a failed Store write. | `go test . -run '^TestStateWriteOutcomesDiscardActivationAndReloadStore$/lost_reply$' -count=1` | Exit 1: `Balance after failed reply = 0, want applied Store value 10`. | +| A failed Set does not commit its candidate scalar value. | Assign the candidate value before the Store write. | `go test . -run '^TestState_WriteErrorLeavesValueAndMarksBinder$' -count=1` | Exit 1: `value after write error = 2, want 1`. | +| A canceled write has a stable persistence code. | Return the context error before persistence error handling. | `go test . -run '^TestStateWriteOutcomesDiscardActivationAndReloadStore$/cancel$' -count=1` | Exit 1: `Deposit Code = ("", false), want ("gor.persistence_failed", true)`. | +| State declarations freeze before the Store read. | Freeze declarations after State load. | `go test . -run '^TestStateDeclarationsFreezeBeforeLoad$' -count=1` | Exit 1: `State declarations were open when the Store read started`. | +| An ignored activation-hook State failure prevents method entry. | Publish the Activation when the hook returns `nil`. | `go test . -run '^TestOnActivateStateFailurePreventsMethodEntry$' -count=1` | Exit 1: `method entered after OnActivate ignored a State failure`. | +| A deactivation-hook State failure reaches the error sink. | Do not inspect State failures after the hook. | `go test . -run '^TestOnDeactivateStateFailureIsReported$' -count=1` | Exit 1: `OnDeactivate State failure was not reported`. | +| Deactivation does not report an old Call error again. | Join the Activation discard error without a failure snapshot. | `go test . -run '^TestOnDeactivateDoesNotReportPriorStateFailureAgain$' -count=1` | Exit 1: `prior Call State failure was reported again`. | +| A named State error names the State. | Remove the State name from persistence diagnostics. | `go test . -run '^TestStateErrorsIncludePersistenceContext/write$' -count=1` | Exit 1: the error did not contain `balance`. | +| A confirmed empty record is malformed. | Treat every zero-length record as absent. | `go test . -run '^TestStateErrorsIncludePersistenceContext/empty_confirmed_record$' -count=1` | Exit 1: `persistence error = nil`. | +| A `null` State record is malformed. | Accept a decoded nil JSON object. | `go test . -run '^TestStateErrorsIncludePersistenceContext/null_confirmed_record$' -count=1` | Exit 1: `persistence error = nil`. | +| One State failure appears once in the Call result. | Always join the returned error with the discard error. | `go test . -run '^TestStateWriteOutcomesDiscardActivationAndReloadStore$/lost_reply$' -count=1` | Exit 1: the result contained two State write diagnostics. | +| A returned deactivation State error appears once. | Join the returned error with the same recorded State error. | `go test . -run '^TestOnDeactivateStateFailureIsReported$/returned_State_error$' -count=1` | Exit 1: the event contained two State write diagnostics. | +| The State read pool has a fixed open-connection limit. | Remove its maximum open-connection setting. | `go test ./store -run '^TestOpenSQLite_ConfiguresBoundedConnectionPools$' -count=1` | Exit 1: `State read MaxOpenConnections = 0, want 16`. | +| Store readiness rejects a failed SQLite integrity check. | Ignore non-`ok` integrity results. | `go test ./store -run '^TestSQLiteCheckRejectsIntegrityFailure$' -count=1` | Exit 1: both damaged databases returned a `nil` Check error. | +| Migration checks copied State before it deletes old State. | Skip the target integrity check before the old table drop. | `go test ./store -run '^TestMigrate_TargetIntegrityFailureLeavesOldDatabaseIntact$' -count=1` | Exit 1: `OpenSQLite error = , want integrity failure`. | +| Store rejects nil record data. | Let Memory Write store nil data. | `go test ./store -run '^TestStoreContract/memory/nil_data_is_rejected$' -count=1` | Exit 1: `Write error = , want ErrInvalidRecordData`. | +| Store rejects empty record data. | Reject nil data but accept a non-nil empty slice. | `go test ./store -run '^TestStoreContract/memory/empty_data_is_rejected$' -count=1` | Exit 1: the empty Write succeeded. | +| Concurrent writes use one compare-and-swap result. | Skip the Memory ETag comparison. | `go test ./store -run '^TestStoreContract/memory/concurrent_compare_and_swap$' -count=1` | Exit 1: both writes succeeded. | +| A Store does not expose its mutable record bytes. | Return the Memory record data without a copy. | `go test ./store -run '^TestStoreContract/memory/data_is_copied$' -count=1` | Exit 1: stored data changed to `Yriginal`. | + +## Restoration Check + +All source mutations were restored. The focused State, conformance, Store, +SQLite recovery, and SQLite integrity tests pass on the restored candidate. +The Batch 5 handoff records the full `make ci` result. diff --git a/research/release-0.1.0-batch-6-mutations.md b/research/release-0.1.0-batch-6-mutations.md new file mode 100644 index 0000000..49a9017 --- /dev/null +++ b/research/release-0.1.0-batch-6-mutations.md @@ -0,0 +1,36 @@ +# Batch 6 Mutation Results + +Date: 2026-08-10 + +## Scope + +These checks cover the critical Reminder rules in Batch 6. Each mutation +compiled. Each focused test failed at the assertion that owns the changed +rule. + +## Method + +For each check: + +1. Apply one source mutation to the Batch 6 release candidate. +2. Run the exact focused test command. +3. Confirm that the command fails at the intended assertion. +4. Restore the source change before the next check. + +## Results + +| Rule | Source mutation | Focused command | Result | +| --- | --- | --- | --- | +| Invalid Reminder input does not enter Store I/O. | Accept an empty Reminder name in `Set`. | `go test . -run '^TestSchedule_ValidatesBeforeReminderStoreIO$' -count=1` | Exit 1: `Set accepted invalid Reminder ""`. | +| An unknown stored GrainType or method is not claimed. | Replace dispatch validation errors with an untyped Call. | `go test . -run '^TestSchedule_UnknownStoredRowsAreReportedBeforeClaim$' -count=1` | Exit 1: both cases produced `ReminderInvocation` instead of `ReminderDispatch`. | +| Worker capacity bounds Claim work. | Add one slot above the configured worker limit. | `go test ./internal/timer -run '^TestPoller_WorkerCapacityBoundsClaims$' -count=1` | Exit 1: `Claim calls while workers are full = 3, want 2`. | +| `CurrentTickTime` is the actual delivery start. | Use the stored due time as `CurrentTickTime`. | `go test ./internal/timer -run '^TestPoller_PassesPeriodicTickStatus$' -count=1` | Exit 1: the current tick was the old due time, not the post-Claim Clock time. | +| A Store cannot return more rows than the page limit. | Allow one row above the configured page limit. | `go test ./internal/timer -run '^TestPoller_RejectsRowsAbovePageLimitBeforeClaim$' -count=1` | Exit 1: `Claim calls = 2, want none for an oversized page`. | +| A cursor must move forward. | Accept a row that is equal to the prior cursor. | `go test ./internal/timer -run '^TestPoller_ReportsAndStopsOnCursorThatDoesNotAdvance$' -count=1` | Exit 1: `ListDue calls = 3, want two pages before cursor rejection`. | +| SQLite has the Reminder due index. | Do not create `schedule_due_idx`. | `go test ./store -run '^TestSQLiteStore_CheckDoesNotChangeData$' -count=1` | Exit 1: Store Check reported that the index had no columns. | + +## Restoration Check + +All source mutations were restored. The focused Reminder, poller, and SQLite +tests pass on the restored candidate. The Batch 6 handoff records the full +`make ci` result. diff --git a/research/release-0.1.0-batch-7-mutations.md b/research/release-0.1.0-batch-7-mutations.md new file mode 100644 index 0000000..80a8f11 --- /dev/null +++ b/research/release-0.1.0-batch-7-mutations.md @@ -0,0 +1,24 @@ +# 0.1.0 Batch 7 Mutation Checks + +Each check made one temporary change. The named test had to fail. The change +was then removed before the next check. + +| Contract | Temporary change | Command | Result | +| --- | --- | --- | --- | +| A variadic Grain method is invalid. | Remove the `Signature.Variadic` rejection. | `go test ./internal/codegen -run '^TestLoad_ContractErrorsHaveSourcePosition$/^variadic_method$' -count=1` | Exit 1. Loader accepted the invalid contract. | +| Runtime rejects a generated-code version mismatch before install. | Disable the version comparison in `InstallType`. | `go test . -run '^TestInstallType_RejectsGeneratedCodeVersionBeforeMutation$' -count=1` | Exit 1. The mismatch returned nil. | +| A failed generation keeps the old file. | Ignore the error from the temporary-file writer. | `go test ./cmd/gorgen -run '^TestWriteFileAtomically_PopulateFailureKeepsOldFile$' -count=1` | Exit 1. The interrupted write returned nil. | +| `-check` rejects stale output. | Disable the generated-byte comparison. | `go test ./cmd/gorgen -run '^TestCheckGenerated_DoesNotChangeOutput$' -count=1` | Exit 1. Stale output passed. | +| `DeactivationReason` is owned by the public `gor` package. | Make it an alias of the internal Runtime type. | `go test . -run '^TestPublicBoundary_OwnsPublicContractTypes$' -count=1` | Exit 1. Reflection reported the internal package path. | +| Generated method names keep Grain and method boundaries. | Join the two names without length boundaries. | `go test ./internal/codegen -run '^TestRender_UsesStableNamesForValidContracts$' -count=1` | Exit 1. `A.BC` and `AB.C` produced the same request and reply names. | +| Import aliases do not hide predeclared Go names. | Stop reserving the predeclared names. | `go test -tags gen ./cmd/gorgen -run '^TestGenerateCollisionFixtureBuilds$' -count=1` | Exit 1. The generated `error` import hid the built-in error type. | +| Import aliases are valid Go import names. | Accept a keyword as an alias candidate. | `go test -tags gen ./cmd/gorgen -run '^TestGenerateCollisionFixtureBuilds$' -count=1` | Exit 1. The generated source had an invalid import name. | +| An import alias cannot be `init`. | Accept `init` as an alias candidate. | `go test ./internal/codegen -run '^TestAliasCandidateRejectsInvalidImportNames$/^init$' -count=1` | Exit 1. The allocator returned `init`. | +| Anonymous contract interfaces cannot embed a private type. | Stop checking embedded interface types. | `go test ./internal/codegen -run '^TestLoad_ContractErrorsHaveSourcePosition$/^anonymous_unexported_embedded_type$' -count=1` | Exit 1. The loader accepted the invalid contract. | +| A Grain interface must be in an importable package. | Accept a marked interface in `package main`. | `go test ./internal/codegen -run '^TestLoad_ContractErrorsHaveSourcePosition$/^main_package$' -count=1` | Exit 1. The loader accepted the invalid package. | + +After all changes were removed, this command passed with exit 0: + +```bash +go test ./internal/codegen ./cmd/gorgen . -count=1 +``` diff --git a/research/release-0.1.0-batch-8a-mutations.md b/research/release-0.1.0-batch-8a-mutations.md new file mode 100644 index 0000000..edaa239 --- /dev/null +++ b/research/release-0.1.0-batch-8a-mutations.md @@ -0,0 +1,14 @@ +# 0.1.0 Batch 8a Mutation Checks + +Each check made one temporary source change. The named test had to fail. The +change was removed before the next check. + +| Contract | Temporary change | Focused command | Result | +| --- | --- | --- | --- | +| A full limit rejects before a new lane. | Make the full-slot branch return nil. | `go test ./internal/runtime -run '^TestRuntime_ActivationLimitRejectsBeforeLaneAllocation$' -count=1` | Exit 1. The test observed extra activation work instead of the limit result. | +| A failed start releases its reservation. | Make `releaseActivationLocked` return without changing the reservation. | `go test ./internal/runtime -run '^TestRuntime_ActivationLimitReleasesFailedStart$' -count=1` | Exit 1. The reservation stayed at one. | +| Deactivation keeps its reservation until completion. | Release the slot when deactivation starts and remove the release at completion. | `go test ./internal/runtime -run '^TestRuntime_ActivationLimitKeepsActiveGrainAvailable$' -count=1` | Exit 1. A second Grain started while `OnDeactivate` was blocked. | +| Activation admission uses the public overload code. | Remove the internal Activation-limit mapping from `publicError`. | `go test . -run 'TestRuntime_ActivationLimitUsesPublicOverload|TestPublicErrorMapsActivationLimit' -count=1` | Exit 1. The internal error crossed the public boundary without `gor.overloaded`. | +| A canceled Call keeps its cancellation result. | Remove the canceled-context check before slot reservation. | `go test ./internal/runtime -run '^TestRuntime_ActivationLimitPreservesCanceledCall$' -count=1` | Exit 1. The canceled Call returned the Activation-limit error. | + +After every temporary change was removed, the focused runtime tests passed. diff --git a/research/release-0.1.0-batch-8b-mutations.md b/research/release-0.1.0-batch-8b-mutations.md new file mode 100644 index 0000000..ed67afe --- /dev/null +++ b/research/release-0.1.0-batch-8b-mutations.md @@ -0,0 +1,22 @@ +# 0.1.0 Batch 8b Mutation Checks + +Each check made one temporary source change. The named test had to fail. The +change was removed before the next check. + +| Contract | Temporary change | Focused command | Result | +| --- | --- | --- | --- | +| Reminder workers do not exceed the configured limit. | Add one slot to the worker channel. | `go test ./internal/timer -run '^TestPoller_LargeBacklogKeepsPagesAndWorkersBounded$' -count=1` | Exit 1. Six Calls started when the limit was five. | +| Reminder Store requests use the configured page limit. | Add one to the limit in `ListDue`. | `go test ./internal/timer -run '^TestPoller_LargeBacklogKeepsPagesAndWorkersBounded$' -count=1` | Exit 1. The oversized page stopped all delivery. | +| Idle eviction closes every Activation mailbox. | Do not close victim mailboxes after idle eviction. | `go test ./internal/runtime -run '^TestRuntime_ActivationChurnReleasesOwnedResources$' -count=1` | Exit 1. The test found a mailbox that was still running. | +| Deactivation releases every Activation reservation. | Do not release the reservation in `finishDeactivation`. | `go test ./internal/runtime -run '^TestRuntime_ActivationChurnReleasesOwnedResources$' -count=1` | Exit 1. Sixteen reservations remained after the first cycle. | +| Grain Timer cleanup removes every Runtime Timer record. | Do not delete the Grain Timer from the Runtime set. | `go test ./internal/runtime -run '^TestRuntime_ActivationChurnReleasesOwnedResources$' -count=1` | Exit 1. Sixteen Grain Timer records remained after the first cycle. | + +After all temporary changes were removed, both focused tests passed. Their race +runs also passed. + +## Verification + +The focused tests each passed 20 consecutive runs. The full Runtime and poller +test packages passed. The root and Store test packages compiled for Linux +amd64, macOS arm64, and Windows amd64. A compile-only run of all packages also +passed for macOS arm64 and Windows amd64. `make ci` passed with exit 0. diff --git a/research/release-0.1.0-batch-8c-mutations.md b/research/release-0.1.0-batch-8c-mutations.md new file mode 100644 index 0000000..ae11ee1 --- /dev/null +++ b/research/release-0.1.0-batch-8c-mutations.md @@ -0,0 +1,43 @@ +# 0.1.0 Batch 8c Gate Checks + +Each mutation made one temporary source change. The named test had to fail. +The change was removed before the next check. + +| Contract | Temporary change | Focused command | Result | +| --- | --- | --- | --- | +| A test target must start at least one test. | Remove the zero-test rejection from `testcheck`. | `go test ./internal/testcheck -run '^TestRunRejectsZeroSelection$' -count=1` | Exit 1. The child command started zero tests but returned success. | +| Request Context must reject more than 4096 wire bytes before JSON decode. | Remove the wire-size check before `json.Unmarshal`. | `go test . -run '^TestDecodeRequestContextRejectsWireSizeBeforeJSONDecode$' -count=1` | Exit 1. The code returned a JSON syntax error instead of the size error. | + +Both focused tests passed after the changes were removed. + +## Fuzz proof + +The default ten-second run passed for each target on Linux amd64 with Go +1.26.5. + +| Target | Executions | Result | +| --- | ---: | --- | +| `FuzzReadFrame` | 1,508,765 | Exit 0. | +| `FuzzDecodeRequestContext` | 200,792 | Exit 0. | +| `FuzzParseGrainMarker` | 2,185,654 | Exit 0. | + +Each target reported one started fuzz test. The normal seed-corpus runs also +passed in the default test gate. + +## Verification + +`make ci` passed with exit 0 on Linux amd64 with Go 1.26.5. It ran the format, +module tidy, static analysis, unit, race, simulation, generator, generated +file, network, resource, and fuzz checks. Every test target reported nonzero +test work. + +The new gate commands and a short Transport fuzz run also passed with Go +1.25.0. Staticcheck reported `v0.7.0`. govulncheck reported `v1.6.0`, and the +full scan found no known vulnerability. + +Actionlint `v1.7.12` accepted the CI workflow. The default, simulation, +generator, and network test sets compiled for macOS arm64 and Windows amd64. +These compile-only checks returned exit 0. + +The hosted Linux, macOS, and Windows jobs were not run. The workflow contains +all three jobs, but a local cross-compile is not hosted test proof. diff --git a/research/release-0.1.0-batch-8d-mutations.md b/research/release-0.1.0-batch-8d-mutations.md new file mode 100644 index 0000000..b84a0d2 --- /dev/null +++ b/research/release-0.1.0-batch-8d-mutations.md @@ -0,0 +1,42 @@ +# 0.1.0 Batch 8d External Proof Checks + +Each mutation made one temporary source change. `make external` had to fail. +The source change was removed before the next check. + +| Contract | Temporary change | Result | +| --- | --- | --- | +| Recovery must run in a second process. | Reuse the `prepare` output instead of starting `recover`. | Exit 1. The graceful restart case found no recovery record in the output. | +| The parent must stop the process after Claim. | Skip `Process.Kill`. | Exit 1. The parent context expired and the test rejected a timeout as kill proof. | +| Recovery must verify the complete receipt. | Skip the `AppliedRecord` comparison. | Exit 1. The receipt mismatch process returned success. | + +The first kill mutation escaped the first test form. The child process reached +its own 15-second recovery timeout and returned a nonzero exit. The test +mistook that exit for parent kill proof. + +The repaired barrier process uses a 24-hour internal recovery timeout. The +parent test still has a 20-second context. Without `Process.Kill`, that context +stops the process and the test fails. The repeated mutation then returned exit +1 at the intended assertion. + +All temporary changes were removed. The restored `make external` run passed +with exit 0 on Linux amd64 with Go 1.26.5. The Quick Start package reported +one started test. The external restart package reported five started tests +and used empty module and build caches. + +The same target builds the normal device shadow command, starts it as a +separate process, and verifies the documented POST and GET requests. It waits +for the listener address from process output. It does not use a fixed wait or +poll loop. + +The release-tagged Quick Start and restart test packages also compiled for +macOS arm64 and Windows amd64. These compile checks do not prove hosted +execution. + +The common test set injects a file information reader to verify symlink +rejection without platform permissions. Unix systems also run the real +filesystem symlink tests. The complete release-tagged conformance package +passed on Linux. + +The complete `make ci` gate passed with exit 0 after these changes. It ran the +default, race, simulation, generator, TCP, resource, and fuzz test groups. Each +of the three fuzz targets ran for 10 seconds and reported a started test. diff --git a/research/release-0.1.0-batch-8e-mutations.md b/research/release-0.1.0-batch-8e-mutations.md new file mode 100644 index 0000000..7e8a336 --- /dev/null +++ b/research/release-0.1.0-batch-8e-mutations.md @@ -0,0 +1,40 @@ +# 0.1.0 Batch 8e Tagged Module Checks + +Each mutation made one temporary source change. The focused configuration test +had to fail. The source change was removed before the next check. + +| Contract | Temporary change | Result | +| --- | --- | --- | +| Tagged proof must use exact version `v0.1.0`. | Change the fixed tagged version to `v0.1.1`. | Exit 1. The test required `v0.1.0`. | +| Tagged proof must reject a replacement. | Remove the resolved `Replace` check. | Exit 1. The tagged replacement case returned no error. | +| Tagged source must be in the empty module cache. | Remove the module cache path check. | Exit 1. The outside-cache case returned no error. | +| Local module settings must not select the source. | Set the fixed proxy to a local file path. | Exit 1. The environment test required the public source path. | + +All temporary changes were removed. The focused configuration test then passed +with 14 started tests. + +## Candidate and tag checks + +`make external` passed with exit 0 on Linux amd64 with Go 1.26.5. The Quick +Start package reported one started test. The conformance package reported 19 +started tests. It built from a clean consumer module with empty module and +build caches. Separate processes proved normal restart and stop-after-Claim +recovery. + +`make external-tagged` returned exit 2 before the tag exists. The configuration +test passed first. The restart test then requested exact version `v0.1.0` +without a replacement and failed with `unknown revision v0.1.0`. This is a +diagnostic result. It is not the final tagged-module proof. + +The release-tagged Quick Start and conformance test packages compiled for +macOS arm64 and Windows amd64. These compile checks do not prove hosted +execution. + +`make ci` passed with exit 0. The default and race lanes each reported 603 +started tests. Simulation reported 19, generation reported 19, Transport +network tests reported 45, and example network tests reported 43. Both +resource checks reported one started test. Each fuzz target ran for 10 seconds +and reported one started test. + +The hosted platform jobs and the successful exact tagged-module run remain +open. Batch 8e must stay pending until both proofs use the final commit. diff --git a/research/release-0.1.0-issue-111-mutation.md b/research/release-0.1.0-issue-111-mutation.md new file mode 100644 index 0000000..cad430a --- /dev/null +++ b/research/release-0.1.0-issue-111-mutation.md @@ -0,0 +1,68 @@ +# 0.1.0 Issue 111 Release Command Check + +The baseline Makefile used a normal `GO_TEST` assignment. These dry-run +commands rendered `true` instead of the internal `testcheck` command: + +- `make -n GO_TEST=true ci`; +- `make -n GO_TEST=true external`; +- `make -n GO_TEST=true external-tagged`. + +The first fix protected `GO_TEST`, but an independent review found a second +bypass. `make -n MAKE=true ci` replaced each recursive Make call with `true`. +Only the direct race command remained. + +The final fix uses literal `testcheck` commands and Make prerequisites. It does +not use a replaceable test command variable or recursive Make command. A +release-tagged contract test compares each complete dry-run plan with these +caller settings: + +- `GO_TEST=true`; +- `MAKE=true`; +- both settings together; +- `-j8` with and without both settings. + +The test also verifies every required command in the specified order. + +## Mutation proof + +The first mutation added a normal `GO_TEST` assignment and used it for the +default test target. This command then returned exit 1: + +```text +go test -tags release ./internal/testcheck \ + -run '^TestMakefile_ReleaseCommandsKeepTestcheck$' -count=1 +``` + +The `ci` subtest reached the command-plan assertion. It found `true ./...` +where the normal plan used `go run ./internal/testcheck ./...`. + +The mutation was removed. The focused command then returned exit 0. +`make -e GO_TEST=false MAKE=false release-command-check` also returned exit 0 +and reported four started tests. The rendered command still used +`internal/testcheck`. + +A second mutation removed the `release-command-check` prerequisite from +`external-tagged`. The focused command returned exit 1. Only the +`external-tagged` subtest failed. It reported that this entry point did not run +the release command check. The mutation was removed, and the focused command +returned exit 0. + +## Verification + +The independent review first found the `MAKE=true` bypass, incomplete command +proof, and GNU-only syntax. The final review found no P0 through P3 issue. + +The test cache was empty before each complete gate. `make ci` returned exit 0 +on Linux amd64 with Go 1.26.5. The new command check reported four started +tests. The other lanes reported these test counts: + +- 604 default tests; +- 19 simulation tests; +- 19 generator tests; +- 45 Transport network tests; +- 43 example network tests; +- two focused resource tests; +- three focused fuzz tests that ran for 10 seconds each. + +`make external` also returned exit 0. It reported four command contract tests, +one Quick Start process test, and 19 external module and restart tests. diff --git a/research/release-0.1.0-issue-112-mutations.md b/research/release-0.1.0-issue-112-mutations.md new file mode 100644 index 0000000..d388807 --- /dev/null +++ b/research/release-0.1.0-issue-112-mutations.md @@ -0,0 +1,121 @@ +# 0.1.0 Issue 112 State Recovery Evidence + +## Baseline + +The old recovery path completed a pending Application action without a State +step. This command reproduced the gap: + +```bash +go test ./examples/shadow \ + -run '^TestConformance_StateFailureKeepsActionPendingUntilRecovery$' \ + -count=1 +``` + +The test failed after restart. The receipt existed, but the Device State was +the zero value instead of `temperature=26`. + +## Recovery contract + +`Device.ApplyPending` now uses this order: + +1. Return success when the receipt already exists. +2. Read the pending action and check its Device GrainKey. +3. Confirm the requested State. Write it only when needed. +4. Complete the pending action and create the receipt. + +The pending action remains when the State step fails. A new Activation reads +Confirmed State from the Store. It can then repeat the action safely. + +The ApplicationStore returns pending actions in save order. This keeps the +Business Action order when one Device has more than one pending action. + +The design follows the Orleans State boundary. A State write is an explicit +Grain action. A persistence conflict ends the current Activation. Application +recovery remains Application code. + +## Design review + +An independent design review found three risks in the first candidate: + +- the requested State fields were not clear; +- pending actions used ActionID order instead of save order; +- a repeated old receipt could overwrite later State. + +The final design defines the requested fields. It keeps pending actions in +save order. It also returns before a State write when the receipt exists. + +A later self-review found that the receipt fast path did not check DeviceKey. +The wrong Device could therefore accept another Device's completed ActionID. +A regression test reproduced the false success. The pending and applied paths +now use the same target check. + +The independent code review found two P1 defects. A repeated completed +`ReportAction` could restore old State. The SQLite save-order change also +required a new column that an existing database did not have. The first fix +checks for the receipt before the State write. The second fix keeps the +existing schema and orders pending SQLite rows by their insertion row ID. + +The independent re-review passed after these fixes. It also confirmed that +pending actions and applied receipts use the same DeviceKey check. + +## Mutation checks + +### Receipt before State + +A mutation called `CompletePending` before the State write. This test failed: + +```bash +go test ./examples/shadow \ + -run '^TestConformance_RecoveryStateFailureDoesNotCompleteAction$' \ + -count=1 +``` + +The test found an applied receipt after the State write failed. Restoring the +State-first order made the test pass. + +### Wrong pending order + +A mutation returned pending actions in reverse save order. This test failed: + +```bash +go test ./examples/shadow \ + -run '^TestConformance_RecoveryUsesPendingSaveOrder$' \ + -count=1 +``` + +The final Device State was `temperature=28`. The correct final value was the +second saved value, `temperature=29`. Restoring save order made the test pass. + +### Repeat a confirmed State write + +A mutation forced a State write when the requested value was already +confirmed. This test failed: + +```bash +go test ./examples/shadow \ + -run '^TestConformance_StateUnknownResultRepeatsAfterRestart$' \ + -count=1 +``` + +The mutation made two Device State writes. The restored implementation made +one committed attempt. + +## Restored candidate + +All three focused mutation tests pass after source restoration. + +The first external run found a diagnostic-order regression. State validation +ran before the existing missing-receipt and receipt-mismatch checks. The +command now keeps those actionable errors and validates State after the +receipt and pending checks. The complete external gate then passed. + +The final candidate passed these clean-cache gates: + +- `make ci`: 4 release command tests, 609 default tests, 19 simulation tests, + 19 generator tests, 45 transport network tests, 48 example network tests, + 2 resource tests, and 3 focused fuzz targets; +- `make external`: 4 release command tests, 1 Quick Start test, and 19 + external module and restart tests; +- `go test -race ./examples/shadow/... -count=1`: pass. + +The same gates run again on the final commit. diff --git a/research/release-0.1.0-issue-113-mutations.md b/research/release-0.1.0-issue-113-mutations.md new file mode 100644 index 0000000..dad9054 --- /dev/null +++ b/research/release-0.1.0-issue-113-mutations.md @@ -0,0 +1,179 @@ +# 0.1.0 Issue 113 Reminder Terminal Evidence + +## Baseline + +The old poller reported an Invalid Reminder on every poll. It did not change +the Reminder row. This command reproduced the problem: + +```bash +go test ./internal/timer \ + -run '^TestPoller_InvalidReminderIsTerminalAcrossPolls$' \ + -count=1 +``` + +The test ran three poll intervals. It received three dispatch failures instead +of one. + +## Design + +The Runtime gives an Invalid Reminder a Terminal Result with a zero-time +Claim. The Claim uses the row ETag and removes the unchanged row. Only the CAS +winner reports `ReminderDispatch`. + +The first deletion design reused ETag 1 after Delete and Put. An old poller +could then change a new Reminder. The final design gives each Put a Store-wide +ETag that is newer than all prior Reminder ETags in that Store. Delete does not +reset this version. + +Memory and the simulation keep the version in memory. SQLite keeps one +`schedule_version` row. SQLite updates this row and the Reminder row in one +local transaction. Open sets the version to at least the largest old Reminder +ETag. + +This design follows the Orleans ETag removal rule. It does not add another +public Store operation. It also does not keep removed Reminder rows. + +## Failure boundary + +Only the terminal Claim winner reports `ReminderDispatch`. A stale result is +normal contention and produces no error. + +A Store error from the terminal Claim reports `ReminderTerminal`. The poller +does not also report the dispatch error because it does not know if the Store +removed the row. If the delete happened before an Unknown Result, a restart +does not report the same row again. + +The `ReminderDispatch` report after a successful Terminal Result is +at-most-once. A process can stop after the Store commits the result and before +`OnError` runs. The Runtime does not retry this report. A `ReminderTerminal` +Store error can occur again when the row remains due. + +## Independent review + +The design review required one ETag CAS winner, bounded worker use, restart +proof, and one error source for the terminal Store step. + +The code review found that Delete and Put reused ETag 1. This also affected a +normal Reminder Claim. The Store now keeps one version across Delete and +restart. The review also found a missing simulation method and an incomplete +SQLite index check. The smaller final design removed the extra method. The +index check now rejects partial indexes. The final review also required direct +Set and Cancel races, two SQLite handles, and a failed migration rollback. + +## Mutation checks + +### Report without a CAS win + +A mutation reported the dispatch failure when the terminal CAS returned +false. This test failed: + +```bash +go test ./internal/timer \ + -run '^TestPoller_(StaleInvalidReminderDoesNotReport|TwoPollersReportOneInvalidReminderOnce)$' \ + -count=1 +``` + +The stale row produced one error. Restoring the winner check removed the stale +error. + +### Reuse an ETag after Delete + +A memory Store mutation returned ETag 1 for every Reminder change. This test +failed: + +```bash +go test ./store \ + -run '^TestMemoryReminderStore/DeleteThenPutDoesNotReuseETag$' \ + -count=1 +``` + +The recreated Reminder had ETag 1 instead of a newer value. + +A SQLite mutation also reset the global Reminder version to 1. This restart +test failed: + +```bash +go test ./store \ + -run '^TestSQLiteReminderStore_DoesNotReuseETagAfterReopen$' \ + -count=1 +``` + +Restoring both Store versions made an old Claim lose after Delete and Put. + +### Accept a partial due index + +A mutation ignored the partial-index flag during `Check`. This test failed: + +```bash +go test ./store \ + -run '^TestSQLiteStore_CheckRejectsPartialReminderDueIndex$' \ + -count=1 +``` + +`Check` returned no error for an index that cannot serve all due rows. +Restoring the exact index check rejected it. + +### Exceed the terminal worker limit + +A mutation made the worker channel three times larger than the configured +limit. This test failed: + +```bash +go test ./internal/timer \ + -run '^TestPoller_WorkerCapacityBoundsTerminalWrites$' \ + -count=1 +``` + +Five terminal Claims started with a configured limit of two. Restoring the +fixed channel capacity limited active terminal Claims to two. + +## Focused verification + +The focused poller, Store, Runtime, conformance, simulation build, and race +tests cover: + +- more than one poll interval; +- stale CAS and two-poller contention; +- Set and Cancel changes that race with a terminal Claim; +- a Put and terminal Claim from two SQLite handles; +- the terminal worker limit; +- Store failure and Unknown Result; +- ETag reuse after Delete, Put, and restart; +- old SQLite schema migration and schema checks; +- rollback and retry after a failed Reminder schema migration; +- Runtime and SQLite restart; +- GrainId, Reminder name, method, and error reporting; +- valid one-shot and periodic Reminder behavior; +- conformance Application use through the public Runtime API. + +## Final review + +The final independent review found no release-blocking issue. It confirmed the +report boundary, the Set and Cancel races, the two-handle SQLite race, and the +migration rollback proof. + +## Full verification + +This command passed from an empty test cache: + +```bash +go clean -testcache +make ci +``` + +The gate reported 4 release command tests, 631 normal tests, 631 race tests, +20 simulation tests, 19 generated-code tests, 45 network tests, 49 conformance +tests, two resource tests, and three 10-second fuzz tests. + +This public consumer gate also passed from an empty test cache: + +```bash +go clean -testcache +make external +``` + +It reported 4 release command tests, 1 clean Quick Start test, and 19 external +module and process-restart tests. + +The deterministic Set and Cancel race test passed 100 consecutive runs. The +two-handle SQLite race test passed 10 consecutive runs. diff --git a/research/release-0.1.0-issue-114-mutations.md b/research/release-0.1.0-issue-114-mutations.md new file mode 100644 index 0000000..896d417 --- /dev/null +++ b/research/release-0.1.0-issue-114-mutations.md @@ -0,0 +1,128 @@ +# 0.1.0 Issue 114 State Failure Lifecycle Evidence + +## Scope + +Issue 114 completes direct proof for two existing State rules. It does not add +a new State API or a Reminder-specific failure path. + +The proof covers these Store result boundaries: + +- A write fails before commit after the Application changes a mutable map from + `State.Get`. +- A Reminder State write commits, but its reply is lost. + +Both failures make the current Activation unusable. The next Call creates a +new Activation and reads the Confirmed State from the Store. + +## Design + +`State.Get` returns the in-memory value without a copy. The Runtime cannot undo +a map or slice change that the Application makes through that value. A failed +`Set` does not change the local ETag or presence mark. It marks the Activation +for discard. + +Calls and Reminders use the same generated dispatch wrapper. The wrapper turns +the State discard marker into `runtime.Discard`. The execution Runtime then +faults only that GrainId's Activation. It does not need a Reminder-specific +State wrapper. + +Returning a deep copy from `Get` was rejected. It would change the public +alias contract. Adding a second discard path for Reminders was also rejected. +It would duplicate the existing dispatch rule. + +## Behavior Tests + +`TestStateMutableAliasFailureCannotRollbackAndNextActivationReloads` proves: + +- a real Runtime Call mutates a map returned by `State.Get`; +- a failed pre-commit write cannot roll back a map alias; +- the State presence mark and local ETag stay confirmed; +- the failed Activation has a discard marker; +- the Store JSON and ETag do not advance; +- the next Call creates a new Activation and reads the older Confirmed State. + +`TestSchedule_StateFailureFaultsOnlyReminderActivationAndReloadsStore` proves: + +- a Reminder State write can commit before its error result; +- `OnError` receives one `ReminderInvocation` with the State failure; +- another GrainId keeps its current Activation; +- the failed Grain gets a new Activation on its next Call; +- the new Activation reads the committed value from the Store. + +The tests use an injected Store result, a fake clock, and `synctest`. They do +not use a sleep, retry, or wider timeout. + +## Mutation Checks + +### Copy the value returned by Get + +A mutation JSON-copied the value in `State.Get`. This command failed: + +```bash +go test . \ + -run '^TestStateMutableAliasFailureCannotRollbackAndNextActivationReloads$' \ + -count=1 -v +``` + +The test found local value 1 instead of the required alias mutation 2. The +command exited 1. Restoring the direct return made the test pass. + +### Do not return runtime.Discard + +A mutation returned the State error directly from the root dispatch wrapper. +It did not return `runtime.Discard`. This command failed: + +```bash +go test . \ + -run '^(TestStateMutableAliasFailureCannotRollbackAndNextActivationReloads|TestSchedule_StateFailureFaultsOnlyReminderActivationAndReloadsStore)$' \ + -count=1 -v +``` + +Both tests reused the failed Activation. The mutable State test read local +value 2 instead of Confirmed State value 1. The Reminder test read local value +7 instead of the committed Store value 8. The command exited 1. Restoring +`runtime.Discard` made both tests pass. + +## Focused Verification + +This command passed with both tests and the race detector: + +```bash +go test -race . \ + -run '^(TestStateMutableAliasFailureCannotRollbackAndNextActivationReloads|TestSchedule_StateFailureFaultsOnlyReminderActivationAndReloadsStore)$' \ + -count=1 -v +``` + +It ran two tests and exited 0. + +The same command with `-count=20` also exited 0. The broader State, Schedule, +and Grain Timer State failure race set exited 0. + +## Independent Review + +The first read-only review found that the mutable alias test used direct Grain +Contexts. It did not prove the Runtime Activation lifecycle. It also asked for +direct Store JSON and ETag checks. The test now uses real Runtime Calls, checks +the Store record, and proves a second factory call reloads Confirmed State. + +The follow-up review confirmed those findings were fixed. It found no blocking +issue. It also found that the test still used the default real clock and +Reminder poller. The test now injects a fake clock and disables that poller. + +## Full Verification + +The pre-commit `make ci` run exited 0 in 85.03 seconds. Its nonzero discovery +counts were: + +- release command contract: 4; +- normal and race: 633 each; +- simulation: 20; +- generator: 19; +- network: 45; +- example: 49; +- Activation and Reminder resource checks: 1 each; +- transport, Request Context, and generator fuzz checks: 1 each. + +The pre-commit `make external` run exited 0 in 19.66 seconds. Its nonzero +discovery counts were 4 release command contract tests, 1 Quick Start process +test, and 19 external configuration and restart tests. diff --git a/research/release-0.1.0-issue-115-upgrade-proof.md b/research/release-0.1.0-issue-115-upgrade-proof.md new file mode 100644 index 0000000..3429f36 --- /dev/null +++ b/research/release-0.1.0-issue-115-upgrade-proof.md @@ -0,0 +1,110 @@ +# 0.1.0 Issue 115 Upgrade Proof + +## Scope + +This follow-up repairs and proves the upgrade from the latest public version. +The source version is the annotated `v0.0.5` tag. Its peeled commit is +`ae2b32dda32f23888b8d140a8388784d686d75cd`. The follow-up starts from local +candidate `4cb95eaf778efcbaefb010f043066dfa070ee277`. The completed implementation +commit is `66ba16047adbca5d74dfc2f8534471d794e2d07b`. + +The exact `v0.0.5` Reminder table uses `entity_type` and `entity_key`. The old +migration fixture used the new column names. It did not prove that a real +`v0.0.5` database could open. + +## Design + +The Store reads all Reminder columns in one transaction. It accepts one of two +complete identity pairs: + +- `entity_type` and `entity_key` from `v0.0.5`; +- `grain_type` and `grain_key` from 0.1.0. + +The Store creates the coordination schema and migrates the Reminder schema in +one transaction. It renames the old pair before it adds the first tick column +and due index. It rejects a mixed or incomplete pair. SQLite updates the +primary key definition as part of each column rename. A failed migration does +not leave the new version or Member tables behind. + +The external proof uses one clean consumer module. It first resolves exact +`v0.0.5` without a replacement. It builds a small writer against the old public +Shadow Application and its old generated package. The writer disables Reminder +polling, writes Device and Workshop State and a far-future Reminder to SQLite, +then closes the old Runtime normally. The candidate is built before the writer +runs. It binds an operating-system-selected port and reports the actual address +through its readiness output. Candidate processes open the same database, read +the old State through the public HTTP path, update the Device configuration, +and prove the update after a forced restart. The release test then uses the +candidate Store code to read and compare the migrated Reminder and Member +records directly. + +## Focused proof + +The following commands passed: + +```text +go test ./store -run '^TestMigrate_(OldDatabaseReadsBackEveryConfirmedState|ReminderIdentityColumnsRejectIncompleteOrMixedSchema|ReminderSchemaFailureRollsBackCoordinationSchema)$' -count=1 +go test ./store -count=1 +go test -tags release ./examples/shadow/cmd/conformance -run '^TestExternalModuleConfiguration$' -count=1 +go test -tags release ./internal/testcheck -run '^TestMakefile_ReleaseCommandsKeepTestcheck$' -count=1 +go test -tags release ./examples/shadow/cmd/conformance -run '^TestExternalModuleUpgradeProof$' -count=1 -v +``` + +The final cold focused external proof passed in 48.01 seconds. It reported exact +source `v0.0.5` and candidate target `v0.0.0`. + +At completed implementation commit +`66ba16047adbca5d74dfc2f8534471d794e2d07b`, `make external` passed with exit 0 +in 48.99 seconds. The release entry point +reported four command contract tests. It also reported one Quick Start process +test and 20 external module tests. The external module set included the upgrade +proof, which passed in 21.13 seconds with warm build artifacts. + +At the same commit, the final `make ci` passed with exit 0 in 57.12 seconds. +The normal and race lanes each selected 637 tests. The gate also selected 20 +simulation tests, 19 generator tests, 45 Transport network tests, 49 Shadow +network tests, two resource tests, and three focused fuzz tests. + +## Review + +An independent read-only review found three problems in the first proof. The +old Shadow Reminder could become due after 30 seconds, the process driver +reserved and released a port before the child bound it, and coordination schema +bootstrap committed before Reminder migration could fail. + +The final proof uses a far-future Reminder and disables the old poller. The +candidate binds port zero and the driver reads its real address from readiness +output. Coordination bootstrap and migration now share one transaction, and +failure tests prove that no new coordination tables remain. The old process +also closes normally before the upgrade. Forced candidate stops remain in the +proof because they verify restart recovery. + +The final review also required full persisted-value comparisons. The external +proof now keeps `ReportedAt` across every candidate restart and checks the old +Reminder due time, interval, and exact ETag. The focused migration proof checks +each Member generation, alive time, ETag, and full suspect vote identity and +expiry. The follow-up review found no remaining P0, P1, or P2 issue in this +batch. + +The integration review found two evidence defects: the first wording assigned +the direct Reminder read to the candidate process, and the research note did +not name the completed implementation commit. The Design and Focused proof +sections above now state the actual ownership and exact commit evidence. + +## Mutation + +The mutation removed both `ALTER TABLE ... RENAME COLUMN` statements. The +focused old-database test failed in both durability tiers: + +```text +OpenSQLite: SQL logic error: no such column: grain_type (1) +``` + +The mutated command exited 1. The implementation was restored. The same +focused command then passed. + +## Remaining release proof + +The reviewed batch still needs integration into `master`. The final `master` +commit then needs hosted platform proof and the exact tagged-module proof. A +local candidate proof does not satisfy those remote steps. diff --git a/runtime/runtime.go b/runtime/runtime.go deleted file mode 100644 index b1b0fed..0000000 --- a/runtime/runtime.go +++ /dev/null @@ -1,628 +0,0 @@ -// Package runtime is gor's local actor engine: it manages entity activation, -// lifecycle, mailboxes, and dispatch within one process. -// -// Application code should use the root package's gor.New, gor.Register, -// gor.Ref, and *gor.Runtime APIs instead of importing runtime directly. -package runtime - -import ( - "context" - "errors" - "fmt" - "sort" - "sync" - "time" - - "github.com/suraciii/gor/clock" - "github.com/suraciii/gor/mail" -) - -var ( - ErrTypeNotRegistered = errors.New("entity type is not registered") - ErrRuntimeClosed = errors.New("runtime closed") - ErrPanic = errors.New("runtime panic") -) - -type GrainId struct { - GrainType string - GrainKey string -} - -type Activation struct { - GrainId GrainId - Queued int -} - -type Dispatch func(context.Context, any, string, any, any) error - -// DeactivationReason describes why an activation left the active state. It is -// fixed at the first transition out of active and is never rewritten by later -// events. -type DeactivationReason uint8 - -const ( - Idle DeactivationReason = iota + 1 - OwnershipLost - RuntimeClosed - Faulted -) - -type Registration struct { - Factory func(context.Context, GrainId) (any, error) - Dispatch Dispatch - OnDeactivate func(context.Context, GrainId, DeactivationReason, any) -} - -type Discard struct { - Err error -} - -func (d Discard) Error() string { - if d.Err == nil { - return "activation discarded" - } - return d.Err.Error() -} - -func (d Discard) Unwrap() error { - return d.Err -} - -type Config struct { - Clock clock.Clock - MailboxCapacity int - IdleTimeout time.Duration - EvictionInterval time.Duration -} - -// engineState tracks the local engine lifecycle independently of the root -// runtime: graceful and sudden stops are distinct, and a sudden stop can -// escalate an in-progress graceful stop. -type engineState uint8 - -const ( - engineRunning engineState = iota - engineClosing - engineKilling -) - -type Runtime struct { - clock clock.Clock - mailboxCapacity int - idleTimeout time.Duration - - mu sync.Mutex - state engineState - registrations map[string]Registration - activations map[GrainId]*activation - pending map[GrainId]*entry - - stop chan struct{} - ticker clock.Ticker - evictionDone chan struct{} - pendingDeactivations int - deactivationsDone chan struct{} - killing chan struct{} - done chan struct{} - killCtx context.Context - killCancel context.CancelFunc -} - -// ActivationState is the explicit state machine an activation walks: an -// activation is created activating, becomes active, then leaves active exactly -// once (deactivating) before it is stopped. The deactivation reason is written -// on that single departure. -type ActivationState uint8 - -const ( - ActivationActivating ActivationState = iota - ActivationActive - ActivationDeactivating - ActivationStopped -) - -type activation struct { - id GrainId - instance any - onDeactivate func(context.Context, GrainId, DeactivationReason, any) - skipOnDeactivate bool - reason DeactivationReason - mailbox *mail.Box - lastUsed time.Time - calls int - state ActivationState - done chan struct{} -} - -type entry struct { - ready chan struct{} - act *activation - err error -} - -func New(config Config) *Runtime { - killCtx, killCancel := context.WithCancel(context.Background()) - r := &Runtime{ - clock: config.Clock, - mailboxCapacity: config.MailboxCapacity, - idleTimeout: config.IdleTimeout, - registrations: make(map[string]Registration), - activations: make(map[GrainId]*activation), - pending: make(map[GrainId]*entry), - stop: make(chan struct{}), - evictionDone: make(chan struct{}), - killing: make(chan struct{}), - done: make(chan struct{}), - killCtx: killCtx, - killCancel: killCancel, - } - if config.IdleTimeout > 0 && config.EvictionInterval > 0 { - r.ticker = config.Clock.NewTicker(config.EvictionInterval) - go r.evictLoop() - } else { - close(r.evictionDone) - } - return r -} - -func (r *Runtime) Register(name string, registration Registration) error { - r.mu.Lock() - defer r.mu.Unlock() - if _, exists := r.registrations[name]; exists { - return fmt.Errorf("entity type %q is already registered", name) - } - r.registrations[name] = registration - return nil -} - -func (r *Runtime) Invoke(ctx context.Context, id GrainId, method string, args any, reply any) error { - if err := checkCycle(ctx, id); err != nil { - return err - } - callCtx, cancel := context.WithCancel(ctx) - defer cancel() - defer context.AfterFunc(r.killCtx, cancel)() - - // The call occupies id from admission on: activation and the method body - // both run with id on the occupied chain, so a nested call that targets - // id again is rejected as a cycle. - callCtx = withOccupied(callCtx, id) - - registration, err := r.registration(id.GrainType) - if err != nil { - return err - } - for { - act, err := r.activationFor(callCtx, id, registration) - if err != nil { - return err - } - if !r.admit(act) { - continue - } - _, err = act.mailbox.Call(callCtx, func(callCtx context.Context) (any, error) { - defer r.callFinished(act) - return nil, r.dispatch(registration, act, callCtx, method, args, reply) - }) - if errors.Is(err, mail.ErrOverloaded) || errors.Is(err, mail.ErrClosed) { - r.callFinished(act) - } - return err - } -} - -// Deactivate stops the activation for id because this node no longer owns it -// (or the view has no active owner). The deactivation hook runs with -// OwnershipLost. -func (r *Runtime) Deactivate(id GrainId) { - r.mu.Lock() - act, ok := r.activations[id] - if !ok { - r.mu.Unlock() - return - } - started := r.deactivateLocked(act, OwnershipLost) - r.mu.Unlock() - if started { - act.mailbox.Close() - } -} - -func (r *Runtime) GrainIds() []GrainId { - r.mu.Lock() - defer r.mu.Unlock() - identities := make([]GrainId, 0, len(r.activations)) - for id := range r.activations { - identities = append(identities, id) - } - return identities -} - -func (r *Runtime) Activations() []Activation { - r.mu.Lock() - activations := make([]Activation, 0, len(r.activations)) - for _, act := range r.activations { - if act.state != ActivationActive { - continue - } - activations = append(activations, Activation{ - GrainId: act.id, - Queued: act.mailbox.Len(), - }) - } - r.mu.Unlock() - - sort.Slice(activations, func(i, j int) bool { - if activations[i].GrainId.GrainType != activations[j].GrainId.GrainType { - return activations[i].GrainId.GrainType < activations[j].GrainId.GrainType - } - return activations[i].GrainId.GrainKey < activations[j].GrainId.GrainKey - }) - return activations -} - -// Done returns a channel that closes when the engine has stopped. For a -// graceful stop that is after deactivation hooks finish; for a sudden stop it -// is after the engine's own goroutines exit, not after user methods. -func (r *Runtime) Done() <-chan struct{} { - return r.done -} - -// BeginClose starts a graceful stop and returns immediately. See Close. -func (r *Runtime) BeginClose() { - r.mu.Lock() - if r.state != engineRunning { - r.mu.Unlock() - return - } - r.state = engineClosing - close(r.stop) - r.beginStopDeactivationsLocked(false) - acts := r.snapshotActivationsLocked() - ticker := r.ticker - r.mu.Unlock() - if ticker != nil { - ticker.Stop() - } - for _, act := range acts { - act.mailbox.Close() - } - go r.drain() -} - -// BeginKill starts a sudden stop, or escalates an in-progress graceful stop, -// and returns immediately. See Kill. -func (r *Runtime) BeginKill() { - r.mu.Lock() - switch r.state { - case engineRunning: - r.state = engineKilling - close(r.stop) - close(r.killing) - r.beginStopDeactivationsLocked(true) - acts := r.snapshotActivationsLocked() - ticker := r.ticker - r.mu.Unlock() - if ticker != nil { - ticker.Stop() - } - r.killCancel() - for _, act := range acts { - act.mailbox.Close() - } - go r.drain() - case engineClosing: - r.state = engineKilling - close(r.killing) - r.markSkipOnDeactivateLocked() - r.mu.Unlock() - r.killCancel() - default: - r.mu.Unlock() - } -} - -// drain waits for the eviction loop to exit, then for either all deactivations -// to finish (graceful) or a sudden stop to overtake them. It never waits for -// user methods once the engine is killing. -func (r *Runtime) drain() { - defer close(r.done) - <-r.evictionDone - r.mu.Lock() - deactivationsDone := r.deactivationsDone - r.mu.Unlock() - if deactivationsDone == nil { - return - } - select { - case <-deactivationsDone: - case <-r.killing: - } -} - -// beginStopDeactivationsLocked marks deactivation hooks to skip when requested, -// begins deactivation of every active activation, and arms the channel that -// closes once every in-flight deactivation has finished. The caller must hold -// r.mu. -func (r *Runtime) beginStopDeactivationsLocked(skip bool) { - for _, act := range r.activations { - if skip { - act.skipOnDeactivate = true - } - if beginDeactivation(act, RuntimeClosed) { - r.startDeactivationWaiterLocked(act) - } - } - r.deactivationsDone = make(chan struct{}) - if r.pendingDeactivations == 0 { - close(r.deactivationsDone) - } -} - -// markSkipOnDeactivateLocked marks every activation's deactivation hook to be -// skipped if it has not started yet. The caller must hold r.mu. -func (r *Runtime) markSkipOnDeactivateLocked() { - for _, act := range r.activations { - act.skipOnDeactivate = true - } -} - -func (r *Runtime) snapshotActivationsLocked() []*activation { - acts := make([]*activation, 0, len(r.activations)) - for _, act := range r.activations { - acts = append(acts, act) - } - return acts -} - -func (r *Runtime) registration(name string) (Registration, error) { - r.mu.Lock() - defer r.mu.Unlock() - if r.state != engineRunning { - return Registration{}, ErrRuntimeClosed - } - registration, ok := r.registrations[name] - if !ok { - return Registration{}, fmt.Errorf("%w: %s", ErrTypeNotRegistered, name) - } - return registration, nil -} - -func (r *Runtime) activationFor(ctx context.Context, id GrainId, registration Registration) (*activation, error) { - for { - r.mu.Lock() - if r.state != engineRunning { - r.mu.Unlock() - return nil, ErrRuntimeClosed - } - if act, ok := r.activations[id]; ok { - switch act.state { - case ActivationActive: - r.mu.Unlock() - return act, nil - case ActivationDeactivating: - done := act.done - r.mu.Unlock() - select { - case <-done: - continue - case <-ctx.Done(): - return nil, ctx.Err() - } - case ActivationStopped: - delete(r.activations, id) - } - } - if pending, ok := r.pending[id]; ok { - ready := pending.ready - r.mu.Unlock() - select { - case <-ready: - if pending.err != nil { - return nil, pending.err - } - return pending.act, nil - case <-ctx.Done(): - return nil, ctx.Err() - } - } - - pending := &entry{ready: make(chan struct{})} - r.pending[id] = pending - r.mu.Unlock() - return r.createActivation(ctx, id, registration, pending) - } -} - -func (r *Runtime) createActivation(ctx context.Context, id GrainId, registration Registration, pending *entry) (act *activation, err error) { - defer func() { - if value := recover(); value != nil { - act = nil - err = fmt.Errorf("%w: activation factory panicked: %v", ErrPanic, value) - } - r.mu.Lock() - delete(r.pending, id) - if err == nil && r.state != engineRunning { - err = ErrRuntimeClosed - } - pending.act = act - pending.err = err - if err == nil { - r.activations[id] = act - } - close(pending.ready) - r.mu.Unlock() - if err != nil && act != nil { - act.mailbox.Close() - } - }() - - act, err = r.activate(ctx, id, registration) - return act, err -} - -func (r *Runtime) activate(ctx context.Context, id GrainId, registration Registration) (*activation, error) { - act := &activation{ - id: id, - onDeactivate: registration.OnDeactivate, - state: ActivationActivating, - done: make(chan struct{}), - } - instance, err := registration.Factory(ctx, id) - if err != nil { - return nil, err - } - act.instance = instance - act.mailbox = mail.New(r.mailboxCapacity) - act.lastUsed = r.clock.Now() - act.state = ActivationActive - return act, nil -} - -func (r *Runtime) admit(act *activation) bool { - r.mu.Lock() - defer r.mu.Unlock() - if act.state != ActivationActive { - return false - } - act.lastUsed = r.clock.Now() - act.calls++ - return true -} - -func (r *Runtime) callFinished(act *activation) { - r.mu.Lock() - act.calls-- - r.mu.Unlock() -} - -func (r *Runtime) dispatch(registration Registration, act *activation, ctx context.Context, method string, args any, reply any) (err error) { - defer func() { - if value := recover(); value != nil { - err = fmt.Errorf("%w: entity method panicked: %v", ErrPanic, value) - r.stopActivation(act) - } - }() - err = registration.Dispatch(ctx, act.instance, method, args, reply) - var discard Discard - if errors.As(err, &discard) { - r.stopActivation(act) - return discard.Err - } - return err -} - -func (r *Runtime) evictLoop() { - defer close(r.evictionDone) - for { - select { - case now := <-r.ticker.C(): - r.evict(now) - case <-r.stop: - return - } - } -} - -func (r *Runtime) evict(now time.Time) { - r.mu.Lock() - if r.state != engineRunning { - r.mu.Unlock() - return - } - victims := make([]*activation, 0) - for _, act := range r.activations { - if act.state == ActivationActive && act.calls == 0 && !now.Before(act.lastUsed.Add(r.idleTimeout)) && r.deactivateLocked(act, Idle) { - victims = append(victims, act) - } - } - r.mu.Unlock() - - for _, act := range victims { - act.mailbox.Close() - } -} - -func (r *Runtime) stopActivation(act *activation) { - r.mu.Lock() - started := r.deactivateLocked(act, Faulted) - r.mu.Unlock() - if !started { - return - } - act.mailbox.Close() -} - -func (r *Runtime) deactivateLocked(act *activation, reason DeactivationReason) bool { - if !beginDeactivation(act, reason) { - return false - } - r.startDeactivationWaiterLocked(act) - return true -} - -func (r *Runtime) startDeactivationWaiterLocked(act *activation) { - r.pendingDeactivations++ - go r.waitForDeactivation(act) -} - -func (r *Runtime) waitForDeactivation(act *activation) { - <-act.mailbox.Done() - r.mu.Lock() - var ( - onDeactivate func(context.Context, GrainId, DeactivationReason, any) - reason DeactivationReason - ) - if !act.skipOnDeactivate { - onDeactivate = act.onDeactivate - reason = act.reason - } - r.mu.Unlock() - if onDeactivate != nil { - onDeactivate(context.Background(), act.id, reason, act.instance) - } - r.finishDeactivation(act) - r.deactivationFinished() -} - -// deactivationFinished records one deactivation waiter completing. When the -// last in-flight deactivation finishes after a stop has begun, it closes the -// channel the graceful drain waits on. -func (r *Runtime) deactivationFinished() { - r.mu.Lock() - r.pendingDeactivations-- - if r.pendingDeactivations == 0 && r.deactivationsDone != nil { - close(r.deactivationsDone) - r.deactivationsDone = nil - } - r.mu.Unlock() -} - -func (r *Runtime) finishDeactivation(act *activation) { - r.mu.Lock() - defer r.mu.Unlock() - if !finishDeactivation(act) { - return - } - if current, ok := r.activations[act.id]; ok && current == act { - delete(r.activations, act.id) - } -} - -func beginDeactivation(act *activation, reason DeactivationReason) bool { - if act.state != ActivationActive { - return false - } - act.state = ActivationDeactivating - act.reason = reason - return true -} - -func finishDeactivation(act *activation) bool { - if act.state != ActivationDeactivating { - return false - } - act.state = ActivationStopped - close(act.done) - return true -} diff --git a/schedule.go b/schedule.go index 395bed6..6efcc0a 100644 --- a/schedule.go +++ b/schedule.go @@ -26,8 +26,7 @@ type ReminderTime struct { } // After returns a one-shot Reminder due delay after Set uses its clock. A zero -// delay is due immediately; a negative delay is due in the past and is -// eligible on the next poller poll. +// delay is due immediately. Set rejects a negative delay. func After(delay time.Duration) ReminderTime { return ReminderTime{delay: delay} } @@ -35,13 +34,14 @@ func After(delay time.Duration) ReminderTime { // Every returns a Reminder whose first due time is interval after Set uses its // clock and whose subsequent due times use the same interval when interval is // positive. Every(0) is accepted and produces a one-shot Reminder, just like -// After(0). A negative interval is accepted and stored. +// After(0). Set rejects a negative interval. func Every(interval time.Duration) ReminderTime { return ReminderTime{delay: interval, interval: interval} } -// TickStatus describes the time represented by one Reminder delivery. +// TickStatus describes one Reminder delivery. type TickStatus struct { + ReminderName string FirstTickTime time.Time Period time.Duration CurrentTickTime time.Time @@ -57,40 +57,72 @@ type MethodHandle[T any] struct { // Handle builds a MethodHandle from a method expression on T's interface, // such as gor.Handle(Account.ApplyInterest). The method name is read off the // expression once, at this call. A closure of the same function type compiles, -// but its name is not a method name and delivery fails with "unknown method". +// but its name is not a method name and Set rejects it. func Handle[T any](m func(T, context.Context, TickStatus) error) MethodHandle[T] { - full := runtime.FuncForPC(reflect.ValueOf(m).Pointer()).Name() - name := full[strings.LastIndexByte(full, '.')+1:] + if m == nil { + return MethodHandle[T]{} + } + function := runtime.FuncForPC(reflect.ValueOf(m).Pointer()) + if function == nil { + return MethodHandle[T]{} + } + full := function.Name() + separator := strings.LastIndexByte(full, '.') + if separator < 0 || separator == len(full)-1 { + return MethodHandle[T]{} + } + name := full[separator+1:] return MethodHandle[T]{method: name} } -// Reminder manages Reminders for the Grain bound to a Binder, typed to the +// Reminder manages Reminders for the Grain bound to a GrainContext, typed to the // Grain's interface T. Obtain one with NewReminder[T]; the zero value has no // reminder store and its operations return ErrReminderStoreUnavailable. type Reminder[T any] struct { identity store.GrainId store store.ReminderStore clock clock.Clock + runtime *Runtime } // NewReminder returns a Reminder manager bound to the Grain represented by b // and typed to its interface T. -func NewReminder[T any](b *Binder) Reminder[T] { +func NewReminder[T any](b *GrainContext) Reminder[T] { return Reminder[T]{ identity: b.identity, store: b.runtime.reminderStore, clock: b.runtime.clock, + runtime: b.runtime, } } // Set creates or replaces the named Reminder for the bound Grain. The new // first due time is used as FirstTickTime and DueAt. A successful Set persists // the Reminder. Each due occurrence is delivered at most once, and an -// invocation that returns an error is not automatically retried. +// invocation that returns an error is not automatically retried. Set rejects +// a blank name, negative time, or invalid method handle before Store I/O. func (s Reminder[T]) Set(ctx context.Context, name string, when ReminderTime, m MethodHandle[T]) error { + if strings.TrimSpace(name) == "" { + return errors.New("invalid Reminder name: must not be empty") + } + if when.interval < 0 { + return errors.New("invalid Reminder period: must not be negative") + } + if when.delay < 0 { + return errors.New("invalid Reminder due time: must not be negative") + } + if m.method == "" { + return errors.New("invalid Reminder method handle") + } if s.store == nil { return ErrReminderStoreUnavailable } + if s.runtime == nil { + return errors.New("invalid Reminder: missing Runtime") + } + if _, err := s.runtime.resolveReminderCall(s.identity, m.method); err != nil { + return err + } firstTickTime := s.clock.Now().Add(when.delay) return s.store.Put(ctx, store.Reminder{ GrainId: s.identity, @@ -103,8 +135,12 @@ func (s Reminder[T]) Set(ctx context.Context, name string, when ReminderTime, m } // Cancel asks the reminder store to delete the named Reminder for the bound -// Grain. Canceling a name that does not exist succeeds as a no-op. +// Grain. Canceling a name that does not exist succeeds as a no-op. Cancel +// rejects a blank name before Store I/O. func (s Reminder[T]) Cancel(ctx context.Context, name string) error { + if strings.TrimSpace(name) == "" { + return errors.New("invalid Reminder name: must not be empty") + } if s.store == nil { return ErrReminderStoreUnavailable } diff --git a/schedule_test.go b/schedule_test.go index 5398738..affe94f 100644 --- a/schedule_test.go +++ b/schedule_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "sync/atomic" "testing" "testing/synctest" @@ -14,12 +15,14 @@ import ( ) type scheduledAccount interface { - Arm(context.Context) error + Arm(context.Context, string) error Wake(context.Context, TickStatus) error Value(context.Context) (int64, error) } -type scheduledAccountArmRequest struct{} +type scheduledAccountArmRequest struct { + A0 string +} type scheduledAccountArmReply struct{} @@ -35,12 +38,15 @@ type scheduledAccountValueReply struct { R0 int64 } -type scheduledAccountEntity struct { +type scheduledAccountGrain struct { value State[int64] schedule Reminder[scheduledAccount] wakeErr error wakeStarted chan struct{} + wakeRelease <-chan struct{} + wakeDone chan<- struct{} wakeCalls *atomic.Int32 + wakeTicks chan<- TickStatus cancelShapedErr bool } @@ -49,14 +55,61 @@ type scheduledAccountProxy struct { id GrainId } -func (a *scheduledAccountEntity) Arm(ctx context.Context) error { - return a.schedule.Set(ctx, "wake", After(12*time.Second), Handle(scheduledAccount.Wake)) +type countingReminderStore struct { + *store.Memory + putCalls atomic.Int32 + deleteCalls atomic.Int32 + claimCalls atomic.Int32 +} + +type terminalClaimErrorStore struct { + *store.Memory + err error } -func (a *scheduledAccountEntity) Wake(ctx context.Context, _ TickStatus) error { +func (s *terminalClaimErrorStore) Claim(ctx context.Context, reminder store.Reminder, nextDueAt time.Time) (bool, error) { + if nextDueAt.IsZero() { + return false, s.err + } + return s.Memory.Claim(ctx, reminder, nextDueAt) +} + +func (s *countingReminderStore) Put(ctx context.Context, reminder store.Reminder) error { + s.putCalls.Add(1) + return s.Memory.Put(ctx, reminder) +} + +func (s *countingReminderStore) Delete(ctx context.Context, id store.GrainId, name string) error { + s.deleteCalls.Add(1) + return s.Memory.Delete(ctx, id, name) +} + +func (s *countingReminderStore) Claim(ctx context.Context, reminder store.Reminder, nextDueAt time.Time) (bool, error) { + s.claimCalls.Add(1) + return s.Memory.Claim(ctx, reminder, nextDueAt) +} + +func (a *scheduledAccountGrain) Arm(ctx context.Context, name string) error { + return a.schedule.Set(ctx, name, After(12*time.Second), Handle(scheduledAccount.Wake)) +} + +func (a *scheduledAccountGrain) Wake(ctx context.Context, tick TickStatus) error { + if a.wakeTicks != nil { + a.wakeTicks <- tick + } if a.wakeCalls != nil { a.wakeCalls.Add(1) } + if a.wakeRelease != nil { + if a.wakeStarted != nil { + close(a.wakeStarted) + } + <-a.wakeRelease + if a.wakeDone != nil { + close(a.wakeDone) + } + return nil + } if a.wakeStarted != nil { close(a.wakeStarted) <-ctx.Done() @@ -73,12 +126,12 @@ func (a *scheduledAccountEntity) Wake(ctx context.Context, _ TickStatus) error { return a.value.Set(ctx, a.value.Get()+1) } -func (a *scheduledAccountEntity) Value(context.Context) (int64, error) { +func (a *scheduledAccountGrain) Value(context.Context) (int64, error) { return a.value.Get(), nil } -func (p *scheduledAccountProxy) Arm(ctx context.Context) error { - return p.invoker.Invoke(ctx, p.id, "Arm", &scheduledAccountArmRequest{}, &scheduledAccountArmReply{}) +func (p *scheduledAccountProxy) Arm(ctx context.Context, name string) error { + return p.invoker.Invoke(ctx, p.id, "Arm", &scheduledAccountArmRequest{A0: name}, &scheduledAccountArmReply{}) } func (p *scheduledAccountProxy) Wake(ctx context.Context, tick TickStatus) error { @@ -94,7 +147,7 @@ func (p *scheduledAccountProxy) Value(ctx context.Context) (int64, error) { func dispatchScheduledAccount(ctx context.Context, instance scheduledAccount, method string, args any, reply any) error { switch method { case "Arm": - return instance.Arm(ctx) + return instance.Arm(ctx, args.(*scheduledAccountArmRequest).A0) case "Wake": return instance.Wake(ctx, args.(*scheduledAccountWakeRequest).A0) case "Value": @@ -132,7 +185,10 @@ func newScheduledAccountReminderCall(method string, status TickStatus) (args any type scheduledAccountConfig struct { wakeErr error wakeStarted chan struct{} + wakeRelease <-chan struct{} + wakeDone chan<- struct{} wakeCalls *atomic.Int32 + wakeTicks chan<- TickStatus cancelShapedErr bool } @@ -142,19 +198,22 @@ func installScheduledAccount(t *testing.T, rt *Runtime, factoryCalls *atomic.Int if len(configs) > 0 { config = configs[0] } - if err := InstallType[scheduledAccount](rt, dispatchScheduledAccount, func(invoker Invoker, id GrainId) scheduledAccount { + if err := InstallType[scheduledAccount](rt, GeneratedCodeVersion, "gor.scheduledAccount", dispatchScheduledAccount, func(invoker Invoker, id GrainId) scheduledAccount { return &scheduledAccountProxy{invoker: invoker, id: id} }, newScheduledAccountCall, newScheduledAccountReminderCall); err != nil { t.Fatal(err) } - if err := Register[scheduledAccount](rt, func(b *Binder) scheduledAccount { + if err := Register[scheduledAccount](rt, func(b *GrainContext) scheduledAccount { factoryCalls.Add(1) - return &scheduledAccountEntity{ + return &scheduledAccountGrain{ value: NewState[int64](b, "value"), schedule: NewReminder[scheduledAccount](b), wakeErr: config.wakeErr, wakeStarted: config.wakeStarted, + wakeRelease: config.wakeRelease, + wakeDone: config.wakeDone, wakeCalls: config.wakeCalls, + wakeTicks: config.wakeTicks, cancelShapedErr: config.cancelShapedErr, } }); err != nil { @@ -180,10 +239,11 @@ func TestSchedule_OnErrorReceivesInvocationFailure(t *testing.T) { errorsSeen <- event }), ) - defer rt.Close() + defer closeRuntime(rt) installScheduledAccount(t, rt, new(atomic.Int32), scheduledAccountConfig{wakeErr: wakeErr, wakeCalls: wakeCalls}) + mustStart(t, rt) - if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background()); err != nil { + if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background(), "wake"); err != nil { t.Fatalf("Arm: %v", err) } fakeClock.Advance(12 * time.Second) @@ -192,9 +252,14 @@ func TestSchedule_OnErrorReceivesInvocationFailure(t *testing.T) { select { case got := <-errorsSeen: source, ok := got.Source.(ReminderInvocation) - wantID := GrainId{GrainType: TypeName[scheduledAccount](), GrainKey: "alice"} - if !ok || got.GrainId != wantID || source.Method != "Wake" || !errors.Is(got.Err, wakeErr) { - t.Fatalf("OnError event = %#v, want identity %v, ReminderInvocation{Method: Wake}, error %v", got, wantID, wakeErr) + wantID := GrainId{GrainType: GrainType("gor.scheduledAccount"), GrainKey: "alice"} + wantStatus := TickStatus{ + ReminderName: "wake", + FirstTickTime: start.Add(12 * time.Second), + CurrentTickTime: start.Add(12 * time.Second), + } + if !ok || got.GrainId != wantID || source.Name != "wake" || source.Method != "Wake" || source.TickStatus != wantStatus || !errors.Is(got.Err, wakeErr) { + t.Fatalf("OnError event = %#v, want identity %v, ReminderInvocation for wake/Wake with status %#v, error %v", got, wantID, wantStatus, wakeErr) } default: t.Fatal("OnError did not receive Reminder invocation failure") @@ -205,25 +270,98 @@ func TestSchedule_OnErrorReceivesInvocationFailure(t *testing.T) { }) } +func TestSchedule_StateFailureFaultsOnlyReminderActivationAndReloadsStore(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(0, 0).UTC() + fakeClock := clock.NewFake(start) + backend := newAppliedOutcomeStore() + aliceID := store.GrainId{GrainType: "gor.scheduledAccount", GrainKey: "alice"} + if _, err := backend.Memory.Write(context.Background(), aliceID, []byte(`{"value":7}`), 0); err != nil { + t.Fatalf("seed Alice State: %v", err) + } + factoryCalls := new(atomic.Int32) + wakeCalls := new(atomic.Int32) + errorsSeen := make(chan BackgroundError, 1) + rt := mustNew(t, + WithStore(backend), + WithReminderStore(backend), + WithClock(fakeClock), + WithIdleTimeout(0), + WithEvictionInterval(0), + WithReminderInterval(time.Second), + OnError(func(event BackgroundError) { errorsSeen <- event }), + ) + defer closeRuntime(rt) + installScheduledAccount(t, rt, factoryCalls, scheduledAccountConfig{wakeCalls: wakeCalls}) + mustStart(t, rt) + + alice := Ref[scheduledAccount](rt, "alice") + bob := Ref[scheduledAccount](rt, "bob") + if err := alice.Arm(context.Background(), "wake"); err != nil { + t.Fatalf("Arm Alice: %v", err) + } + if value, err := bob.Value(context.Background()); err != nil || value != 0 { + t.Fatalf("Bob value before Reminder = (%d, %v), want (0, nil)", value, err) + } + if got := factoryCalls.Load(); got != 2 { + t.Fatalf("factory calls before Reminder = %d, want 2", got) + } + + writeErr := errors.New("Reminder State write reply was lost") + backend.failNextWrite(writeErr) + fakeClock.Advance(12 * time.Second) + synctest.Wait() + + select { + case event := <-errorsSeen: + source, ok := event.Source.(ReminderInvocation) + if !ok || event.GrainId != (GrainId{GrainType: "gor.scheduledAccount", GrainKey: "alice"}) || + source.Name != "wake" || source.Method != "Wake" || !errors.Is(event.Err, writeErr) || + !errors.Is(event.Err, ErrPersistenceFailed) { + t.Fatalf("Reminder State failure event = %#v", event) + } + default: + t.Fatal("Reminder State failure was not reported") + } + if got := wakeCalls.Load(); got != 1 { + t.Fatalf("Wake calls = %d, want 1", got) + } + if got := factoryCalls.Load(); got != 2 { + t.Fatalf("factory calls after Reminder failure = %d, want 2 before the next Call", got) + } + + if value, err := bob.Value(context.Background()); err != nil || value != 0 { + t.Fatalf("Bob value after Alice failure = (%d, %v), want (0, nil)", value, err) + } + if got := factoryCalls.Load(); got != 2 { + t.Fatalf("Bob Activation was replaced after Alice failure: factory calls = %d, want 2", got) + } + if value, err := alice.Value(context.Background()); err != nil || value != 8 { + t.Fatalf("Alice value after Reminder failure = (%d, %v), want confirmed value (8, nil)", value, err) + } + if got := factoryCalls.Load(); got != 3 { + t.Fatalf("factory calls after Alice reload = %d, want 3", got) + } + }) +} + func TestSchedule_DropsInvocationFailureWithoutOnError(t *testing.T) { synctest.Test(t, func(t *testing.T) { start := time.Unix(0, 0).UTC() fakeClock := clock.NewFake(start) backend := store.NewMemory() rt := newScheduledRuntime(t, backend, fakeClock) - defer rt.Close() + defer closeRuntime(rt) installScheduledAccount(t, rt, new(atomic.Int32), scheduledAccountConfig{wakeErr: errors.New("scheduled wake failed")}) + mustStart(t, rt) - if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background()); err != nil { + if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background(), "wake"); err != nil { t.Fatalf("Arm: %v", err) } fakeClock.Advance(12 * time.Second) synctest.Wait() - rows, err := backend.ListDue(context.Background(), start.Add(12*time.Second)) - if err != nil { - t.Fatalf("ListDue: %v", err) - } + rows := readDueReminders(t, backend, start.Add(12*time.Second)) if len(rows) != 0 { t.Fatalf("reminders after failed one-shot invocation = %#v, want none", rows) } @@ -248,8 +386,9 @@ func TestSchedule_DropsCancellationOnRuntimeClose(t *testing.T) { }), ) installScheduledAccount(t, rt, new(atomic.Int32), scheduledAccountConfig{wakeStarted: wakeStarted}) + mustStart(t, rt) - if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background()); err != nil { + if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background(), "wake"); err != nil { t.Fatalf("Arm: %v", err) } fakeClock.Advance(12 * time.Second) @@ -260,7 +399,7 @@ func TestSchedule_DropsCancellationOnRuntimeClose(t *testing.T) { t.Fatal("scheduled Wake did not start") } - rt.Close() + closeRuntime(rt) select { case got := <-errorsSeen: t.Fatalf("OnError received shutdown cancellation: %#v", got) @@ -269,10 +408,92 @@ func TestSchedule_DropsCancellationOnRuntimeClose(t *testing.T) { }) } -func newScheduledRuntime(t *testing.T, backend store.Store, sourceClock clock.Clock) *Runtime { +func TestSchedule_AbruptShutdownDoesNotWaitForReminderMethodThatIgnoresCancellation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(0, 0).UTC() + fakeClock := clock.NewFake(start) + backend := store.NewMemory() + wakeStarted := make(chan struct{}) + wakeRelease := make(chan struct{}) + wakeDone := make(chan struct{}) + released := false + defer func() { + if !released { + close(wakeRelease) + } + }() + + rt := newScheduledRuntime(t, backend, fakeClock) + installScheduledAccount(t, rt, new(atomic.Int32), scheduledAccountConfig{ + wakeStarted: wakeStarted, + wakeRelease: wakeRelease, + wakeDone: wakeDone, + }) + mustStart(t, rt) + if err := Ref[scheduledAccount](rt, "alice").Arm(context.Background(), "wake"); err != nil { + t.Fatalf("Arm: %v", err) + } + fakeClock.Advance(12 * time.Second) + synctest.Wait() + select { + case <-wakeStarted: + default: + t.Fatal("scheduled Wake did not start") + } + + shutdownContext, cancel := context.WithCancel(context.Background()) + cancel() + shutdownDone := make(chan error, 1) + go func() { + shutdownDone <- rt.Shutdown(shutdownContext) + }() + synctest.Wait() + select { + case err := <-shutdownDone: + if !errors.Is(err, context.Canceled) { + t.Fatalf("Shutdown error = %v, want context canceled", err) + } + default: + t.Fatal("abrupt Shutdown waited for a Reminder method that ignored cancellation") + } + select { + case <-wakeDone: + t.Fatal("Reminder method ended before the test released it") + default: + } + + close(wakeRelease) + released = true + synctest.Wait() + select { + case <-wakeDone: + default: + t.Fatal("Reminder method did not end after release") + } + }) +} + +type scheduledStore interface { + store.Store + store.ReminderStore +} + +type observedScheduledStore struct { + scheduledStore + scans chan struct{} +} + +func (s *observedScheduledStore) ListDue(ctx context.Context, now time.Time, after *store.ReminderCursor, limit int) (store.ReminderPage, error) { + page, err := s.scheduledStore.ListDue(ctx, now, after, limit) + s.scans <- struct{}{} + return page, err +} + +func newScheduledRuntime(t *testing.T, backend scheduledStore, sourceClock clock.Clock) *Runtime { t.Helper() rt := mustNew(t, WithStore(backend), + WithReminderStore(backend), WithClock(sourceClock), WithIdleTimeout(5*time.Second), WithEvictionInterval(time.Second), @@ -285,7 +506,11 @@ func TestSchedule_SetOverwritesAndCancelDeletes(t *testing.T) { start := time.Unix(0, 0).UTC() fakeClock := clock.NewFake(start) backend := store.NewMemory() - schedule := NewReminder[scheduledAccount](newTestBinder(GrainId{GrainType: "account", GrainKey: "alice"}, backend, backend, fakeClock)) + grain := newTestGrainContext(GrainId{GrainType: "account", GrainKey: "alice"}, backend, backend, fakeClock) + grain.runtime.grainTypes = map[GrainType]*typeRegistration{ + "account": {newReminderCall: newScheduledAccountReminderCall}, + } + schedule := NewReminder[scheduledAccount](grain) if err := schedule.Set(context.Background(), "wake", After(time.Second), Handle(scheduledAccount.Wake)); err != nil { t.Fatal(err) @@ -293,10 +518,7 @@ func TestSchedule_SetOverwritesAndCancelDeletes(t *testing.T) { if err := schedule.Set(context.Background(), "wake", Every(2*time.Second), Handle(scheduledAccount.Wake)); err != nil { t.Fatal(err) } - rows, err := backend.ListDue(context.Background(), start.Add(3*time.Second)) - if err != nil { - t.Fatal(err) - } + rows := readDueReminders(t, backend, start.Add(3*time.Second)) if len(rows) != 1 || rows[0].Method != "Wake" || rows[0].Interval != 2*time.Second || !rows[0].FirstTickTime.Equal(start.Add(2*time.Second)) || !rows[0].DueAt.Equal(start.Add(2*time.Second)) { t.Fatalf("overwritten schedule = %#v", rows) } @@ -304,17 +526,264 @@ func TestSchedule_SetOverwritesAndCancelDeletes(t *testing.T) { if err := schedule.Cancel(context.Background(), "wake"); err != nil { t.Fatal(err) } - rows, err = backend.ListDue(context.Background(), start.Add(3*time.Second)) + rows = readDueReminders(t, backend, start.Add(3*time.Second)) + if len(rows) != 0 { + t.Fatalf("reminders after cancel = %#v, want none", rows) + } +} + +func TestSchedule_ValidatesBeforeReminderStoreIO(t *testing.T) { + start := time.Unix(0, 0).UTC() + backend := &countingReminderStore{Memory: store.NewMemory()} + grain := newTestGrainContext(GrainId{GrainType: "account", GrainKey: "alice"}, backend, backend, clock.NewFake(start)) + grain.runtime.grainTypes = map[GrainType]*typeRegistration{ + "account": {newReminderCall: newScheduledAccountReminderCall}, + } + schedule := NewReminder[scheduledAccount](grain) + closure := func(scheduledAccount, context.Context, TickStatus) error { return nil } + + tests := []struct { + name string + when ReminderTime + handle MethodHandle[scheduledAccount] + }{ + {name: "", when: After(0), handle: Handle(scheduledAccount.Wake)}, + {name: " ", when: After(0), handle: Handle(scheduledAccount.Wake)}, + {name: "zero-handle", when: After(0), handle: MethodHandle[scheduledAccount]{}}, + {name: "nil-handle", when: After(0), handle: Handle[scheduledAccount](nil)}, + {name: "closure", when: After(0), handle: Handle(closure)}, + {name: "negative-due", when: After(-time.Nanosecond), handle: Handle(scheduledAccount.Wake)}, + {name: "negative-period", when: Every(-time.Nanosecond), handle: Handle(scheduledAccount.Wake)}, + } + for _, test := range tests { + if err := schedule.Set(context.Background(), test.name, test.when, test.handle); err == nil { + t.Fatalf("Set accepted invalid Reminder %q", test.name) + } + } + if got := backend.putCalls.Load(); got != 0 { + t.Fatalf("Put calls for invalid Reminders = %d, want 0", got) + } + for _, name := range []string{"", " "} { + if err := schedule.Cancel(context.Background(), name); err == nil { + t.Fatalf("Cancel accepted invalid Reminder name %q", name) + } + } + if got := backend.deleteCalls.Load(); got != 0 { + t.Fatalf("Delete calls for invalid Reminder names = %d, want 0", got) + } + + if err := schedule.Set(context.Background(), "once", Every(0), Handle(scheduledAccount.Wake)); err != nil { + t.Fatalf("Set Every(0): %v", err) + } + if got := backend.putCalls.Load(); got != 1 { + t.Fatalf("Put calls after valid Every(0) = %d, want 1", got) + } +} + +func TestSchedule_UnknownStoredRowsHaveOneTerminalReport(t *testing.T) { + tests := []struct { + name string + grainType string + method string + wantCode Code + }{ + {name: "unknown GrainType", grainType: "missing.Account", method: "Wake", wantCode: ErrTypeNotInstalled}, + {name: "unknown method", grainType: "gor.scheduledAccount", method: "Missing", wantCode: ErrUnknownMethod}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(0, 0).UTC() + fakeClock := clock.NewFake(start) + backend := &countingReminderStore{Memory: store.NewMemory()} + errorsSeen := make(chan BackgroundError, 4) + rt := mustNew(t, + WithStore(backend), + WithReminderStore(backend), + WithClock(fakeClock), + WithIdleTimeout(0), + WithEvictionInterval(0), + WithReminderInterval(time.Second), + OnError(func(event BackgroundError) { errorsSeen <- event }), + ) + defer closeRuntime(rt) + installScheduledAccount(t, rt, new(atomic.Int32), scheduledAccountConfig{}) + mustStart(t, rt) + + row := store.Reminder{ + GrainId: store.GrainId{GrainType: test.grainType, GrainKey: "alice"}, + Name: "wake", + Method: test.method, + FirstTickTime: start, + DueAt: start, + ETag: 1, + } + if err := backend.Memory.Put(context.Background(), row); err != nil { + t.Fatal(err) + } + fakeClock.Advance(time.Second) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + fakeClock.Advance(time.Second) + synctest.Wait() + + select { + case event := <-errorsSeen: + source, ok := event.Source.(ReminderDispatch) + wantID := GrainId{GrainType: GrainType(test.grainType), GrainKey: "alice"} + if !ok || event.GrainId != wantID || source.Name != row.Name || source.Method != row.Method || !errors.Is(event.Err, test.wantCode) { + t.Fatalf("event = %#v, want ReminderDispatch for %#v with %s", event, row, test.wantCode) + } + default: + t.Fatal("unknown stored Reminder was not reported") + } + if got := len(errorsSeen); got != 0 { + t.Fatalf("later dispatch failures = %d, want 0", got) + } + if got := backend.claimCalls.Load(); got != 1 { + t.Fatalf("terminal Claim calls = %d, want 1", got) + } + rows := readDueReminders(t, backend, start.Add(time.Second)) + if len(rows) != 0 { + t.Fatalf("due rows after terminal result = %#v, want none", rows) + } + }) + }) + } +} + +func TestSchedule_TerminalStoreErrorKeepsReminderIdentity(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Unix(0, 0).UTC() + fakeClock := clock.NewFake(start) + terminalErr := errors.New("terminal Store write failed") + backend := &terminalClaimErrorStore{Memory: store.NewMemory(), err: terminalErr} + errorsSeen := make(chan BackgroundError, 1) + rt := mustNew(t, + WithStore(backend), + WithReminderStore(backend), + WithClock(fakeClock), + WithIdleTimeout(0), + WithEvictionInterval(0), + WithReminderInterval(time.Second), + OnError(func(event BackgroundError) { errorsSeen <- event }), + ) + defer closeRuntime(rt) + installScheduledAccount(t, rt, new(atomic.Int32)) + mustStart(t, rt) + + row := store.Reminder{ + GrainId: store.GrainId{GrainType: "gor.scheduledAccount", GrainKey: "alice"}, + Name: "wake", + Method: "Missing", + FirstTickTime: start, + DueAt: start, + } + if err := backend.Put(context.Background(), row); err != nil { + t.Fatal(err) + } + fakeClock.Advance(time.Second) + synctest.Wait() + + event := <-errorsSeen + source, ok := event.Source.(ReminderTerminal) + wantID := GrainId{GrainType: "gor.scheduledAccount", GrainKey: "alice"} + if !ok || event.GrainId != wantID || source.Name != row.Name || source.Method != row.Method || !errors.Is(event.Err, terminalErr) { + t.Fatalf("event = %#v, want ReminderTerminal for %#v", event, row) + } + }) +} + +func TestSchedule_InvalidReminderTerminalResultSurvivesSQLiteRestart(t *testing.T) { + start := time.Unix(0, 0).UTC() + fakeClock := clock.NewFake(start) + databasePath := filepath.Join(t.TempDir(), "terminal-reminder.db") + row := store.Reminder{ + GrainId: store.GrainId{GrainType: "gor.scheduledAccount", GrainKey: "alice"}, + Name: "wake", + Method: "Missing", + FirstTickTime: start, + DueAt: start, + } + + firstStore, err := store.OpenSQLite(databasePath) if err != nil { t.Fatal(err) } - if len(rows) != 0 { - t.Fatalf("reminders after cancel = %#v, want none", rows) + firstErrors := make(chan BackgroundError, 1) + first := mustNew(t, + WithStore(firstStore), + WithReminderStore(firstStore), + WithClock(fakeClock), + WithIdleTimeout(0), + WithEvictionInterval(0), + WithReminderInterval(time.Second), + OnError(func(event BackgroundError) { firstErrors <- event }), + ) + installScheduledAccount(t, first, new(atomic.Int32)) + mustStart(t, first) + if err := firstStore.Put(context.Background(), row); err != nil { + killRuntime(first) + firstStore.Close() + t.Fatal(err) + } + fakeClock.Advance(time.Second) + event := <-firstErrors + if source, ok := event.Source.(ReminderDispatch); !ok || source.Name != row.Name || source.Method != row.Method { + killRuntime(first) + firstStore.Close() + t.Fatalf("first event = %#v, want ReminderDispatch for wake/Missing", event) + } + closeRuntime(first) + if err := firstStore.Close(); err != nil { + t.Fatalf("close first Store: %v", err) + } + + secondStore, err := store.OpenSQLite(databasePath) + if err != nil { + t.Fatal(err) + } + observed := &observedScheduledStore{scheduledStore: secondStore, scans: make(chan struct{}, 1)} + secondErrors := make(chan BackgroundError, 1) + second := mustNew(t, + WithStore(observed), + WithReminderStore(observed), + WithClock(fakeClock), + WithIdleTimeout(0), + WithEvictionInterval(0), + WithReminderInterval(time.Second), + OnError(func(event BackgroundError) { secondErrors <- event }), + ) + installScheduledAccount(t, second, new(atomic.Int32)) + mustStart(t, second) + fakeClock.Advance(time.Second) + <-observed.scans + if got := len(secondErrors); got != 0 { + killRuntime(second) + secondStore.Close() + t.Fatalf("errors after SQLite restart = %d, want 0", got) + } + closeRuntime(second) + if err := secondStore.Close(); err != nil { + t.Fatalf("close second Store: %v", err) + } +} + +func readDueReminders(t *testing.T, backend store.ReminderStore, now time.Time) []store.Reminder { + t.Helper() + page, err := backend.ListDue(context.Background(), now, nil, 1024) + if err != nil { + t.Fatalf("ListDue: %v", err) + } + if page.Next != nil { + t.Fatal("Reminder test helper limit is too small") } + return page.Rows } func TestSchedule_ReturnsUnavailableWithoutReminderStore(t *testing.T) { - schedule := NewReminder[scheduledAccount](newTestBinder(GrainId{GrainType: "account", GrainKey: "alice"}, failingWriteStore{}, nil, clock.Real{})) + schedule := NewReminder[scheduledAccount](newTestGrainContext(GrainId{GrainType: "account", GrainKey: "alice"}, failingWriteStore{}, nil, clock.Real{})) if err := schedule.Set(context.Background(), "wake", After(time.Second), Handle(scheduledAccount.Wake)); !errors.Is(err, ErrReminderStoreUnavailable) { t.Fatalf("Set error = %v, want ErrReminderStoreUnavailable", err) } @@ -329,34 +798,38 @@ func TestNew_ReminderStoreOptionIsOrderIndependent(t *testing.T) { if first.reminderStore != explicit { t.Fatal("WithStore replaced an earlier explicit ReminderStore") } - first.Close() + closeRuntime(first) second := mustNew(t, WithStore(failingWriteStore{}), WithReminderStore(explicit), WithReminderInterval(0), WithEvictionInterval(0)) if second.reminderStore != explicit { t.Fatal("WithReminderStore did not replace the default ReminderStore") } - second.Close() + closeRuntime(second) backend := store.NewMemory() - third := mustNew(t, WithStore(backend), WithReminderInterval(0), WithEvictionInterval(0)) - if third.reminderStore != backend { - t.Fatal("New did not derive ReminderStore from Store") + third, err := New(WithStore(backend), WithReminderInterval(0), WithEvictionInterval(0)) + if err != nil { + t.Fatal(err) + } + if err := third.Start(context.Background()); !errors.Is(err, ErrInvalidSetup) { + t.Fatalf("Start error = %v, want ErrInvalidSetup for missing Reminder Store", err) } - third.Close() + closeRuntime(third) } -func TestSchedule_ReactivatesEvictedEntity(t *testing.T) { +func TestSchedule_ReactivatesEvictedGrain(t *testing.T) { synctest.Test(t, func(t *testing.T) { start := time.Unix(0, 0).UTC() fakeClock := clock.NewFake(start) backend := store.NewMemory() factoryCalls := new(atomic.Int32) rt := newScheduledRuntime(t, backend, fakeClock) - defer rt.Close() + defer closeRuntime(rt) installScheduledAccount(t, rt, factoryCalls) + mustStart(t, rt) account := Ref[scheduledAccount](rt, "alice") - if err := account.Arm(context.Background()); err != nil { + if err := account.Arm(context.Background(), "wake"); err != nil { t.Fatal(err) } if got := factoryCalls.Load(); got != 1 { @@ -389,14 +862,16 @@ func TestSchedule_SurvivesRuntimeRestart(t *testing.T) { first := newScheduledRuntime(t, backend, fakeClock) installScheduledAccount(t, first, factoryCalls) - if err := Ref[scheduledAccount](first, "alice").Arm(context.Background()); err != nil { + mustStart(t, first) + if err := Ref[scheduledAccount](first, "alice").Arm(context.Background(), "wake"); err != nil { t.Fatal(err) } - first.Kill() + killRuntime(first) second := newScheduledRuntime(t, backend, fakeClock) - defer second.Close() + defer closeRuntime(second) installScheduledAccount(t, second, factoryCalls) + mustStart(t, second) synctest.Wait() fakeClock.Advance(12 * time.Second) synctest.Wait() @@ -407,3 +882,75 @@ func TestSchedule_SurvivesRuntimeRestart(t *testing.T) { } }) } + +func TestSchedule_PassesDynamicReminderNamesAfterSQLiteRestart(t *testing.T) { + start := time.Unix(0, 0).UTC() + fakeClock := clock.NewFake(start) + databasePath := filepath.Join(t.TempDir(), "reminders.db") + factoryCalls := new(atomic.Int32) + + firstStore, err := store.OpenSQLite(databasePath) + if err != nil { + t.Fatal(err) + } + first := newScheduledRuntime(t, firstStore, fakeClock) + firstClosed := false + t.Cleanup(func() { + if !firstClosed { + killRuntime(first) + _ = firstStore.Close() + } + }) + installScheduledAccount(t, first, factoryCalls) + mustStart(t, first) + account := Ref[scheduledAccount](first, "alice") + for _, name := range []string{"schedule/alpha", "schedule/beta"} { + if err := account.Arm(context.Background(), name); err != nil { + t.Fatalf("Arm %q: %v", name, err) + } + } + killRuntime(first) + if err := firstStore.Close(); err != nil { + t.Fatalf("close first Store: %v", err) + } + firstClosed = true + + secondStore, err := store.OpenSQLite(databasePath) + if err != nil { + t.Fatal(err) + } + ticks := make(chan TickStatus, 2) + second := newScheduledRuntime(t, secondStore, fakeClock) + secondClosed := false + t.Cleanup(func() { + if !secondClosed { + closeRuntime(second) + _ = secondStore.Close() + } + }) + installScheduledAccount(t, second, factoryCalls, scheduledAccountConfig{wakeTicks: ticks}) + mustStart(t, second) + fakeClock.Advance(12 * time.Second) + + wantNames := map[string]bool{"schedule/alpha": true, "schedule/beta": true} + for range 2 { + tick := <-ticks + if !wantNames[tick.ReminderName] { + t.Fatalf("ReminderName = %q, want one dynamic name", tick.ReminderName) + } + delete(wantNames, tick.ReminderName) + wantTime := start.Add(12 * time.Second) + if !tick.FirstTickTime.Equal(wantTime) || + !tick.CurrentTickTime.Equal(wantTime) || tick.Period != 0 { + t.Fatalf("TickStatus = %#v, want one-shot delivery after restart", tick) + } + } + if len(wantNames) != 0 { + t.Fatalf("missing Reminder names after restart: %v", wantNames) + } + closeRuntime(second) + if err := secondStore.Close(); err != nil { + t.Fatalf("close second Store: %v", err) + } + secondClosed = true +} diff --git a/sim/cluster.go b/sim/cluster.go index a31c204..972a697 100644 --- a/sim/cluster.go +++ b/sim/cluster.go @@ -41,10 +41,15 @@ type simulationCluster struct { driverLive []bool nodes []*clusterNode network *simulationNetwork + setup func(*gor.Runtime) error operationSequence atomic.Int64 } func newSimulationCluster(backend *fakeStore, count int, tracker *timerTracker) (*simulationCluster, error) { + return newSimulationClusterWithSetup(backend, count, tracker, nil) +} + +func newSimulationClusterWithSetup(backend *fakeStore, count int, tracker *timerTracker, setup func(*gor.Runtime) error) (*simulationCluster, error) { cluster := &simulationCluster{ backend: backend, tracker: tracker, @@ -53,6 +58,7 @@ func newSimulationCluster(backend *fakeStore, count int, tracker *timerTracker) driverLive: make([]bool, count), nodes: make([]*clusterNode, count), network: newSimulationNetwork(backend), + setup: setup, } for id := range cluster.driverLive { cluster.driverLive[id] = true @@ -72,9 +78,10 @@ func newSimulationCluster(backend *fakeStore, count int, tracker *timerTracker) func (c *simulationCluster) newRuntime(id, generation int) (*gor.Runtime, error) { addr := fmt.Sprintf("node-%d", id) members, network := c.network.addNode(addr) - return newCounterRuntimeWithOptions( + return newCounterRuntimeWithSetup( c.backend, c.tracker, + c.setup, gor.WithClock(c.clock), gor.WithMemberStore(members), gor.WithReminderStore(&nodeReminderStore{backend: c.backend, addr: addr}), @@ -97,7 +104,7 @@ func (c *simulationCluster) newRuntime(id, generation int) (*gor.Runtime, error) func (c *simulationCluster) close() { for _, node := range c.nodes { if node.rt != nil { - node.rt.Close() + _ = shutdownRuntime(node.rt) node.rt = nil } } @@ -111,7 +118,9 @@ func (c *simulationCluster) crash(id int) error { if node.rt == nil { return fmt.Errorf("node %d is already stopped", id) } - node.rt.Kill() + if err := crashRuntime(node.rt); err != nil { + return err + } node.rt = nil c.driverLive[id] = false return nil @@ -125,13 +134,14 @@ func (c *simulationCluster) leave(id int) error { if node.rt == nil { return fmt.Errorf("node %d is already stopped", id) } - done := make(chan struct{}) + done := make(chan error, 1) go func() { - node.rt.Close() - close(done) + done <- shutdownRuntime(node.rt) }() c.advance(10 * simulationStepDuration) - <-done + if err := <-done; err != nil { + return err + } node.rt = nil c.driverLive[id] = false return nil @@ -146,7 +156,9 @@ func (c *simulationCluster) restart(id int) error { if !runtimeStopped(node.rt) { return fmt.Errorf("node %d is already running", id) } - node.rt.Close() + if err := shutdownRuntime(node.rt); err != nil { + return err + } node.rt = nil } generation := c.generations[id] + 1 @@ -380,7 +392,7 @@ func executeDecisions(cluster *simulationCluster, decisions []decision, crashNod } } synctest.Wait() - // Kill lets the caller return while the entity method is still sleeping in + // Kill lets the caller return while the Grain method is still sleeping in // the store; if root exits, the fake clock stops and synctest reports a leak. cluster.backend.waitForIdle() cluster.network.waitForIdle() @@ -388,7 +400,7 @@ func executeDecisions(cluster *simulationCluster, decisions []decision, crashNod outcomes := make([]string, 0, len(decisions)) for range decisions { result := <-results - history.add(storeIdentity(result.id), result.operation) + history.add(storeIdgrain(result.id), result.operation) outcome, err := classifyOutcome(result.err) if err != nil { return nil, err diff --git a/sim/cluster_test.go b/sim/cluster_test.go index 96326d3..fd34325 100644 --- a/sim/cluster_test.go +++ b/sim/cluster_test.go @@ -14,13 +14,13 @@ import ( "github.com/suraciii/gor/store" ) -type dualActivationEntity struct { +type dualActivationGrain struct { value gor.State[int64] gate <-chan struct{} entered chan<- struct{} } -func (c *dualActivationEntity) Add(ctx context.Context, delta int64) (int64, error) { +func (c *dualActivationGrain) Add(ctx context.Context, delta int64) (int64, error) { c.entered <- struct{}{} <-c.gate next := c.value.Get() + delta @@ -30,15 +30,15 @@ func (c *dualActivationEntity) Add(ctx context.Context, delta int64) (int64, err return next, nil } -func (*dualActivationEntity) Arm(context.Context, string, time.Duration, time.Duration) error { +func (*dualActivationGrain) Arm(context.Context, string, time.Duration, time.Duration) error { return nil } -func (*dualActivationEntity) Disarm(context.Context, string) error { +func (*dualActivationGrain) Disarm(context.Context, string) error { return nil } -func (*dualActivationEntity) Tick(context.Context, gor.TickStatus) error { +func (*dualActivationGrain) Tick(context.Context, gor.TickStatus) error { return nil } @@ -48,17 +48,21 @@ func newDualActivationRuntime(backend *fakeStore, gate <-chan struct{}, entered return nil, err } if err := installCounterType(rt); err != nil { - rt.Close() + _ = shutdownRuntime(rt) return nil, err } - if err := registerCounter(rt, func(b *gor.Binder) counter { - return &dualActivationEntity{ + if err := registerCounter(rt, func(b *gor.GrainContext) counter { + return &dualActivationGrain{ value: gor.NewState[int64](b, "value"), gate: gate, entered: entered, } }); err != nil { - rt.Close() + _ = shutdownRuntime(rt) + return nil, err + } + if err := rt.Start(context.Background()); err != nil { + _ = shutdownRuntime(rt) return nil, err } return rt, nil @@ -75,13 +79,13 @@ func TestSim_DoubleActivationRejectsETagConflict(t *testing.T) { } second, err := newDualActivationRuntime(backend, gate, entered) if err != nil { - first.Close() + _ = shutdownRuntime(first) t.Fatal(err) } - defer first.Close() - defer second.Close() + defer shutdownRuntime(first) + defer shutdownRuntime(second) - id := gor.GrainId{GrainType: gor.TypeName[counter](), GrainKey: "dual"} + id := gor.GrainId{GrainType: gor.GrainType("sim.counter"), GrainKey: "dual"} firstCall := invokeAsync(first, id, 3) secondCall := invokeAsync(second, id, 5) synctest.Wait() @@ -108,7 +112,7 @@ func TestSim_DoubleActivationRejectsETagConflict(t *testing.T) { t.Fatalf("dual activation results = (%v, %v), want one success and one conflict", firstResult.err, secondResult.err) } - storeID := store.GrainId{GrainType: id.GrainType, GrainKey: id.GrainKey} + storeID := storeIdgrain(id) record := backend.snapshot([]store.GrainId{storeID})[storeID] if record.ETag != 1 { t.Fatalf("dual activation ETag = %d, want 1", record.ETag) @@ -137,10 +141,10 @@ func TestSim_NetworkPartitionCreatesDualActivationAndRecovers(t *testing.T) { defer cluster.close() cluster.advance(5 * simulationStepDuration) - counterType := gor.TypeName[counter]() - localID, remoteID := findPartitionIdentities(t, cluster, counterType) - local := gor.GrainId(localID) - remote := gor.GrainId(remoteID) + counterType := gor.GrainType("sim.counter") + localID, remoteID := findPartitionIdgrains(t, cluster, string(counterType)) + local := grainIdgrain(localID) + remote := grainIdgrain(remoteID) seed := awaitCall(invokeAsync(cluster.nodes[0].rt, local, 2)) if seed.err != nil || seed.value != 2 { @@ -208,9 +212,9 @@ func TestSim_NetworkPartitionCreatesDualActivationAndRecovers(t *testing.T) { t.Fatalf("healed local ownership = (%t, %t), want exactly one owner", cluster.nodes[0].rt.Owns(localID), cluster.nodes[1].rt.Owns(localID)) } - healedRemote := findIdentityOwnedBy(t, cluster, 1, counterType) + healedRemote := findIdgrainOwnedBy(t, cluster, 1, string(counterType)) sendsBefore, deliveredBefore, droppedBefore, _, _, _, _ := cluster.network.stats() - recovered := awaitCall(invokeAsync(cluster.nodes[0].rt, gor.GrainId(healedRemote), 7)) + recovered := awaitCall(invokeAsync(cluster.nodes[0].rt, grainIdgrain(healedRemote), 7)) if recovered.err != nil || recovered.value != 7 { t.Fatalf("healed forwarded call = (%d, %v), want (7, nil)", recovered.value, recovered.err) } @@ -221,11 +225,11 @@ func TestSim_NetworkPartitionCreatesDualActivationAndRecovers(t *testing.T) { }) } -func findPartitionIdentities(t *testing.T, cluster *simulationCluster, entityType string) (store.GrainId, store.GrainId) { +func findPartitionIdgrains(t *testing.T, cluster *simulationCluster, grainType string) (store.GrainId, store.GrainId) { t.Helper() var local, remote store.GrainId for index := 0; index < 4096 && (local == (store.GrainId{}) || remote == (store.GrainId{})); index++ { - id := store.GrainId{GrainType: entityType, GrainKey: fmt.Sprintf("partition-%04d", index)} + id := store.GrainId{GrainType: grainType, GrainKey: fmt.Sprintf("partition-%04d", index)} switch { case cluster.nodes[0].rt.Owns(id) && local == (store.GrainId{}): local = id @@ -239,10 +243,10 @@ func findPartitionIdentities(t *testing.T, cluster *simulationCluster, entityTyp return local, remote } -func findIdentityOwnedBy(t *testing.T, cluster *simulationCluster, nodeID int, entityType string) store.GrainId { +func findIdgrainOwnedBy(t *testing.T, cluster *simulationCluster, nodeID int, grainType string) store.GrainId { t.Helper() for index := 0; index < 4096; index++ { - id := store.GrainId{GrainType: entityType, GrainKey: fmt.Sprintf("partition-%04d", index)} + id := store.GrainId{GrainType: grainType, GrainKey: fmt.Sprintf("partition-%04d", index)} if cluster.nodes[nodeID].rt.Owns(id) { return id } @@ -259,25 +263,27 @@ func TestSim_CrashRestartRestoresState(t *testing.T) { t.Fatal(err) } - id := gor.GrainId{GrainType: gor.TypeName[counter](), GrainKey: "restart"} + id := gor.GrainId{GrainType: gor.GrainType("sim.counter"), GrainKey: "restart"} result := awaitCall(invokeAsync(rt, id, 4)) if result.err != nil || result.value != 4 { t.Fatalf("initial call = (%d, %v), want (4, nil)", result.value, result.err) } - rt.Kill() + if err := crashRuntime(rt); err != nil { + t.Fatal(err) + } synctest.Wait() rt, err = newCounterRuntimeWithOptions(backend, newTimerTracker()) if err != nil { t.Fatal(err) } - defer rt.Close() + defer shutdownRuntime(rt) result = awaitCall(invokeAsync(rt, id, 6)) if result.err != nil || result.value != 10 { t.Fatalf("restarted call = (%d, %v), want (10, nil)", result.value, result.err) } - storeID := store.GrainId{GrainType: id.GrainType, GrainKey: id.GrainKey} + storeID := storeIdgrain(id) record := backend.snapshot([]store.GrainId{storeID})[storeID] value, err := counterValue(record) if err != nil { diff --git a/sim/cycle_test.go b/sim/cycle_test.go index 6f11fef..606c1e4 100644 --- a/sim/cycle_test.go +++ b/sim/cycle_test.go @@ -13,7 +13,7 @@ import ( "github.com/suraciii/gor" ) -// cycleCaller walks a ring of entity keys: Chain calls the first key's Chain +// cycleCaller walks a ring of grain keys: Chain calls the first key's Chain // with the rest of the ring. A test closes the ring on a key already walked, // so the runtime must detect the cycle across the two nodes. type cycleCaller interface { @@ -26,11 +26,11 @@ type cycleCallerRequest struct { type cycleCallerReply struct{} -type cycleCallerEntity struct { - b *gor.Binder +type cycleCallerGrain struct { + b *gor.GrainContext } -func (e *cycleCallerEntity) Chain(ctx context.Context, ring []string) error { +func (e *cycleCallerGrain) Chain(ctx context.Context, ring []string) error { if len(ring) == 0 { return nil } @@ -64,36 +64,35 @@ func newCycleCallerCall(method string) (args any, reply any) { } } +func newCycleCallerReminderCall(string, gor.TickStatus) (any, any) { + return nil, nil +} + func installCycleCaller(rt *gor.Runtime) error { - if err := gor.InstallType[cycleCaller](rt, dispatchCycleCaller, func(invoker gor.Invoker, id gor.GrainId) cycleCaller { + if err := gor.InstallType[cycleCaller](rt, gor.GeneratedCodeVersion, "sim.cycleCaller", dispatchCycleCaller, func(invoker gor.Invoker, id gor.GrainId) cycleCaller { return &cycleCallerProxy{invoker: invoker, id: id} - }, newCycleCallerCall, nil); err != nil { + }, newCycleCallerCall, newCycleCallerReminderCall); err != nil { return err } - return gor.Register[cycleCaller](rt, func(b *gor.Binder) cycleCaller { - return &cycleCallerEntity{b: b} + return gor.Register[cycleCaller](rt, func(b *gor.GrainContext) cycleCaller { + return &cycleCallerGrain{b: b} }) } func TestSim_CallCycleAcrossNodesNamesTheCycle(t *testing.T) { synctest.Test(t, func(t *testing.T) { backend := newFakeStore(newTimerTracker()) - cluster, err := newSimulationCluster(backend, clusterNodeCount, newTimerTracker()) + cluster, err := newSimulationClusterWithSetup(backend, clusterNodeCount, newTimerTracker(), installCycleCaller) if err != nil { t.Fatal(err) } defer cluster.close() - for _, node := range cluster.nodes { - if err := installCycleCaller(node.rt); err != nil { - t.Fatal(err) - } - } cluster.advance(5 * simulationStepDuration) - cycleType := gor.TypeName[cycleCaller]() - local, remote := findPartitionIdentities(t, cluster, cycleType) - first := gor.GrainId(local) - second := gor.GrainId(remote) + cycleType := gor.GrainType("sim.cycleCaller") + local, remote := findPartitionIdgrains(t, cluster, string(cycleType)) + first := grainIdgrain(local) + second := grainIdgrain(remote) ctx, cancel := context.WithTimeout(context.Background(), 20*simulationStepDuration) defer cancel() diff --git a/sim/fault_test.go b/sim/fault_test.go index 7a2fb87..029dad3 100644 --- a/sim/fault_test.go +++ b/sim/fault_test.go @@ -14,12 +14,12 @@ import ( "github.com/suraciii/gor/store" ) -type gatedCounterEntity struct { +type gatedCounterGrain struct { value gor.State[int64] gate <-chan struct{} } -func (c *gatedCounterEntity) Add(ctx context.Context, delta int64) (int64, error) { +func (c *gatedCounterGrain) Add(ctx context.Context, delta int64) (int64, error) { <-c.gate next := c.value.Get() + delta if err := c.value.Set(ctx, next); err != nil { @@ -28,15 +28,15 @@ func (c *gatedCounterEntity) Add(ctx context.Context, delta int64) (int64, error return next, nil } -func (*gatedCounterEntity) Arm(context.Context, string, time.Duration, time.Duration) error { +func (*gatedCounterGrain) Arm(context.Context, string, time.Duration, time.Duration) error { return nil } -func (*gatedCounterEntity) Disarm(context.Context, string) error { +func (*gatedCounterGrain) Disarm(context.Context, string) error { return nil } -func (*gatedCounterEntity) Tick(context.Context, gor.TickStatus) error { +func (*gatedCounterGrain) Tick(context.Context, gor.TickStatus) error { return nil } @@ -51,6 +51,7 @@ func TestSim_FaultsAndMailbox(t *testing.T) { gate := make(chan struct{}) rt, err := gor.New( gor.WithStore(backend), + gor.WithReminderStore(backend), gor.WithIdleTimeout(0), gor.WithEvictionInterval(0), gor.WithMailboxCapacity(4), @@ -58,22 +59,25 @@ func TestSim_FaultsAndMailbox(t *testing.T) { if err != nil { t.Fatal(err) } - defer rt.Close() if err := installCounterType(rt); err != nil { t.Fatal(err) } - if err := registerCounter(rt, func(b *gor.Binder) counter { - return &gatedCounterEntity{ + if err := registerCounter(rt, func(b *gor.GrainContext) counter { + return &gatedCounterGrain{ value: gor.NewState[int64](b, "value"), gate: gate, } }); err != nil { t.Fatal(err) } + if err := rt.Start(context.Background()); err != nil { + t.Fatal(err) + } + defer shutdownRuntime(rt) - counterType := gor.TypeName[counter]() + counterType := gor.GrainType("sim.counter") id := gor.GrainId{GrainType: counterType, GrainKey: "mailbox"} - storeID := store.GrainId{GrainType: id.GrainType, GrainKey: id.GrainKey} + storeID := storeIdgrain(id) backend.setFaultPlans(nil) first := invokeAsync(rt, id, 1) @@ -81,12 +85,12 @@ func TestSim_FaultsAndMailbox(t *testing.T) { synctest.Wait() select { case <-first: - t.Fatal("first same-entity call completed before the gate opened") + t.Fatal("first same-grain call completed before the gate opened") default: } select { case <-second: - t.Fatal("second same-entity call completed before the gate opened") + t.Fatal("second same-grain call completed before the gate opened") default: } close(gate) @@ -94,12 +98,12 @@ func TestSim_FaultsAndMailbox(t *testing.T) { firstResult := <-first secondResult := <-second if firstResult.err != nil || secondResult.err != nil { - t.Fatalf("same-entity calls returned errors: %v, %v", firstResult.err, secondResult.err) + t.Fatalf("same-grain calls returned errors: %v, %v", firstResult.err, secondResult.err) } values := []int64{firstResult.value, secondResult.value} sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) if values[0] != 1 || values[1] != 2 { - t.Fatalf("same-entity results = %v, want [1 2]", values) + t.Fatalf("same-grain results = %v, want [1 2]", values) } observed := newObservations() @@ -128,7 +132,7 @@ func TestSim_FaultsAndMailbox(t *testing.T) { } readID := gor.GrainId{GrainType: counterType, GrainKey: "read-failure"} - readStoreID := store.GrainId{GrainType: readID.GrainType, GrainKey: readID.GrainKey} + readStoreID := storeIdgrain(readID) backend.setFaultPlans(map[store.GrainId]faultPlan{ readStoreID: {read: faultSpec{kind: faultReadError}}, }) @@ -146,7 +150,7 @@ func TestSim_FaultsAndMailbox(t *testing.T) { } appliedID := gor.GrainId{GrainType: counterType, GrainKey: "applied-error"} - appliedStoreID := store.GrainId{GrainType: appliedID.GrainType, GrainKey: appliedID.GrainKey} + appliedStoreID := storeIdgrain(appliedID) backend.setFaultPlans(map[store.GrainId]faultPlan{ appliedStoreID: {write: faultSpec{kind: faultWriteAppliedError}}, }) @@ -164,7 +168,7 @@ func TestSim_FaultsAndMailbox(t *testing.T) { } delayID := gor.GrainId{GrainType: counterType, GrainKey: "delay"} - delayStoreID := store.GrainId{GrainType: delayID.GrainType, GrainKey: delayID.GrainKey} + delayStoreID := storeIdgrain(delayID) backend.setFaultPlans(map[store.GrainId]faultPlan{ delayStoreID: {read: faultSpec{kind: faultDelay, delay: time.Millisecond}}, }) diff --git a/sim/history_test.go b/sim/history_test.go index 85eab5e..0b0ed57 100644 --- a/sim/history_test.go +++ b/sim/history_test.go @@ -31,8 +31,8 @@ func TestSim_DroppedReplyEffectReadByLaterCall(t *testing.T) { defer cluster.close() cluster.advance(2 * simulationStepDuration) - counterType := gor.TypeName[counter]() - _, remote := findPartitionIdentities(t, cluster, counterType) + counterType := gor.GrainType("sim.counter") + _, remote := findPartitionIdgrains(t, cluster, string(counterType)) if !cluster.nodes[1].rt.Owns(remote) { t.Fatalf("remote identity %s/%s is not owned by node 1", remote.GrainType, remote.GrainKey) } @@ -46,7 +46,7 @@ func TestSim_DroppedReplyEffectReadByLaterCall(t *testing.T) { // The literal timestamps order the two operations in real time: the // second call is awaited after the first, so the history's intervals // must not overlap. - first := awaitCall(invokeAsync(cluster.nodes[0].rt, gor.GrainId(remote), 1)) + first := awaitCall(invokeAsync(cluster.nodes[0].rt, grainIdgrain(remote), 1)) if !errors.Is(first.err, gor.ErrTransportFailed) { t.Fatalf("dropped-reply forward error = %v, want %v", first.err, gor.ErrTransportFailed) } @@ -66,7 +66,7 @@ func TestSim_DroppedReplyEffectReadByLaterCall(t *testing.T) { } cluster.network.setDrops(networkDropSpec{}) - second := awaitCall(invokeAsync(cluster.nodes[0].rt, gor.GrainId(remote), 1)) + second := awaitCall(invokeAsync(cluster.nodes[0].rt, grainIdgrain(remote), 1)) if second.err != nil || second.value != 2 { t.Fatalf("call after dropped reply = (%d, %v), want (2, nil)", second.value, second.err) } diff --git a/sim/log.go b/sim/log.go index f043c85..41a00b3 100644 --- a/sim/log.go +++ b/sim/log.go @@ -25,15 +25,15 @@ func (l *eventLog) addDecisionEvent(format string, args ...any) { } func (l *eventLog) addCallDecision(nodes []int, id store.GrainId, plan faultPlan, deltas []int64) { - l.addDecisionEvent("call nodes=[%s] entity=%s/%s deltas=[%s] fault=%s", formatIntList(nodes), id.GrainType, id.GrainKey, formatInt64List(deltas), plan.eventName()) + l.addDecisionEvent("call nodes=[%s] grain=%s/%s deltas=[%s] fault=%s", formatIntList(nodes), id.GrainType, id.GrainKey, formatInt64List(deltas), plan.eventName()) } func (l *eventLog) addScheduleDecision(node int, id store.GrainId, name string, delay, interval time.Duration, fault scheduleFaultSpec, member ...memberFaultSpec) { - l.addDecisionEvent("schedule node=%d entity=%s/%s name=%s after=%s every=%s fault=%s%s", node, id.GrainType, id.GrainKey, name, delay, interval, fault.eventName(), memberFaultSuffix(member)) + l.addDecisionEvent("schedule node=%d grain=%s/%s name=%s after=%s every=%s fault=%s%s", node, id.GrainType, id.GrainKey, name, delay, interval, fault.eventName(), memberFaultSuffix(member)) } func (l *eventLog) addDisarmDecision(node int, id store.GrainId, name string, member ...memberFaultSpec) { - l.addDecisionEvent("disarm node=%d entity=%s/%s name=%s%s", node, id.GrainType, id.GrainKey, name, memberFaultSuffix(member)) + l.addDecisionEvent("disarm node=%d grain=%s/%s name=%s%s", node, id.GrainType, id.GrainKey, name, memberFaultSuffix(member)) } func (l *eventLog) addCrashDecision(node int, member ...memberFaultSpec) { diff --git a/sim/network.go b/sim/network.go index aacdce9..044ec4f 100644 --- a/sim/network.go +++ b/sim/network.go @@ -326,7 +326,11 @@ func (t *simulationTransport) Send(ctx context.Context, addr string, payload []b func (t *simulationTransport) Close() error { t.closeOnce.Do(func() { close(t.closed) }) - <-t.done + select { + case <-t.served: + <-t.done + default: + } return nil } @@ -334,7 +338,11 @@ func (t *simulationTransport) Close() error { // gracefully, and in-flight handler results are dropped. func (t *simulationTransport) Kill() error { t.closeOnce.Do(func() { close(t.closed) }) - <-t.done + select { + case <-t.served: + <-t.done + default: + } return nil } @@ -420,8 +428,12 @@ type nodeReminderStore struct { var _ store.ReminderStore = (*nodeReminderStore)(nil) -func (s *nodeReminderStore) ListDue(ctx context.Context, now time.Time) ([]store.Reminder, error) { - return s.backend.listDueFor(ctx, s.addr, now) +func (s *nodeReminderStore) Check(ctx context.Context) error { + return s.backend.Check(ctx) +} + +func (s *nodeReminderStore) ListDue(ctx context.Context, now time.Time, after *store.ReminderCursor, limit int) (store.ReminderPage, error) { + return s.backend.listDueFor(ctx, s.addr, now, after, limit) } func (s *nodeReminderStore) Claim(ctx context.Context, schedule store.Reminder, nextDueAt time.Time) (bool, error) { diff --git a/sim/network_test.go b/sim/network_test.go index 1f833bc..c6ee70a 100644 --- a/sim/network_test.go +++ b/sim/network_test.go @@ -29,8 +29,8 @@ func TestSim_ForwardedReplyLandsAfterCallerDeadline(t *testing.T) { defer cluster.close() cluster.advance(2 * simulationStepDuration) - counterType := gor.TypeName[counter]() - _, remote := findPartitionIdentities(t, cluster, counterType) + counterType := gor.GrainType("sim.counter") + _, remote := findPartitionIdgrains(t, cluster, string(counterType)) if !cluster.nodes[1].rt.Owns(remote) { t.Fatalf("remote identity %s/%s is not owned by node 1", remote.GrainType, remote.GrainKey) } @@ -47,7 +47,7 @@ func TestSim_ForwardedReplyLandsAfterCallerDeadline(t *testing.T) { done := make(chan testCallResult, 1) go func() { var reply counterAddReply - err := cluster.nodes[0].rt.Invoke(ctx, gor.GrainId(remote), "Add", &counterAddRequest{A0: 1}, &reply) + err := cluster.nodes[0].rt.Invoke(ctx, grainIdgrain(remote), "Add", &counterAddRequest{A0: 1}, &reply) done <- testCallResult{value: reply.R0, err: err} }() // The bubble clock fires the caller's deadline, then releases the held @@ -80,7 +80,7 @@ func TestSim_ForwardedReplyLandsAfterCallerDeadline(t *testing.T) { } // TestSim_DroppedForwardedRequestNeverTakesEffect drops the request half of -// a forwarded call. The message never reaches the handler, so the entity's +// a forwarded call. The message never reaches the handler, so the grain's // state is untouched; the caller sees a transport failure, and no delivery // was ever started. This is the coarse side of per-message drop — partition // already produces "nothing happened" — asserted here so the request-side @@ -96,8 +96,8 @@ func TestSim_DroppedForwardedRequestNeverTakesEffect(t *testing.T) { defer cluster.close() cluster.advance(2 * simulationStepDuration) - counterType := gor.TypeName[counter]() - _, remote := findPartitionIdentities(t, cluster, counterType) + counterType := gor.GrainType("sim.counter") + _, remote := findPartitionIdgrains(t, cluster, string(counterType)) if !cluster.nodes[1].rt.Owns(remote) { t.Fatalf("remote identity %s/%s is not owned by node 1", remote.GrainType, remote.GrainKey) } @@ -110,7 +110,7 @@ func TestSim_DroppedForwardedRequestNeverTakesEffect(t *testing.T) { defer cluster.network.setDrops(networkDropSpec{}) sendsBefore, deliveredBefore, _, dropRequestsBefore, _, heldBefore, completedBefore := cluster.network.stats() - result := awaitCall(invokeAsync(cluster.nodes[0].rt, gor.GrainId(remote), 1)) + result := awaitCall(invokeAsync(cluster.nodes[0].rt, grainIdgrain(remote), 1)) if !errors.Is(result.err, gor.ErrTransportFailed) { t.Fatalf("dropped forward error = %v, want %v", result.err, gor.ErrTransportFailed) } @@ -135,7 +135,7 @@ func TestSim_DroppedForwardedRequestNeverTakesEffect(t *testing.T) { // TestSim_DroppedReplyStillTakesEffect drops the reply half of a forwarded // call. The handler already ran — the write landed — but the reply never -// reaches the caller, who sees a transport failure. The entity's state +// reaches the caller, who sees a transport failure. The grain's state // reflects the call while the caller does not know; this is the regime // partition cannot produce, where the request side of the exchange was never // silent. @@ -150,8 +150,8 @@ func TestSim_DroppedReplyStillTakesEffect(t *testing.T) { defer cluster.close() cluster.advance(2 * simulationStepDuration) - counterType := gor.TypeName[counter]() - _, remote := findPartitionIdentities(t, cluster, counterType) + counterType := gor.GrainType("sim.counter") + _, remote := findPartitionIdgrains(t, cluster, string(counterType)) if !cluster.nodes[1].rt.Owns(remote) { t.Fatalf("remote identity %s/%s is not owned by node 1", remote.GrainType, remote.GrainKey) } @@ -164,7 +164,7 @@ func TestSim_DroppedReplyStillTakesEffect(t *testing.T) { defer cluster.network.setDrops(networkDropSpec{}) sendsBefore, deliveredBefore, _, _, dropRepliesBefore, _, completedBefore := cluster.network.stats() - result := awaitCall(invokeAsync(cluster.nodes[0].rt, gor.GrainId(remote), 1)) + result := awaitCall(invokeAsync(cluster.nodes[0].rt, grainIdgrain(remote), 1)) if !errors.Is(result.err, gor.ErrTransportFailed) { t.Fatalf("dropped-reply forward error = %v, want %v", result.err, gor.ErrTransportFailed) } diff --git a/sim/observability_test.go b/sim/observability_test.go index eb8459b..cdffbc9 100644 --- a/sim/observability_test.go +++ b/sim/observability_test.go @@ -11,7 +11,6 @@ import ( "github.com/suraciii/gor" "github.com/suraciii/gor/clock" - "github.com/suraciii/gor/store" ) func TestSim_ForwardedCallProducesOneObservationAtOrigin(t *testing.T) { @@ -59,12 +58,12 @@ func TestSim_ForwardedCallProducesOneObservationAtOrigin(t *testing.T) { if err != nil { t.Fatal(err) } - defer first.Close() + defer shutdownRuntime(first) second, err := newCounterRuntimeWithOptions(backend, tracker, secondOptions...) if err != nil { t.Fatal(err) } - defer second.Close() + defer shutdownRuntime(second) synctest.Wait() <-firstTransport.served @@ -76,8 +75,8 @@ func TestSim_ForwardedCallProducesOneObservationAtOrigin(t *testing.T) { var remote gor.GrainId for index := 0; index < 4096; index++ { - candidate := gor.GrainId{GrainType: gor.TypeName[counter](), GrainKey: fmt.Sprintf("observed-%04d", index)} - if !first.Owns(store.GrainId(candidate)) && second.Owns(store.GrainId(candidate)) { + candidate := gor.GrainId{GrainType: gor.GrainType("sim.counter"), GrainKey: fmt.Sprintf("observed-%04d", index)} + if !first.Owns(storeIdgrain(candidate)) && second.Owns(storeIdgrain(candidate)) { remote = candidate break } @@ -96,7 +95,7 @@ func TestSim_ForwardedCallProducesOneObservationAtOrigin(t *testing.T) { select { case observation := <-firstEvents: - if observation.GrainType != gor.TypeName[counter]() || observation.Method != "Add" || observation.Err != nil { + if observation.GrainType != gor.GrainType("sim.counter") || observation.Method != "Add" || observation.Err != nil { t.Fatalf("origin observation = %#v, want Add with nil error", observation) } default: diff --git a/sim/probe_test.go b/sim/probe_test.go index afc2060..a3bce3d 100644 --- a/sim/probe_test.go +++ b/sim/probe_test.go @@ -174,7 +174,7 @@ func (c *probeScenario) advanceClock(duration time.Duration) { func (c *probeScenario) close() { for _, node := range c.nodes { if node != nil && node.rt != nil { - node.rt.Close() + _ = shutdownRuntime(node.rt) node.rt = nil } } @@ -365,7 +365,7 @@ func TestSim_ExpiredProbeVoteDoesNotKillHealthyNode(t *testing.T) { oldRule.enabled.Store(true) oldID := 0 fmt.Sscanf(oldVoter, "node-%d", &oldID) - scenario.nodes[oldID].rt.Kill() + _ = crashRuntime(scenario.nodes[oldID].rt) scenario.nodes[oldID].rt = nil scenario.advance(7 * simulationStepDuration) @@ -515,7 +515,7 @@ func TestSim_InterleavedPartitionCanKillAllAndNewGenerationRecovers(t *testing.T } } for _, node := range scenario.nodes { - node.rt.Close() + _ = shutdownRuntime(node.rt) node.rt = nil } replacement, err := scenario.addRuntime(0, 1, scenario.backend) diff --git a/sim/sim.go b/sim/sim.go index 812a21e..9797722 100644 --- a/sim/sim.go +++ b/sim/sim.go @@ -59,14 +59,14 @@ type counterTickRequest struct { type counterTickReply struct{} -type counterEntity struct { +type counterGrain struct { value gor.State[int64] id gor.GrainId schedule gor.Reminder[counter] tracker *timerTracker } -func (c *counterEntity) Add(ctx context.Context, delta int64) (int64, error) { +func (c *counterGrain) Add(ctx context.Context, delta int64) (int64, error) { next := c.value.Get() + delta if err := c.value.Set(ctx, next); err != nil { return 0, err @@ -74,7 +74,7 @@ func (c *counterEntity) Add(ctx context.Context, delta int64) (int64, error) { return next, nil } -func (c *counterEntity) Arm(ctx context.Context, name string, delay, interval time.Duration) error { +func (c *counterGrain) Arm(ctx context.Context, name string, delay, interval time.Duration) error { when := gor.After(delay) if interval > 0 { when = gor.Every(interval) @@ -82,12 +82,12 @@ func (c *counterEntity) Arm(ctx context.Context, name string, delay, interval ti return c.schedule.Set(ctx, name, when, gor.Handle(counter.Tick)) } -func (c *counterEntity) Disarm(ctx context.Context, name string) error { +func (c *counterGrain) Disarm(ctx context.Context, name string) error { return c.schedule.Cancel(ctx, name) } -func (c *counterEntity) Tick(context.Context, gor.TickStatus) error { - c.tracker.deliver(storeIdentity(c.id)) +func (c *counterGrain) Tick(context.Context, gor.TickStatus) error { + c.tracker.deliver(storeIdgrain(c.id)) return nil } @@ -161,12 +161,13 @@ func newCounterReminderCall(method string, status gor.TickStatus) (args any, rep } func installCounterType(rt *gor.Runtime) error { - return gor.InstallType[counter](rt, dispatchCounter, func(invoker gor.Invoker, id gor.GrainId) counter { + return gor.InstallType[counter](rt, gor.GeneratedCodeVersion, "sim.counter", dispatchCounter, func(invoker gor.Invoker, id gor.GrainId) counter { return &counterProxy{invoker: invoker, id: id} }, newCounterCall, newCounterReminderCall) + } -func registerCounter(rt *gor.Runtime, factory func(*gor.Binder) counter) error { +func registerCounter(rt *gor.Runtime, factory func(*gor.GrainContext) counter) error { return gor.Register[counter](rt, factory) } @@ -174,8 +175,8 @@ func installCounterWithTracker(rt *gor.Runtime, tracker *timerTracker) error { if err := installCounterType(rt); err != nil { return err } - return registerCounter(rt, func(b *gor.Binder) counter { - return &counterEntity{ + return registerCounter(rt, func(b *gor.GrainContext) counter { + return &counterGrain{ value: gor.NewState[int64](b, "value"), id: gor.Self(b), schedule: gor.NewReminder[counter](b), @@ -187,6 +188,7 @@ func installCounterWithTracker(rt *gor.Runtime, tracker *timerTracker) error { func baseRuntimeOptions(backend *fakeStore) []gor.Option { return []gor.Option{ gor.WithStore(backend), + gor.WithReminderStore(backend), gor.WithIdleTimeout(0), gor.WithEvictionInterval(0), gor.WithReminderInterval(simulationStepDuration), @@ -199,27 +201,59 @@ func newRuntime(backend *fakeStore) (*gor.Runtime, error) { } func newCounterRuntimeWithOptions(backend *fakeStore, tracker *timerTracker, options ...gor.Option) (*gor.Runtime, error) { + return newCounterRuntimeWithSetup(backend, tracker, nil, options...) +} + +func newCounterRuntimeWithSetup(backend *fakeStore, tracker *timerTracker, setup func(*gor.Runtime) error, options ...gor.Option) (*gor.Runtime, error) { rt, err := gor.New(append(baseRuntimeOptions(backend), options...)...) if err != nil { return nil, err } if err := installCounterWithTracker(rt, tracker); err != nil { - rt.Close() + _ = rt.Shutdown(context.Background()) + return nil, err + } + if setup != nil { + if err := setup(rt); err != nil { + _ = rt.Shutdown(context.Background()) + return nil, err + } + } + if err := rt.Start(context.Background()); err != nil { + _ = rt.Shutdown(context.Background()) return nil, err } return rt, nil } -func storeIdentity(id gor.GrainId) store.GrainId { - return store.GrainId{GrainType: id.GrainType, GrainKey: id.GrainKey} +func storeIdgrain(id gor.GrainId) store.GrainId { + return store.GrainId{GrainType: string(id.GrainType), GrainKey: id.GrainKey} +} + +func grainIdgrain(id store.GrainId) gor.GrainId { + return gor.GrainId{GrainType: gor.GrainType(id.GrainType), GrainKey: id.GrainKey} +} + +func shutdownRuntime(rt *gor.Runtime) error { + return rt.Shutdown(context.Background()) +} + +func crashRuntime(rt *gor.Runtime) error { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := rt.Shutdown(ctx) + if errors.Is(err, context.Canceled) { + return nil + } + return err } const probeCount = 64 -func probeIdentities(entityType string) []store.GrainId { +func probeIdgrains(grainType string) []store.GrainId { probes := make([]store.GrainId, probeCount) for index := range probes { - probes[index] = store.GrainId{GrainType: entityType, GrainKey: fmt.Sprintf("probe-%03d", index)} + probes[index] = store.GrainId{GrainType: grainType, GrainKey: fmt.Sprintf("probe-%03d", index)} } return probes } @@ -408,7 +442,7 @@ func simErrorIs(err, target error) bool { return errors.Is(err, target) } -func logEntityStates(log *eventLog, backend *fakeStore, ids []store.GrainId) error { +func logGrainStates(log *eventLog, backend *fakeStore, ids []store.GrainId) error { records := backend.snapshot(ids) ordered := append([]store.GrainId(nil), ids...) sort.Slice(ordered, func(i, j int) bool { @@ -491,13 +525,13 @@ func runSimulation(seed uint64, nodeCount int) (string, error) { } defer cluster.close() - counterType := gor.TypeName[counter]() - entities := []store.GrainId{ - {GrainType: counterType, GrainKey: "a"}, - {GrainType: counterType, GrainKey: "b"}, + counterType := gor.GrainType("sim.counter") + grains := []store.GrainId{ + {GrainType: string(counterType), GrainKey: "a"}, + {GrainType: string(counterType), GrainKey: "b"}, } - probes := probeIdentities(counterType) - checkedIdentities := append(append([]store.GrainId(nil), entities...), probes...) + probes := probeIdgrains(string(counterType)) + checkedIdgrains := append(append([]store.GrainId(nil), grains...), probes...) rng := rand.New(rand.NewPCG(seed, seed^0x517cc1b727220a95)) observations := newObservations() history := newCounterHistory() @@ -523,11 +557,11 @@ func runSimulation(seed uint64, nodeCount int) (string, error) { log.addNetworkDecision(networkDelay, networkDrop) switch action { case clusterCall: - entity := entities[rng.IntN(len(entities))] - id := gor.GrainId{GrainType: entity.GrainType, GrainKey: entity.GrainKey} + grain := grains[rng.IntN(len(grains))] + id := grainIdgrain(grain) plan := chooseFaultPlan(rng) plan.member = memberFault - storeID := storeIdentity(id) + storeID := storeIdgrain(id) cluster.backend.setFaultPlans(map[store.GrainId]faultPlan{storeID: plan}) cluster.backend.setMemberFault(plan.member) @@ -586,10 +620,10 @@ func runSimulation(seed uint64, nodeCount int) (string, error) { case clusterSchedule: liveIDs := cluster.liveNodeIDs() nodeID := liveIDs[rng.IntN(len(liveIDs))] - entity := entities[rng.IntN(len(entities))] - id := gor.GrainId{GrainType: entity.GrainType, GrainKey: entity.GrainKey} + grain := grains[rng.IntN(len(grains))] + id := grainIdgrain(grain) backend.setFaultPlans(nil) - name := "wake-" + entity.GrainKey + name := "wake-" + grain.GrainKey delay := time.Duration(rng.IntN(3)+1) * simulationStepDuration interval := time.Duration(0) if rng.IntN(2) == 1 { @@ -597,7 +631,7 @@ func runSimulation(seed uint64, nodeCount int) (string, error) { delay = interval } fault := chooseScheduleFault(rng, cluster) - log.addScheduleDecision(nodeID, entity, name, delay, interval, fault, memberFault) + log.addScheduleDecision(nodeID, grain, name, delay, interval, fault, memberFault) err := cluster.nodes[nodeID].rt.Invoke(context.Background(), id, "Arm", &counterArmRequest{A0: name, A1: delay, A2: interval}, &counterArmReply{}) outcome, classifyErr := classifyOutcome(err) if classifyErr != nil { @@ -605,16 +639,16 @@ func runSimulation(seed uint64, nodeCount int) (string, error) { } log.addScheduleOutcome("schedule", nodeID, outcome) if err == nil { - backend.setScheduleFault(storeIdentity(id), fault) + backend.setScheduleFault(storeIdgrain(id), fault) } case clusterDisarm: liveIDs := cluster.liveNodeIDs() nodeID := liveIDs[rng.IntN(len(liveIDs))] - entity := entities[rng.IntN(len(entities))] - id := gor.GrainId{GrainType: entity.GrainType, GrainKey: entity.GrainKey} - name := "wake-" + entity.GrainKey + grain := grains[rng.IntN(len(grains))] + id := grainIdgrain(grain) + name := "wake-" + grain.GrainKey backend.setFaultPlans(nil) - log.addDisarmDecision(nodeID, entity, name, memberFault) + log.addDisarmDecision(nodeID, grain, name, memberFault) err := cluster.nodes[nodeID].rt.Invoke(context.Background(), id, "Disarm", &counterDisarmRequest{A0: name}, &counterDisarmReply{}) outcome, classifyErr := classifyOutcome(err) if classifyErr != nil { @@ -624,10 +658,10 @@ func runSimulation(seed uint64, nodeCount int) (string, error) { } synctest.Wait() - if err := observations.check(backend, entities); err != nil { + if err := observations.check(backend, grains); err != nil { return log.String(), fmt.Errorf("step %d: %w", step, err) } - if err := logEntityStates(&log, backend, entities); err != nil { + if err := logGrainStates(&log, backend, grains); err != nil { return log.String(), fmt.Errorf("step %d: %w", step, err) } if err := history.check(); err != nil { @@ -639,7 +673,7 @@ func runSimulation(seed uint64, nodeCount int) (string, error) { } cluster.advance(simulationStepDuration) cluster.settle() - if err := cluster.checkInvariants(checkedIdentities); err != nil { + if err := cluster.checkInvariants(checkedIdgrains); err != nil { return log.String(), fmt.Errorf("step %d: %w", step, err) } log.addMemberObservation(backend.memberStatsSnapshot()) diff --git a/sim/store.go b/sim/store.go index 763d0bc..4ff3191 100644 --- a/sim/store.go +++ b/sim/store.go @@ -77,7 +77,7 @@ const ( // scheduleFaultSpec carries the seed-drawn target for the list kinds: the // fault fires only on a ListDue issued by targetNode's poller. Claim kinds are -// keyed by entity identity and take no target. +// keyed by GrainId and take no target. type scheduleFaultSpec struct { kind scheduleFaultKind targetNode int @@ -145,6 +145,7 @@ type fakeStore struct { members map[fakeMemberKey]store.Member memberFault memberFaultSpec reminders map[reminderKey]store.Reminder + reminderETag store.ETag scheduleListFault scheduleFaultSpec scheduleClaimFaults map[store.GrainId]scheduleFaultKind timerTracker *timerTracker @@ -178,6 +179,10 @@ func newFakeStore(tracker *timerTracker) *fakeStore { } } +func (s *fakeStore) Check(ctx context.Context) error { + return ctx.Err() +} + func (s *fakeStore) setMemberClock(memberClock clock.Clock) { s.mu.Lock() s.memberClock = memberClock @@ -408,34 +413,23 @@ func (s *fakeStore) checkMemberStatuses() error { return nil } -func (s *fakeStore) snapshotDue(now time.Time, caller string) ([]store.Reminder, scheduleFaultSpec) { +func (s *fakeStore) snapshotDue(now time.Time, caller string, after *store.ReminderCursor, limit int) (store.ReminderPage, scheduleFaultSpec) { s.mu.Lock() defer s.mu.Unlock() s.stats.listCalls++ - result := make([]store.Reminder, 0) + readLimit := scheduleReadLimit(limit) + result := make([]store.Reminder, 0, min(readLimit, len(s.reminders))) for _, schedule := range s.reminders { - if !schedule.DueAt.After(now) { - result = append(result, schedule) + if !schedule.DueAt.After(now) && scheduleAfterCursor(schedule, after) { + result = insertBoundedSchedule(result, schedule, readLimit) } } - sort.Slice(result, func(i, j int) bool { - if !result[i].DueAt.Equal(result[j].DueAt) { - return result[i].DueAt.Before(result[j].DueAt) - } - if result[i].GrainId.GrainType != result[j].GrainId.GrainType { - return result[i].GrainId.GrainType < result[j].GrainId.GrainType - } - if result[i].GrainId.GrainKey != result[j].GrainId.GrainKey { - return result[i].GrainId.GrainKey < result[j].GrainId.GrainKey - } - return result[i].Name < result[j].Name - }) fault := s.scheduleListFault if fault.kind != scheduleFaultNone && caller != nodeAddress(fault.targetNode) { - return result, scheduleFaultSpec{} + return schedulePage(result, limit), scheduleFaultSpec{} } if fault.kind == scheduleListDelay && len(result) == 0 { - return result, scheduleFaultSpec{} + return schedulePage(result, limit), scheduleFaultSpec{} } s.scheduleListFault = scheduleFaultSpec{} switch fault.kind { @@ -444,18 +438,21 @@ func (s *fakeStore) snapshotDue(now time.Time, caller string) ([]store.Reminder, case scheduleListDelay: s.stats.listDelays++ } - return result, fault + return schedulePage(result, limit), fault } -func (s *fakeStore) ListDue(ctx context.Context, now time.Time) ([]store.Reminder, error) { - return s.listDueFor(ctx, "", now) +func (s *fakeStore) ListDue(ctx context.Context, now time.Time, after *store.ReminderCursor, limit int) (store.ReminderPage, error) { + return s.listDueFor(ctx, "", now, after, limit) } -func (s *fakeStore) listDueFor(_ context.Context, caller string, now time.Time) ([]store.Reminder, error) { +func (s *fakeStore) listDueFor(_ context.Context, caller string, now time.Time, after *store.ReminderCursor, limit int) (store.ReminderPage, error) { defer s.endOperation(s.beginOperation()) - result, fault := s.snapshotDue(now, caller) + if limit <= 0 { + return store.ReminderPage{}, errors.New("sim: Reminder page limit must be positive") + } + result, fault := s.snapshotDue(now, caller, after, limit) if fault.kind == scheduleListError { - return nil, errScheduleListFailure + return store.ReminderPage{}, errScheduleListFailure } if fault.kind == scheduleListDelay { s.recordDelay() @@ -464,10 +461,75 @@ func (s *fakeStore) listDueFor(_ context.Context, caller string, now time.Time) return result, nil } +func scheduleAfterCursor(reminder store.Reminder, cursor *store.ReminderCursor) bool { + if cursor == nil { + return true + } + if !reminder.DueAt.Equal(cursor.DueAt) { + return reminder.DueAt.After(cursor.DueAt) + } + if reminder.GrainId.GrainType != cursor.GrainId.GrainType { + return reminder.GrainId.GrainType > cursor.GrainId.GrainType + } + if reminder.GrainId.GrainKey != cursor.GrainId.GrainKey { + return reminder.GrainId.GrainKey > cursor.GrainId.GrainKey + } + return reminder.Name > cursor.Name +} + +func scheduleLess(left store.Reminder, right store.Reminder) bool { + if !left.DueAt.Equal(right.DueAt) { + return left.DueAt.Before(right.DueAt) + } + if left.GrainId.GrainType != right.GrainId.GrainType { + return left.GrainId.GrainType < right.GrainId.GrainType + } + if left.GrainId.GrainKey != right.GrainId.GrainKey { + return left.GrainId.GrainKey < right.GrainId.GrainKey + } + return left.Name < right.Name +} + +func insertBoundedSchedule(rows []store.Reminder, reminder store.Reminder, maximum int) []store.Reminder { + index := sort.Search(len(rows), func(index int) bool { + return !scheduleLess(rows[index], reminder) + }) + if index >= maximum { + return rows + } + if len(rows) < maximum { + rows = append(rows, store.Reminder{}) + } + copy(rows[index+1:], rows[index:len(rows)-1]) + rows[index] = reminder + return rows +} + +func schedulePage(rows []store.Reminder, limit int) store.ReminderPage { + if len(rows) <= limit { + return store.ReminderPage{Rows: rows} + } + rows = rows[:limit] + last := rows[len(rows)-1] + return store.ReminderPage{ + Rows: rows, + Next: &store.ReminderCursor{DueAt: last.DueAt, GrainId: last.GrainId, Name: last.Name}, + } +} + +func scheduleReadLimit(limit int) int { + maximumInt := int(^uint(0) >> 1) + if limit == maximumInt { + return limit + } + return limit + 1 +} + func (s *fakeStore) Claim(_ context.Context, schedule store.Reminder, nextDueAt time.Time) (bool, error) { defer s.endOperation(s.beginOperation()) s.mu.Lock() - current, ok := s.reminders[reminderKey{identity: schedule.GrainId, name: schedule.Name}] + key := reminderKey{identity: schedule.GrainId, name: schedule.Name} + current, ok := s.reminders[key] if !ok || current.ETag != schedule.ETag { s.stats.claimLost++ s.mu.Unlock() @@ -481,11 +543,11 @@ func (s *fakeStore) Claim(_ context.Context, schedule store.Reminder, nextDueAt return false, errScheduleClaimFailure } if nextDueAt.IsZero() { - delete(s.reminders, reminderKey{identity: schedule.GrainId, name: schedule.Name}) + delete(s.reminders, key) } else { current.DueAt = nextDueAt - current.ETag++ - s.reminders[reminderKey{identity: schedule.GrainId, name: schedule.Name}] = current + current.ETag = s.nextReminderETag() + s.reminders[key] = current } s.stats.claimWon++ if fault == scheduleClaimAppliedError { @@ -503,11 +565,7 @@ func (s *fakeStore) Put(_ context.Context, schedule store.Reminder) error { defer s.endOperation(s.beginOperation()) s.mu.Lock() key := reminderKey{identity: schedule.GrainId, name: schedule.Name} - current, ok := s.reminders[key] - schedule.ETag = 1 - if ok { - schedule.ETag = current.ETag + 1 - } + schedule.ETag = s.nextReminderETag() s.reminders[key] = schedule s.mu.Unlock() return nil @@ -516,11 +574,17 @@ func (s *fakeStore) Put(_ context.Context, schedule store.Reminder) error { func (s *fakeStore) Delete(_ context.Context, id store.GrainId, name string) error { defer s.endOperation(s.beginOperation()) s.mu.Lock() - delete(s.reminders, reminderKey{identity: id, name: name}) + key := reminderKey{identity: id, name: name} + delete(s.reminders, key) s.mu.Unlock() return nil } +func (s *fakeStore) nextReminderETag() store.ETag { + s.reminderETag++ + return s.reminderETag +} + func (s *fakeStore) recordDelay() { s.mu.Lock() s.delays++ diff --git a/sim/store_test.go b/sim/store_test.go new file mode 100644 index 0000000..c435372 --- /dev/null +++ b/sim/store_test.go @@ -0,0 +1,47 @@ +//go:build sim + +package sim + +import ( + "context" + "testing" + "time" + + "github.com/suraciii/gor/store" +) + +func TestSim_FakeStoreReminderETagSurvivesDelete(t *testing.T) { + backend := newFakeStore(newTimerTracker()) + ctx := context.Background() + now := time.Unix(10, 0).UTC() + setting := store.Reminder{ + GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + DueAt: now, + } + if err := backend.Put(ctx, setting); err != nil { + t.Fatalf("Put old setting: %v", err) + } + oldPage, err := backend.ListDue(ctx, now, nil, 1) + if err != nil || len(oldPage.Rows) != 1 { + t.Fatalf("ListDue old setting = (%#v, %v), want one row", oldPage.Rows, err) + } + old := oldPage.Rows[0] + if err := backend.Delete(ctx, setting.GrainId, setting.Name); err != nil { + t.Fatalf("Delete old setting: %v", err) + } + if err := backend.Put(ctx, setting); err != nil { + t.Fatalf("Put new setting: %v", err) + } + newPage, err := backend.ListDue(ctx, now, nil, 1) + if err != nil || len(newPage.Rows) != 1 { + t.Fatalf("ListDue new setting = (%#v, %v), want one row", newPage.Rows, err) + } + if newPage.Rows[0].ETag <= old.ETag { + t.Fatalf("new ETag = %d, want newer than %d", newPage.Rows[0].ETag, old.ETag) + } + if won, err := backend.Claim(ctx, old, now.Add(time.Hour)); err != nil || won { + t.Fatalf("Claim old generation = (%v, %v), want (false, nil)", won, err) + } +} diff --git a/startup_test.go b/startup_test.go new file mode 100644 index 0000000..a514d29 --- /dev/null +++ b/startup_test.go @@ -0,0 +1,667 @@ +package gor + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/suraciii/gor/clock" + "github.com/suraciii/gor/store" + "github.com/suraciii/gor/transport" +) + +func TestRuntimeNewStartsNoWork(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + backend := store.NewMemory() + stateStore := &startupStateStore{backend: backend} + reminderStore := &startupReminderStore{backend: backend} + sourceClock := &startupClock{Fake: clock.NewFake(time.Unix(0, 0).UTC())} + configuredTransport := &startupTransport{} + + rt, err := New( + WithStore(stateStore), + WithReminderStore(reminderStore), + WithClock(sourceClock), + WithMemberStore(backend), + WithTransport(configuredTransport), + ) + if err != nil { + t.Fatal(err) + } + synctest.Wait() + + if got := stateStore.totalCalls(); got != 0 { + t.Fatalf("State Store calls before Start = %d, want 0", got) + } + if got := reminderStore.totalCalls(); got != 0 { + t.Fatalf("Reminder Store calls before Start = %d, want 0", got) + } + if got := sourceClock.tickers.Load(); got != 0 { + t.Fatalf("Clock tickers before Start = %d, want 0", got) + } + if got := configuredTransport.serves.Load(); got != 0 { + t.Fatalf("Transport Serve calls before Start = %d, want 0", got) + } + assertChannelOpen(t, "Stopping before Shutdown", rt.Stopping()) + assertChannelOpen(t, "Done before Shutdown", rt.Done()) + + if err := rt.Shutdown(context.Background()); err != nil { + t.Fatalf("Shutdown constructed Runtime: %v", err) + } + assertChannelClosed(t, "Stopping after Shutdown", rt.Stopping()) + assertChannelClosed(t, "Done after Shutdown", rt.Done()) + if got := configuredTransport.closes.Load(); got != 1 { + t.Fatalf("Transport Close calls = %d, want 1", got) + } + if got := configuredTransport.kills.Load(); got != 0 { + t.Fatalf("Transport Kill calls = %d, want 0", got) + } + }) +} + +func TestRuntimeCallBeforeStartHandlesTypedNilClock(t *testing.T) { + backend := store.NewMemory() + var sourceClock *startupClock + observations := new(atomic.Int32) + rt, err := New( + WithStore(backend), + WithReminderStore(backend), + WithClock(sourceClock), + OnCall(func(CallObservation) { observations.Add(1) }), + ) + if err != nil { + t.Fatal(err) + } + + err = rt.Invoke(context.Background(), GrainId{GrainType: "gor.Account", GrainKey: "alice"}, "Balance", nil, nil) + if !errors.Is(err, ErrRuntimeNotStarted) { + t.Fatalf("Invoke before Start = %v, want ErrRuntimeNotStarted", err) + } + if got := observations.Load(); got != 1 { + t.Fatalf("Call observations = %d, want 1", got) + } + if err := rt.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestRuntimeStartChecksStoresBeforeAcceptingCalls(t *testing.T) { + backend := store.NewMemory() + stateEntered := make(chan struct{}) + stateRelease := make(chan struct{}) + reminderEntered := make(chan struct{}) + reminderRelease := make(chan struct{}) + stateStore := &startupStateStore{ + backend: backend, + check: func(ctx context.Context) error { + close(stateEntered) + select { + case <-stateRelease: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }, + } + reminderStore := &startupReminderStore{ + backend: backend, + check: func(ctx context.Context) error { + close(reminderEntered) + select { + case <-reminderRelease: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }, + } + rt, err := New( + WithStore(stateStore), + WithReminderStore(reminderStore), + WithIdleTimeout(0), + WithEvictionInterval(0), + WithReminderInterval(0), + ) + if err != nil { + t.Fatal(err) + } + registerAccount(t, rt) + startDone := make(chan error, 1) + go func() { startDone <- rt.Start(context.Background()) }() + + <-stateEntered + assertRuntimeNotStarted(t, rt) + if got := reminderStore.checks.Load(); got != 0 { + t.Fatalf("Reminder Store checks while State Store is blocked = %d, want 0", got) + } + close(stateRelease) + <-reminderEntered + assertRuntimeNotStarted(t, rt) + if got := reminderStore.listCalls.Load() + reminderStore.claimCalls.Load(); got != 0 { + t.Fatalf("Reminder work before Start completed = %d calls, want 0", got) + } + close(reminderRelease) + if err := <-startDone; err != nil { + t.Fatalf("Start: %v", err) + } + if _, err := Ref[Account](rt, "alice").Balance(context.Background()); err != nil { + t.Fatalf("Call after Start: %v", err) + } + if err := rt.Shutdown(context.Background()); err != nil { + t.Fatalf("Shutdown: %v", err) + } +} + +func TestRuntimeSuccessfulStartFreezesSetup(t *testing.T) { + rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0), WithReminderInterval(0)) + registerAccount(t, rt) + if err := rt.Start(context.Background()); err != nil { + t.Fatal(err) + } + defer closeRuntime(rt) + + if err := InstallType[scopeAccount](rt, GeneratedCodeVersion, "gor.scopeAccount", dispatchScopeAccount, func(invoker Invoker, id GrainId) scopeAccount { + return &scopeAccountProxy{invoker: invoker, id: id} + }, newScopeAccountCall, noReminderCall); !errors.Is(err, ErrSetupFrozen) { + t.Fatalf("InstallType after Start = %v, want ErrSetupFrozen", err) + } + if err := Register[Account](rt, func(grain *GrainContext) Account { + return &account{value: NewState[int64](grain, "value")} + }); !errors.Is(err, ErrSetupFrozen) { + t.Fatalf("Register after Start = %v, want ErrSetupFrozen", err) + } + if err := rt.Start(context.Background()); !errors.Is(err, ErrSetupFrozen) { + t.Fatalf("second Start = %v, want ErrSetupFrozen", err) + } +} + +func TestRuntimeFailedStartFreezesSetupAndClaimsNoReminder(t *testing.T) { + checkErr := errors.New("State Store is unavailable") + backend := store.NewMemory() + stateStore := &startupStateStore{backend: backend, check: func(context.Context) error { return checkErr }} + reminderStore := &startupReminderStore{backend: backend} + configuredTransport := &startupTransport{} + rt, err := New( + WithStore(stateStore), + WithReminderStore(reminderStore), + WithMemberStore(backend), + WithTransport(configuredTransport), + ) + if err != nil { + t.Fatal(err) + } + registerAccount(t, rt) + + err = rt.Start(context.Background()) + if !errors.Is(err, ErrInvalidSetup) || !errors.Is(err, checkErr) { + t.Fatalf("Start error = %v, want ErrInvalidSetup and Store error", err) + } + if got := reminderStore.totalCalls(); got != 0 { + t.Fatalf("Reminder Store calls after failed State check = %d, want 0", got) + } + assertRuntimeNotStarted(t, rt) + if err := InstallType[scopeAccount](rt, GeneratedCodeVersion, "gor.scopeAccount", dispatchScopeAccount, func(invoker Invoker, id GrainId) scopeAccount { + return &scopeAccountProxy{invoker: invoker, id: id} + }, newScopeAccountCall, nil); !errors.Is(err, ErrSetupFrozen) { + t.Fatalf("InstallType after failed Start = %v, want ErrSetupFrozen", err) + } + if err := Register[Account](rt, func(grain *GrainContext) Account { + return &account{value: NewState[int64](grain, "value")} + }); !errors.Is(err, ErrSetupFrozen) { + t.Fatalf("Register after failed Start = %v, want ErrSetupFrozen", err) + } + if err := rt.Start(context.Background()); !errors.Is(err, ErrSetupFrozen) { + t.Fatalf("second Start = %v, want ErrSetupFrozen", err) + } + assertChannelClosed(t, "Stopping after failed Start", rt.Stopping()) + assertChannelClosed(t, "Done after failed Start", rt.Done()) + if got := configuredTransport.kills.Load(); got != 1 { + t.Fatalf("Transport Kill calls after failed Start = %d, want 1", got) + } + if got := configuredTransport.serves.Load(); got != 0 { + t.Fatalf("Transport Serve calls after failed Start = %d, want 0", got) + } + if err := rt.Shutdown(context.Background()); err != nil { + t.Fatalf("Shutdown failed Runtime: %v", err) + } +} + +func TestRuntimeStartRejectsInvalidConfiguration(t *testing.T) { + tests := []struct { + name string + options func(*store.Memory) []Option + text string + }{ + {name: "State Store", options: func(memory *store.Memory) []Option { return []Option{WithReminderStore(memory)} }, text: "State Store is required"}, + {name: "Reminder Store", options: func(memory *store.Memory) []Option { return []Option{WithStore(memory)} }, text: "Reminder Store is required"}, + {name: "Clock", options: func(memory *store.Memory) []Option { + return []Option{WithStore(memory), WithReminderStore(memory), WithClock(nil)} + }, text: "Clock is required"}, + {name: "zero mailbox capacity", options: func(memory *store.Memory) []Option { + return []Option{WithStore(memory), WithReminderStore(memory), WithMailboxCapacity(0)} + }, text: "mailbox capacity"}, + {name: "negative mailbox capacity", options: func(memory *store.Memory) []Option { + return []Option{WithStore(memory), WithReminderStore(memory), WithMailboxCapacity(-1)} + }, text: "mailbox capacity"}, + {name: "zero activation limit", options: func(memory *store.Memory) []Option { + return []Option{WithStore(memory), WithReminderStore(memory), WithMaxActivations(0)} + }, text: "activation limit"}, + {name: "negative activation limit", options: func(memory *store.Memory) []Option { + return []Option{WithStore(memory), WithReminderStore(memory), WithMaxActivations(-1)} + }, text: "activation limit"}, + {name: "idle timeout", options: func(memory *store.Memory) []Option { + return []Option{WithStore(memory), WithReminderStore(memory), WithIdleTimeout(-1)} + }, text: "idle timeout"}, + {name: "eviction interval", options: func(memory *store.Memory) []Option { + return []Option{WithStore(memory), WithReminderStore(memory), WithEvictionInterval(-1)} + }, text: "eviction interval"}, + {name: "Reminder interval", options: func(memory *store.Memory) []Option { + return []Option{WithStore(memory), WithReminderStore(memory), WithReminderInterval(-1)} + }, text: "Reminder interval"}, + {name: "zero Reminder page size", options: func(memory *store.Memory) []Option { + return []Option{WithStore(memory), WithReminderStore(memory), WithReminderPageSize(0)} + }, text: "Reminder page size"}, + {name: "negative Reminder page size", options: func(memory *store.Memory) []Option { + return []Option{WithStore(memory), WithReminderStore(memory), WithReminderPageSize(-1)} + }, text: "Reminder page size"}, + {name: "zero Reminder worker limit", options: func(memory *store.Memory) []Option { + return []Option{WithStore(memory), WithReminderStore(memory), WithReminderWorkers(0)} + }, text: "Reminder worker limit"}, + {name: "negative Reminder worker limit", options: func(memory *store.Memory) []Option { + return []Option{WithStore(memory), WithReminderStore(memory), WithReminderWorkers(-1)} + }, text: "Reminder worker limit"}, + {name: "typed nil State Store", options: func(memory *store.Memory) []Option { + var stateStore *startupStateStore + return []Option{WithStore(stateStore), WithReminderStore(memory)} + }, text: "State Store is required"}, + {name: "typed nil Transport", options: func(memory *store.Memory) []Option { + var configuredTransport *startupTransport + return []Option{WithStore(memory), WithReminderStore(memory), WithMemberStore(memory), WithTransport(configuredTransport)} + }, text: "configured together"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + backend := store.NewMemory() + rt, err := New(test.options(backend)...) + if err != nil { + t.Fatal(err) + } + err = rt.Start(context.Background()) + if !errors.Is(err, ErrInvalidSetup) || !strings.Contains(err.Error(), test.text) { + t.Fatalf("Start error = %v, want ErrInvalidSetup containing %q", err, test.text) + } + assertChannelClosed(t, "Done after invalid setup", rt.Done()) + if err := rt.Shutdown(context.Background()); err != nil { + t.Fatalf("Shutdown: %v", err) + } + }) + } +} + +func TestRuntimeStartRequiresCompleteGeneratedBindingsAndFactory(t *testing.T) { + tests := []struct { + name string + install func(*testing.T, *Runtime) + }{ + {name: "missing factory", install: func(t *testing.T, rt *Runtime) { installAccount(t, rt) }}, + {name: "missing dispatch", install: func(t *testing.T, rt *Runtime) { + installAccountBindings(t, rt, nil, accountProxyFactory, newAccountCall) + registerAccountFactory(t, rt) + }}, + {name: "missing proxy", install: func(t *testing.T, rt *Runtime) { + installAccountBindings(t, rt, dispatchAccount, nil, newAccountCall) + registerAccountFactory(t, rt) + }}, + {name: "missing call factory", install: func(t *testing.T, rt *Runtime) { + installAccountBindings(t, rt, dispatchAccount, accountProxyFactory, nil) + registerAccountFactory(t, rt) + }}, + {name: "missing Reminder call factory", install: func(t *testing.T, rt *Runtime) { + if err := InstallType[Account](rt, GeneratedCodeVersion, "gor.Account", dispatchAccount, accountProxyFactory, newAccountCall, nil); err != nil { + t.Fatal(err) + } + registerAccountFactory(t, rt) + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + rt := mustNew(t) + test.install(t, rt) + err := rt.Start(context.Background()) + if !errors.Is(err, ErrInvalidSetup) { + t.Fatalf("Start error = %v, want ErrInvalidSetup", err) + } + if err := rt.Shutdown(context.Background()); err != nil { + t.Fatalf("Shutdown: %v", err) + } + }) + } +} + +func TestRuntimeStartRejectsDuplicateGrainType(t *testing.T) { + rt := mustNew(t) + installAccountBindings(t, rt, dispatchAccount, accountProxyFactory, newAccountCall) + registerAccountFactory(t, rt) + if err := InstallType[scopeAccount](rt, GeneratedCodeVersion, "gor.Account", dispatchScopeAccount, func(invoker Invoker, id GrainId) scopeAccount { + return &scopeAccountProxy{invoker: invoker, id: id} + }, newScopeAccountCall, noReminderCall); err != nil { + t.Fatal(err) + } + if err := Register[scopeAccount](rt, func(*GrainContext) scopeAccount { return &scopeAccountGrain{} }); err != nil { + t.Fatal(err) + } + + err := rt.Start(context.Background()) + if !errors.Is(err, ErrInvalidSetup) || !strings.Contains(err.Error(), `GrainType "gor.Account" is used by`) { + t.Fatalf("Start error = %v, want clear duplicate GrainType error", err) + } + if err := rt.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestGrainTypeIsStableAcrossReferenceAndState(t *testing.T) { + backend := store.NewMemory() + rt, err := New( + WithStore(backend), + WithReminderStore(backend), + WithIdleTimeout(0), + WithEvictionInterval(0), + WithReminderInterval(0), + ) + if err != nil { + t.Fatal(err) + } + const grainType GrainType = "billing.account" + if err := InstallType[Account](rt, GeneratedCodeVersion, grainType, dispatchAccount, accountProxyFactory, newAccountCall, noReminderCall); err != nil { + t.Fatal(err) + } + registerAccountFactory(t, rt) + if got := GrainTypeOf[Account](rt); got != grainType { + t.Fatalf("GrainTypeOf before Start = %q, want %q", got, grainType) + } + if err := rt.Start(context.Background()); err != nil { + t.Fatal(err) + } + defer rt.Shutdown(context.Background()) + if got := GrainTypeOf[Account](rt); got != grainType { + t.Fatalf("GrainTypeOf after Start = %q, want %q", got, grainType) + } + proxy := Ref[Account](rt, "alice").(*accountProxy) + if proxy.id.GrainType != grainType { + t.Fatalf("Reference GrainType = %q, want %q", proxy.id.GrainType, grainType) + } + if _, err := proxy.Deposit(context.Background(), 7); err != nil { + t.Fatal(err) + } + record, err := backend.Read(context.Background(), store.GrainId{GrainType: string(grainType), GrainKey: "alice"}) + if err != nil { + t.Fatal(err) + } + if record.ETag != 1 { + t.Fatalf("stable GrainType record = %#v, want ETag 1", record) + } +} + +func TestDefaultGrainTypeReadsExistingState(t *testing.T) { + backend := store.NewMemory() + id := store.GrainId{GrainType: "gor.Account", GrainKey: "alice"} + if _, err := backend.Write(context.Background(), id, []byte(`{"value":41}`), 0); err != nil { + t.Fatal(err) + } + rt, err := New( + WithStore(backend), + WithReminderStore(backend), + WithIdleTimeout(0), + WithEvictionInterval(0), + WithReminderInterval(0), + ) + if err != nil { + t.Fatal(err) + } + installAccountBindings(t, rt, dispatchAccount, accountProxyFactory, newAccountCall) + registerAccountFactory(t, rt) + if err := rt.Start(context.Background()); err != nil { + t.Fatal(err) + } + defer closeRuntime(rt) + + got, err := Ref[Account](rt, "alice").Balance(context.Background()) + if err != nil { + t.Fatal(err) + } + if got != 41 { + t.Fatalf("Balance from existing default GrainType row = %d, want 41", got) + } +} + +func TestNewStateAfterFactoryPanics(t *testing.T) { + rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0), WithReminderInterval(0)) + installAccount(t, rt) + var captured *GrainContext + if err := Register[Account](rt, func(grain *GrainContext) Account { + captured = grain + return &account{value: NewState[int64](grain, "value")} + }); err != nil { + t.Fatal(err) + } + mustStart(t, rt) + defer closeRuntime(rt) + if _, err := Ref[Account](rt, "alice").Balance(context.Background()); err != nil { + t.Fatal(err) + } + + defer func() { + if got := recover(); got != "gor: NewState called after Grain factory" { + t.Fatalf("late NewState panic = %#v, want stable programming error", got) + } + }() + NewState[string](captured, "late") +} + +func TestRuntimeStoppingPrecedesDone(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rt := mustNew(t, WithIdleTimeout(0), WithEvictionInterval(0), WithReminderInterval(0)) + started := make(chan struct{}) + release := make(chan struct{}) + installAccountWithDispatch(t, rt, func(ctx context.Context, instance Account, method string, args any, reply any) error { + if method == "Block" { + close(started) + <-release + return nil + } + return dispatchAccount(ctx, instance, method, args, reply) + }) + registerAccountFactory(t, rt) + mustStart(t, rt) + + callDone := make(chan error, 1) + go func() { + callDone <- rt.Invoke(context.Background(), GrainId{GrainType: "gor.Account", GrainKey: "alice"}, "Block", nil, nil) + }() + synctest.Wait() + <-started + shutdownDone := make(chan error, 1) + go func() { shutdownDone <- rt.Shutdown(context.Background()) }() + synctest.Wait() + + assertChannelClosed(t, "Stopping during drain", rt.Stopping()) + assertChannelOpen(t, "Done during drain", rt.Done()) + select { + case err := <-shutdownDone: + t.Fatalf("Shutdown returned during drain: %v", err) + default: + } + close(release) + synctest.Wait() + if err := <-callDone; err != nil { + t.Fatalf("admitted Call: %v", err) + } + if err := <-shutdownDone; err != nil { + t.Fatalf("Shutdown: %v", err) + } + assertChannelClosed(t, "Done after drain", rt.Done()) + }) +} + +func installAccountBindings(t *testing.T, rt *Runtime, dispatch func(context.Context, Account, string, any, any) error, newProxy func(Invoker, GrainId) Account, newCall func(string) (any, any)) { + t.Helper() + if err := InstallType[Account](rt, GeneratedCodeVersion, "gor.Account", dispatch, newProxy, newCall, noReminderCall); err != nil { + t.Fatal(err) + } +} + +func accountProxyFactory(invoker Invoker, id GrainId) Account { + return &accountProxy{invoker: invoker, id: id} +} + +func registerAccountFactory(t *testing.T, rt *Runtime) { + t.Helper() + if err := Register[Account](rt, func(grain *GrainContext) Account { + return &account{value: NewState[int64](grain, "value")} + }); err != nil { + t.Fatal(err) + } +} + +func assertRuntimeNotStarted(t *testing.T, rt *Runtime) { + t.Helper() + err := rt.Invoke(context.Background(), GrainId{GrainType: "gor.Account", GrainKey: "alice"}, "Balance", &accountBalanceRequest{}, &accountBalanceReply{}) + if !errors.Is(err, ErrRuntimeNotStarted) { + t.Fatalf("Invoke before successful Start = %v, want ErrRuntimeNotStarted", err) + } +} + +func assertChannelOpen(t *testing.T, name string, channel <-chan struct{}) { + t.Helper() + select { + case <-channel: + t.Fatalf("%s is closed", name) + default: + } +} + +func assertChannelClosed(t *testing.T, name string, channel <-chan struct{}) { + t.Helper() + select { + case <-channel: + default: + t.Fatalf("%s is open", name) + } +} + +type startupStateStore struct { + backend store.Store + check func(context.Context) error + checks atomic.Int32 + reads atomic.Int32 + writes atomic.Int32 +} + +func (s *startupStateStore) Check(ctx context.Context) error { + s.checks.Add(1) + if s.check != nil { + return s.check(ctx) + } + return ctx.Err() +} + +func (s *startupStateStore) Read(ctx context.Context, id store.GrainId) (store.Record, error) { + s.reads.Add(1) + return s.backend.Read(ctx, id) +} + +func (s *startupStateStore) Write(ctx context.Context, id store.GrainId, data []byte, etag store.ETag) (store.ETag, error) { + s.writes.Add(1) + return s.backend.Write(ctx, id, data, etag) +} + +func (s *startupStateStore) totalCalls() int32 { + return s.checks.Load() + s.reads.Load() + s.writes.Load() +} + +type startupReminderStore struct { + backend store.ReminderStore + check func(context.Context) error + checks atomic.Int32 + listCalls atomic.Int32 + claimCalls atomic.Int32 + putCalls atomic.Int32 + deleteCalls atomic.Int32 +} + +func (s *startupReminderStore) Check(ctx context.Context) error { + s.checks.Add(1) + if s.check != nil { + return s.check(ctx) + } + return ctx.Err() +} + +func (s *startupReminderStore) ListDue(ctx context.Context, now time.Time, after *store.ReminderCursor, limit int) (store.ReminderPage, error) { + s.listCalls.Add(1) + return s.backend.ListDue(ctx, now, after, limit) +} + +func (s *startupReminderStore) Claim(ctx context.Context, reminder store.Reminder, next time.Time) (bool, error) { + s.claimCalls.Add(1) + return s.backend.Claim(ctx, reminder, next) +} + +func (s *startupReminderStore) Put(ctx context.Context, reminder store.Reminder) error { + s.putCalls.Add(1) + return s.backend.Put(ctx, reminder) +} + +func (s *startupReminderStore) Delete(ctx context.Context, id store.GrainId, name string) error { + s.deleteCalls.Add(1) + return s.backend.Delete(ctx, id, name) +} + +func (s *startupReminderStore) totalCalls() int32 { + return s.checks.Load() + s.listCalls.Load() + s.claimCalls.Load() + s.putCalls.Load() + s.deleteCalls.Load() +} + +type startupClock struct { + *clock.Fake + tickers atomic.Int32 +} + +func (c *startupClock) NewTicker(interval time.Duration) clock.Ticker { + c.tickers.Add(1) + return c.Fake.NewTicker(interval) +} + +type startupTransport struct { + serves atomic.Int32 + closes atomic.Int32 + kills atomic.Int32 +} + +func (t *startupTransport) Send(context.Context, string, []byte) ([]byte, error) { + return nil, errors.New("unexpected Send") +} + +func (t *startupTransport) Serve(ctx context.Context, _ transport.Handler) error { + t.serves.Add(1) + <-ctx.Done() + return ctx.Err() +} + +func (*startupTransport) Addr() string { + return "startup" +} + +func (t *startupTransport) Close() error { + t.closes.Add(1) + return nil +} + +func (t *startupTransport) Kill() error { + t.kills.Add(1) + return nil +} diff --git a/state.go b/state.go index e8b9cd6..cb41062 100644 --- a/state.go +++ b/state.go @@ -5,19 +5,26 @@ import ( "encoding/json" "errors" "fmt" + "sync" "github.com/suraciii/gor/store" ) -// Binder is the runtime-bound context passed to an entity factory. Entity code -// uses the Binder to create state, reminders, and references; application code -// should use the supplied Binder rather than construct one. -type Binder struct { - runtime *Runtime - identity store.GrainId - etag store.ETag - states map[string]stateCell - discard error +// GrainContext is the Runtime-bound context passed to a Grain factory. A +// factory uses it to declare State, create Grain Timers and Reminders, create +// Grain References, and use lifecycle controls. +type GrainContext struct { + runtime *Runtime + identity store.GrainId + deactivateOnIdle func() + registerGrainTimer grainTimerRegistrar + etag store.ETag + states map[string]stateCell + discard error + lastStateFailure error + stateFailureCount uint64 + stateMu sync.Mutex + stateOpen bool } type stateCell interface { @@ -26,32 +33,53 @@ type stateCell interface { isPresent() bool } -func newBinder(runtime *Runtime, id GrainId) *Binder { - return &Binder{ - runtime: runtime, - identity: store.GrainId{GrainType: id.GrainType, GrainKey: id.GrainKey}, - states: make(map[string]stateCell), +func newGrainContext(runtime *Runtime, id GrainId) *GrainContext { + return &GrainContext{ + runtime: runtime, + identity: store.GrainId{GrainType: string(id.GrainType), GrainKey: id.GrainKey}, + states: make(map[string]stateCell), + stateOpen: true, } } -// Self returns the identity of the entity bound to b. -func Self(b *Binder) GrainId { - return GrainId{GrainType: b.identity.GrainType, GrainKey: b.identity.GrainKey} +// Self returns the GrainId bound to b. +func Self(b *GrainContext) GrainId { + return GrainId{GrainType: GrainType(b.identity.GrainType), GrainKey: b.identity.GrainKey} } -// NewState registers a named persistent value for the entity bound to b and +// DeactivateOnIdle asks the Runtime to stop the current Activation after the +// current Grain method returns. Calls that are already queued keep their order +// and enter a new Activation. Repeated requests in one method are safe. +func DeactivateOnIdle(b *GrainContext) { + if b.deactivateOnIdle != nil { + b.deactivateOnIdle() + } +} + +// NewState registers a named persistent value for the Grain bound to b and // returns its handle. The name must be unique within that Grain; registering // the same name twice panics. A newly registered State is absent and has its // type's zero value until activation data is loaded or Set succeeds. -func NewState[T any](b *Binder, name string) State[T] { +func NewState[T any](b *GrainContext, name string) State[T] { + b.stateMu.Lock() + defer b.stateMu.Unlock() + if !b.stateOpen { + panic("gor: NewState called after Grain factory") + } if _, exists := b.states[name]; exists { panic(fmt.Sprintf("state %q is already registered", name)) } - cell := &stateCellValue[T]{binder: b} + cell := &stateCellValue[T]{grain: b, name: name} b.states[name] = cell return State[T]{cell: cell} } +func (b *GrainContext) freezeStateDeclarations() { + b.stateMu.Lock() + b.stateOpen = false + b.stateMu.Unlock() +} + // State is a handle to one named JSON-encoded value in a Grain's persistent // state record. All State handles for one Grain share one JSON object, one // store record, and one ETag; setting or clearing one handle rewrites the @@ -78,7 +106,7 @@ func (s State[T]) Exists() bool { // T's zero value after the write succeeds. The write uses the current Grain // ETag and has the same conflict and store failure behavior as Set. func (s State[T]) Clear(ctx context.Context) error { - if err := s.cell.binder.persist(ctx, s.cell, nil, false); err != nil { + if err := s.cell.grain.persist(ctx, "clear State", s.cell, nil, false); err != nil { return err } var zero T @@ -89,21 +117,23 @@ func (s State[T]) Clear(ctx context.Context) error { // Set JSON-encodes value and persists the Grain's complete state record using // ctx. A JSON encoding error for value or another registered state leaves the -// current value unchanged and is returned without a store write. Store errors -// leave the current in-memory value unchanged, but do not establish whether the -// store wrote the record; callers must not assume the write failed or retry -// unconditionally. Store errors are returned as well; in particular, +// current value unchanged and is returned without a store write. A Store error +// does not commit the value argument, presence mark, or ETag. The Runtime cannot +// undo a change made through a reference value returned by Get. A Store error +// does not establish whether the Store wrote the record. Callers must not +// assume the write failed or retry unconditionally. In particular, // errors.Is(err, store.ErrConflict) and errors.Is(err, ErrPersistenceConflict) // report an ETag conflict. -// A store write failure also discards the current entity activation after the -// containing call completes, so the next call creates a fresh activation. -// On success, subsequent Get calls return value. +// A Store write failure makes the current Activation unusable. During setup, +// it prevents method entry. During a Call, Reminder, or Grain Timer turn, the +// Activation ends after the turn. During OnDeactivate, the Runtime reports the +// failure through OnError. On success, subsequent Get calls return value. func (s State[T]) Set(ctx context.Context, value T) error { encoded, err := json.Marshal(value) if err != nil { - return err + return s.cell.grain.stateError("encode State", s.cell.name, err) } - if err := s.cell.binder.persist(ctx, s.cell, encoded, true); err != nil { + if err := s.cell.grain.persist(ctx, "write State", s.cell, encoded, true); err != nil { return err } s.cell.value = value @@ -112,7 +142,8 @@ func (s State[T]) Set(ctx context.Context, value T) error { } type stateCellValue[T any] struct { - binder *Binder + grain *GrainContext + name string value T present bool } @@ -133,22 +164,22 @@ func (s *stateCellValue[T]) isPresent() bool { return s.present } -func (b *Binder) load(ctx context.Context) error { +func (b *GrainContext) load(ctx context.Context) error { record, err := b.runtime.store.Read(ctx, b.identity) if err != nil { - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return err - } - return withCode(ErrPersistenceFailed, err) + return b.codedStateError(ErrPersistenceFailed, "read State record", "", err) } - if len(record.Data) == 0 { + if record.Data == nil && record.ETag == 0 { b.etag = record.ETag return nil } var document map[string]json.RawMessage if err := json.Unmarshal(record.Data, &document); err != nil { - return err + return b.codedStateError(ErrPersistenceFailed, "decode State record", "", err) + } + if document == nil { + return b.codedStateError(ErrPersistenceFailed, "decode State record", "", errors.New("State record must be a JSON object")) } for name, cell := range b.states { data, ok := document[name] @@ -156,17 +187,19 @@ func (b *Binder) load(ctx context.Context) error { continue } if err := cell.decode(data); err != nil { - return err + return b.codedStateError(ErrPersistenceFailed, "decode State", name, err) } } b.etag = record.ETag return nil } -func (b *Binder) persist(ctx context.Context, changed stateCell, changedData []byte, changedPresent bool) error { +func (b *GrainContext) persist(ctx context.Context, operation string, changed stateCell, changedData []byte, changedPresent bool) error { + changedName := "" document := make(map[string]json.RawMessage, len(b.states)) for name, cell := range b.states { if cell == changed { + changedName = name if !changedPresent { continue } @@ -178,31 +211,73 @@ func (b *Binder) persist(ctx context.Context, changed stateCell, changedData []b } data, err := cell.encode() if err != nil { - return err + return b.stateError("encode State", name, err) } document[name] = json.RawMessage(data) } data, err := json.Marshal(document) if err != nil { - return err + return b.stateError("encode State record", "", err) } etag, err := b.runtime.store.Write(ctx, b.identity, data, b.etag) if err != nil { - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return err - } + code := ErrPersistenceFailed if errors.Is(err, store.ErrConflict) { - b.discard = withCode(ErrPersistenceConflict, err) - } else { - b.discard = withCode(ErrPersistenceFailed, err) + code = ErrPersistenceConflict } - return b.discard + failure := b.codedStateError(code, operation, changedName, err) + b.markDiscard(failure) + return failure } b.etag = etag return nil } -func (b *Binder) discardError() error { +func (b *GrainContext) stateError(operation string, name string, err error) error { + if name == "" { + return fmt.Errorf("%s for Grain %q/%q: %w", operation, b.identity.GrainType, b.identity.GrainKey, err) + } + return fmt.Errorf("%s %q for Grain %q/%q: %w", operation, name, b.identity.GrainType, b.identity.GrainKey, err) +} + +func (b *GrainContext) codedStateError(code Code, operation string, name string, err error) error { + return withCode(code, b.stateError(operation, name, err)) +} + +func (b *GrainContext) markDiscard(err error) { + if b.discard == nil { + b.discard = err + } + b.lastStateFailure = err + b.stateFailureCount++ +} + +func (b *GrainContext) discardError() error { return b.discard } + +func (b *GrainContext) resultWithDiscard(err error) (error, bool) { + discard := b.discardError() + if discard == nil { + return err, false + } + if err == nil { + return discard, true + } + if errors.Is(err, discard) { + return err, true + } + return errors.Join(err, discard), true +} + +func (b *GrainContext) stateFailureSnapshot() uint64 { + return b.stateFailureCount +} + +func (b *GrainContext) stateFailureAfter(snapshot uint64) error { + if b.stateFailureCount == snapshot { + return nil + } + return b.lastStateFailure +} diff --git a/state_failure_test.go b/state_failure_test.go new file mode 100644 index 0000000..866b065 --- /dev/null +++ b/state_failure_test.go @@ -0,0 +1,713 @@ +package gor + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/suraciii/gor/clock" + "github.com/suraciii/gor/store" +) + +type appliedOutcomeStore struct { + *store.Memory + + mu sync.Mutex + nextErr error +} + +type preCommitFailureStore struct { + *store.Memory + + mu sync.Mutex + nextErr error +} + +type mutableStateAliasAccount interface { + MutateAndFail(context.Context) error + Snapshot(context.Context) (mutableStateAliasSnapshot, error) +} + +type mutableStateAliasSnapshot struct { + Count int + Exists bool + ETag store.ETag +} + +type mutableStateAliasFailure struct { + mutableStateAliasSnapshot + DiscardErr error +} + +type mutableStateAliasMutateRequest struct{} + +type mutableStateAliasMutateReply struct{} + +type mutableStateAliasSnapshotRequest struct{} + +type mutableStateAliasSnapshotReply struct { + R0 mutableStateAliasSnapshot +} + +type mutableStateAliasGrain struct { + grain *GrainContext + value State[map[string]int] + failures chan<- mutableStateAliasFailure +} + +type mutableStateAliasProxy struct { + invoker Invoker + id GrainId +} + +func (a *mutableStateAliasGrain) MutateAndFail(ctx context.Context) error { + alias := a.value.Get() + alias["count"] = 2 + err := a.value.Set(ctx, alias) + if err != nil { + a.failures <- mutableStateAliasFailure{ + mutableStateAliasSnapshot: mutableStateAliasSnapshot{ + Count: a.value.Get()["count"], + Exists: a.value.Exists(), + ETag: a.grain.etag, + }, + DiscardErr: a.grain.discardError(), + } + } + return err +} + +func (a *mutableStateAliasGrain) Snapshot(context.Context) (mutableStateAliasSnapshot, error) { + return mutableStateAliasSnapshot{ + Count: a.value.Get()["count"], + Exists: a.value.Exists(), + ETag: a.grain.etag, + }, nil +} + +func (p *mutableStateAliasProxy) MutateAndFail(ctx context.Context) error { + return p.invoker.Invoke(ctx, p.id, "MutateAndFail", &mutableStateAliasMutateRequest{}, &mutableStateAliasMutateReply{}) +} + +func (p *mutableStateAliasProxy) Snapshot(ctx context.Context) (mutableStateAliasSnapshot, error) { + var reply mutableStateAliasSnapshotReply + err := p.invoker.Invoke(ctx, p.id, "Snapshot", &mutableStateAliasSnapshotRequest{}, &reply) + return reply.R0, err +} + +func dispatchMutableStateAlias(ctx context.Context, instance mutableStateAliasAccount, method string, _ any, reply any) error { + switch method { + case "MutateAndFail": + return instance.MutateAndFail(ctx) + case "Snapshot": + typedReply := reply.(*mutableStateAliasSnapshotReply) + value, err := instance.Snapshot(ctx) + if err == nil { + typedReply.R0 = value + } + return err + default: + return fmt.Errorf("unknown method %q", method) + } +} + +func newMutableStateAliasCall(method string) (args any, reply any) { + switch method { + case "MutateAndFail": + return &mutableStateAliasMutateRequest{}, &mutableStateAliasMutateReply{} + case "Snapshot": + return &mutableStateAliasSnapshotRequest{}, &mutableStateAliasSnapshotReply{} + default: + return nil, nil + } +} + +func installMutableStateAliasAccount( + t *testing.T, + rt *Runtime, + factoryCalls *atomic.Int32, + failures chan<- mutableStateAliasFailure, +) { + t.Helper() + if err := InstallType[mutableStateAliasAccount](rt, GeneratedCodeVersion, "gor.mutableStateAliasAccount", dispatchMutableStateAlias, func(invoker Invoker, id GrainId) mutableStateAliasAccount { + return &mutableStateAliasProxy{invoker: invoker, id: id} + }, newMutableStateAliasCall, noReminderCall); err != nil { + t.Fatal(err) + } + if err := Register[mutableStateAliasAccount](rt, func(grain *GrainContext) mutableStateAliasAccount { + factoryCalls.Add(1) + return &mutableStateAliasGrain{ + grain: grain, + value: NewState[map[string]int](grain, "value"), + failures: failures, + } + }); err != nil { + t.Fatal(err) + } +} + +func newAppliedOutcomeStore() *appliedOutcomeStore { + return &appliedOutcomeStore{Memory: store.NewMemory()} +} + +func newPreCommitFailureStore() *preCommitFailureStore { + return &preCommitFailureStore{Memory: store.NewMemory()} +} + +func (s *appliedOutcomeStore) failNextWrite(err error) { + s.mu.Lock() + s.nextErr = err + s.mu.Unlock() +} + +func (s *preCommitFailureStore) failNextWrite(err error) { + s.mu.Lock() + s.nextErr = err + s.mu.Unlock() +} + +func (s *preCommitFailureStore) Write(ctx context.Context, id store.GrainId, data []byte, expect store.ETag) (store.ETag, error) { + s.mu.Lock() + err := s.nextErr + s.nextErr = nil + s.mu.Unlock() + if err != nil { + return 0, err + } + return s.Memory.Write(ctx, id, data, expect) +} + +func (s *appliedOutcomeStore) Write(ctx context.Context, id store.GrainId, data []byte, expect store.ETag) (store.ETag, error) { + // Apply the write before the injected result. A cancellation or timeout + // result therefore has the same unknown outcome as a lost reply. + etag, err := s.Memory.Write(ctx, id, data, expect) + if err != nil { + return 0, err + } + s.mu.Lock() + err = s.nextErr + s.nextErr = nil + s.mu.Unlock() + if err != nil { + return 0, err + } + return etag, nil +} + +func TestStateMutableAliasFailureCannotRollbackAndNextActivationReloads(t *testing.T) { + backend := newPreCommitFailureStore() + id := store.GrainId{GrainType: "gor.mutableStateAliasAccount", GrainKey: "alice"} + confirmedETag, err := backend.Memory.Write(context.Background(), id, []byte(`{"value":{"count":1}}`), 0) + if err != nil { + t.Fatalf("seed State: %v", err) + } + factoryCalls := new(atomic.Int32) + failures := make(chan mutableStateAliasFailure, 1) + rt := mustNew(t, + WithStore(backend), + WithClock(clock.NewFake(time.Unix(0, 0).UTC())), + WithIdleTimeout(0), + WithEvictionInterval(0), + WithReminderInterval(0), + ) + defer closeRuntime(rt) + installMutableStateAliasAccount(t, rt, factoryCalls, failures) + mustStart(t, rt) + account := Ref[mutableStateAliasAccount](rt, "alice") + + writeErr := errors.New("State write failed before commit") + backend.failNextWrite(writeErr) + if err := account.MutateAndFail(context.Background()); !errors.Is(err, writeErr) { + t.Fatalf("MutateAndFail error = %v, want %v", err, writeErr) + } else if code, ok := CodeOf(err); !ok || code != ErrPersistenceFailed { + t.Fatalf("MutateAndFail Code = (%q, %v), want (%q, true)", code, ok, ErrPersistenceFailed) + } + var failed mutableStateAliasFailure + select { + case failed = <-failures: + default: + t.Fatal("failed Activation did not report its local State") + } + if failed.Count != 2 { + t.Fatalf("failed Activation alias = %d, want unconfirmed mutation 2", failed.Count) + } + if !failed.Exists { + t.Fatal("failed Set changed the State presence mark") + } + if failed.ETag != confirmedETag { + t.Fatalf("failed Activation ETag = %d, want confirmed ETag %d", failed.ETag, confirmedETag) + } + if !errors.Is(failed.DiscardErr, writeErr) { + t.Fatalf("discard error = %v, want %v", failed.DiscardErr, writeErr) + } + if got := factoryCalls.Load(); got != 1 { + t.Fatalf("factory calls after failed Set = %d, want 1 before the next Call", got) + } + + record, err := backend.Read(context.Background(), id) + if err != nil { + t.Fatalf("read Confirmed State: %v", err) + } + if string(record.Data) != `{"value":{"count":1}}` || record.ETag != confirmedETag { + t.Fatalf("Confirmed State = (%s, ETag %d), want ({\"value\":{\"count\":1}}, ETag %d)", record.Data, record.ETag, confirmedETag) + } + + restarted, err := account.Snapshot(context.Background()) + if err != nil { + t.Fatalf("Snapshot after failed Set: %v", err) + } + if restarted.Count != 1 || !restarted.Exists || restarted.ETag != confirmedETag { + t.Fatalf("reloaded State = %#v, want count 1, present, ETag %d", restarted, confirmedETag) + } + if got := factoryCalls.Load(); got != 2 { + t.Fatalf("factory calls after reload = %d, want 2", got) + } +} + +func TestStateWriteOutcomesDiscardActivationAndReloadStore(t *testing.T) { + for _, test := range []struct { + name string + err error + }{ + {name: "cancel", err: context.Canceled}, + {name: "timeout", err: context.DeadlineExceeded}, + {name: "lost_reply", err: errors.New("State write reply was lost")}, + } { + t.Run(test.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + backend := newAppliedOutcomeStore() + backend.failNextWrite(test.err) + rt := mustNew(t, WithStore(backend), WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + + var factoryCalls atomic.Int32 + installAccount(t, rt) + if err := Register[Account](rt, func(grain *GrainContext) Account { + factoryCalls.Add(1) + return &account{value: NewState[int64](grain, "value")} + }); err != nil { + t.Fatal(err) + } + mustStart(t, rt) + + account := Ref[Account](rt, "alice") + if _, err := account.Deposit(context.Background(), 10); !errors.Is(err, test.err) { + t.Fatalf("Deposit error = %v, want %v", err, test.err) + } else if code, ok := CodeOf(err); !ok || code != ErrPersistenceFailed { + t.Fatalf("Deposit Code = (%q, %v), want (%q, true)", code, ok, ErrPersistenceFailed) + } else if count := strings.Count(err.Error(), "write State"); count != 1 { + t.Fatalf("Deposit error contains %d State write diagnostics, want 1: %v", count, err) + } + + balance, err := account.Balance(context.Background()) + if err != nil { + t.Fatalf("Balance after failed reply: %v", err) + } + if balance != 10 { + t.Fatalf("Balance after failed reply = %d, want applied Store value 10", balance) + } + if calls := factoryCalls.Load(); calls != 2 { + t.Fatalf("factory calls = %d, want 2 after State failure", calls) + } + }) + }) + } +} + +func TestStateClearUnknownResultKeepsCellAndReloadsStore(t *testing.T) { + backend := newAppliedOutcomeStore() + id := GrainId{GrainType: "account", GrainKey: "alice"} + grain := newTestGrainContext(id, backend, nil, clock.Real{}) + value := NewState[int](grain, "value") + if err := value.Set(context.Background(), 7); err != nil { + t.Fatalf("Set: %v", err) + } + + unknownErr := errors.New("State Clear reply was lost") + backend.failNextWrite(unknownErr) + if err := value.Clear(context.Background()); !errors.Is(err, unknownErr) { + t.Fatalf("Clear error = %v, want %v", err, unknownErr) + } + if !value.Exists() || value.Get() != 7 { + t.Fatalf("current State after failed Clear = (exists=%v, value=%d), want (true, 7)", value.Exists(), value.Get()) + } + if !errors.Is(grain.discardError(), unknownErr) { + t.Fatalf("discard error = %v, want %v", grain.discardError(), unknownErr) + } + + restarted := newTestGrainContext(id, backend, nil, clock.Real{}) + restartedValue := NewState[int](restarted, "value") + if err := restarted.load(context.Background()); err != nil { + t.Fatalf("restart load: %v", err) + } + if restartedValue.Exists() || restartedValue.Get() != 0 { + t.Fatalf("restarted State = (exists=%v, value=%d), want applied Clear", restartedValue.Exists(), restartedValue.Get()) + } +} + +type readErrorStore struct { + store.Store + err error +} + +type stateDeclarationProbeStore struct { + *store.Memory + grain *GrainContext + sawFrozen atomic.Bool +} + +func (s *stateDeclarationProbeStore) Read(ctx context.Context, id store.GrainId) (store.Record, error) { + if s.grain != nil && !s.grain.stateOpen { + s.sawFrozen.Store(true) + } + return s.Memory.Read(ctx, id) +} + +func TestStateDeclarationsFreezeBeforeLoad(t *testing.T) { + backend := &stateDeclarationProbeStore{Memory: store.NewMemory()} + rt := mustNew(t, WithStore(backend), WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + installAccount(t, rt) + if err := Register[Account](rt, func(grain *GrainContext) Account { + backend.grain = grain + return &account{value: NewState[int64](grain, "value")} + }); err != nil { + t.Fatal(err) + } + mustStart(t, rt) + + if _, err := Ref[Account](rt, "alice").Balance(context.Background()); err != nil { + t.Fatalf("Balance: %v", err) + } + if !backend.sawFrozen.Load() { + t.Fatal("State declarations were open when the Store read started") + } +} + +func (s readErrorStore) Read(context.Context, store.GrainId) (store.Record, error) { + return store.Record{}, s.err +} + +func TestStateErrorsIncludePersistenceContext(t *testing.T) { + id := GrainId{GrainType: "account", GrainKey: "alice"} + + t.Run("read_record", func(t *testing.T) { + readErr := errors.New("read failed") + grain := newTestGrainContext(id, readErrorStore{Store: store.NewMemory(), err: readErr}, nil, clock.Real{}) + NewState[int](grain, "balance") + err := grain.load(context.Background()) + assertPersistenceContext(t, err, readErr, "read State record", "account", "alice") + }) + + t.Run("decode_record", func(t *testing.T) { + backend := store.NewMemory() + if _, err := backend.Write(context.Background(), toStoreGrainID(id), []byte("{"), 0); err != nil { + t.Fatal(err) + } + grain := newTestGrainContext(id, backend, nil, clock.Real{}) + NewState[int](grain, "balance") + err := grain.load(context.Background()) + assertPersistenceContext(t, err, nil, "decode State record", "account", "alice") + }) + + t.Run("decode_named_State", func(t *testing.T) { + backend := store.NewMemory() + if _, err := backend.Write(context.Background(), toStoreGrainID(id), []byte(`{"balance":"bad"}`), 0); err != nil { + t.Fatal(err) + } + grain := newTestGrainContext(id, backend, nil, clock.Real{}) + NewState[int](grain, "balance") + err := grain.load(context.Background()) + assertPersistenceContext(t, err, nil, "decode State", "balance", "account", "alice") + }) + + t.Run("empty_confirmed_record", func(t *testing.T) { + backend := failingWriteStore{record: store.Record{Data: []byte{}, ETag: 1}} + grain := newTestGrainContext(id, backend, nil, clock.Real{}) + NewState[int](grain, "balance") + err := grain.load(context.Background()) + assertPersistenceContext(t, err, nil, "decode State record", "account", "alice") + }) + + t.Run("null_confirmed_record", func(t *testing.T) { + backend := store.NewMemory() + if _, err := backend.Write(context.Background(), toStoreGrainID(id), []byte("null"), 0); err != nil { + t.Fatal(err) + } + grain := newTestGrainContext(id, backend, nil, clock.Real{}) + NewState[int](grain, "balance") + err := grain.load(context.Background()) + assertPersistenceContext(t, err, nil, "decode State record", "JSON object", "account", "alice") + }) + + t.Run("write", func(t *testing.T) { + writeErr := errors.New("write failed") + grain := newTestGrainContext(id, failingWriteStore{err: writeErr}, nil, clock.Real{}) + balance := NewState[int](grain, "balance") + err := balance.Set(context.Background(), 1) + assertPersistenceContext(t, err, writeErr, "write State", "balance", "account", "alice") + }) + + t.Run("clear", func(t *testing.T) { + writeErr := errors.New("clear failed") + grain := newTestGrainContext(id, failingWriteStore{ + err: writeErr, + record: store.Record{Data: []byte(`{"balance":1}`), ETag: 1}, + }, nil, clock.Real{}) + balance := NewState[int](grain, "balance") + if err := grain.load(context.Background()); err != nil { + t.Fatal(err) + } + err := balance.Clear(context.Background()) + assertPersistenceContext(t, err, writeErr, "clear State", "balance", "account", "alice") + }) + + t.Run("encode", func(t *testing.T) { + grain := newTestGrainContext(id, store.NewMemory(), nil, clock.Real{}) + value := NewState[func()](grain, "callback") + err := value.Set(context.Background(), func() {}) + if err == nil { + t.Fatal("Set error = nil, want JSON encoding error") + } + for _, part := range []string{"encode State", "callback", "account", "alice"} { + if !strings.Contains(err.Error(), part) { + t.Fatalf("Set error %q does not contain %q", err, part) + } + } + if grain.discardError() != nil { + t.Fatalf("encoding error marked Activation for discard: %v", grain.discardError()) + } + }) +} + +func assertPersistenceContext(t *testing.T, err error, cause error, parts ...string) { + t.Helper() + if err == nil { + t.Fatal("persistence error = nil") + } + if cause != nil && !errors.Is(err, cause) { + t.Fatalf("persistence error = %v, want cause %v", err, cause) + } + if code, ok := CodeOf(err); !ok || code != ErrPersistenceFailed { + t.Fatalf("persistence Code = (%q, %v), want (%q, true)", code, ok, ErrPersistenceFailed) + } + for _, part := range parts { + if !strings.Contains(err.Error(), part) { + t.Fatalf("persistence error %q does not contain %q", err, part) + } + } +} + +type activationStateAccount struct { + value State[int64] + entered chan<- struct{} +} + +func (a *activationStateAccount) OnActivate(ctx context.Context) error { + _ = a.value.Set(ctx, a.value.Get()+1) + return nil +} + +func (a *activationStateAccount) Deposit(context.Context, int64) (int64, error) { + a.entered <- struct{}{} + return a.value.Get(), nil +} + +func (a *activationStateAccount) Balance(context.Context) (int64, error) { + a.entered <- struct{}{} + return a.value.Get(), nil +} + +func TestOnActivateStateFailurePreventsMethodEntry(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + writeErr := errors.New("activation State reply was lost") + backend := newAppliedOutcomeStore() + backend.failNextWrite(writeErr) + rt := mustNew(t, WithStore(backend), WithIdleTimeout(0), WithEvictionInterval(0)) + defer closeRuntime(rt) + + entered := make(chan struct{}, 1) + var factoryCalls atomic.Int32 + installAccount(t, rt) + if err := Register[Account](rt, func(grain *GrainContext) Account { + factoryCalls.Add(1) + return &activationStateAccount{ + value: NewState[int64](grain, "value"), + entered: entered, + } + }); err != nil { + t.Fatal(err) + } + mustStart(t, rt) + + account := Ref[Account](rt, "alice") + if _, err := account.Balance(context.Background()); !errors.Is(err, writeErr) { + t.Fatalf("first Balance error = %v, want %v", err, writeErr) + } + select { + case <-entered: + t.Fatal("method entered after OnActivate ignored a State failure") + default: + } + + balance, err := account.Balance(context.Background()) + if err != nil { + t.Fatalf("second Balance: %v", err) + } + if balance != 2 { + t.Fatalf("second Balance = %d, want Store value 2", balance) + } + if calls := factoryCalls.Load(); calls != 2 { + t.Fatalf("factory calls = %d, want 2", calls) + } + }) +} + +type deactivationStateAccount struct { + value State[int64] + returnStateError bool + hookError error +} + +func (a *deactivationStateAccount) Deposit(context.Context, int64) (int64, error) { + return a.value.Get(), nil +} + +func (a *deactivationStateAccount) Balance(context.Context) (int64, error) { + return a.value.Get(), nil +} + +func (a *deactivationStateAccount) OnDeactivate(ctx context.Context, _ DeactivationReason) error { + err := a.value.Set(ctx, 1) + if a.hookError != nil { + return a.hookError + } + if a.returnStateError { + return err + } + return nil +} + +func TestOnDeactivateStateFailureIsReported(t *testing.T) { + for _, test := range []struct { + name string + returnStateError bool + returnHookError bool + }{ + {name: "ignored"}, + {name: "returned_State_error", returnStateError: true}, + {name: "hook_and_State_errors", returnHookError: true}, + } { + t.Run(test.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + writeErr := errors.New("deactivation State write failed") + hookErr := errors.New("deactivation hook failed") + var configuredHookErr error + if test.returnHookError { + configuredHookErr = hookErr + } + events := make(chan BackgroundError, 1) + rt := mustNew(t, + WithStore(failingWriteStore{err: writeErr}), + WithIdleTimeout(0), + WithEvictionInterval(0), + OnError(func(event BackgroundError) { + events <- event + }), + ) + installAccount(t, rt) + if err := Register[Account](rt, func(grain *GrainContext) Account { + return &deactivationStateAccount{ + value: NewState[int64](grain, "value"), + returnStateError: test.returnStateError, + hookError: configuredHookErr, + } + }); err != nil { + t.Fatal(err) + } + mustStart(t, rt) + + if _, err := Ref[Account](rt, "alice").Balance(context.Background()); err != nil { + t.Fatalf("Balance: %v", err) + } + if err := rt.Shutdown(context.Background()); err != nil { + t.Fatalf("Shutdown: %v", err) + } + + select { + case event := <-events: + if !errors.Is(event.Err, writeErr) || !errors.Is(event.Err, ErrPersistenceFailed) { + t.Fatalf("OnError = %#v, want persistence failure", event) + } + if test.returnHookError && !errors.Is(event.Err, hookErr) { + t.Fatalf("OnError = %#v, want hook error %v", event, hookErr) + } + if _, ok := event.Source.(Deactivation); !ok { + t.Fatalf("OnError source = %#v, want Deactivation", event.Source) + } + if count := strings.Count(event.Err.Error(), "write State"); count != 1 { + t.Fatalf("OnError contains %d State write diagnostics, want 1: %v", count, event.Err) + } + default: + t.Fatal("OnDeactivate State failure was not reported") + } + }) + }) + } +} + +type callStateFailureAccount struct { + value State[int64] +} + +func (a *callStateFailureAccount) Deposit(ctx context.Context, amount int64) (int64, error) { + next := a.value.Get() + amount + return next, a.value.Set(ctx, next) +} + +func (a *callStateFailureAccount) Balance(context.Context) (int64, error) { + return a.value.Get(), nil +} + +func (*callStateFailureAccount) OnDeactivate(context.Context, DeactivationReason) error { + return nil +} + +func TestOnDeactivateDoesNotReportPriorStateFailureAgain(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + writeErr := errors.New("Call State write failed") + events := make(chan BackgroundError, 1) + rt := mustNew(t, + WithStore(failingWriteStore{err: writeErr}), + WithIdleTimeout(0), + WithEvictionInterval(0), + OnError(func(event BackgroundError) { + events <- event + }), + ) + defer closeRuntime(rt) + installAccount(t, rt) + if err := Register[Account](rt, func(grain *GrainContext) Account { + return &callStateFailureAccount{value: NewState[int64](grain, "value")} + }); err != nil { + t.Fatal(err) + } + mustStart(t, rt) + + if _, err := Ref[Account](rt, "alice").Deposit(context.Background(), 1); !errors.Is(err, writeErr) { + t.Fatalf("Deposit error = %v, want %v", err, writeErr) + } + synctest.Wait() + select { + case event := <-events: + t.Fatalf("prior Call State failure was reported again: %#v", event) + default: + } + }) +} diff --git a/state_test.go b/state_test.go index abe8edc..99a69b9 100644 --- a/state_test.go +++ b/state_test.go @@ -10,15 +10,15 @@ import ( "github.com/suraciii/gor/store" ) -func newTestBinder(id GrainId, backend store.Store, reminders store.ReminderStore, sourceClock clock.Clock) *Binder { - return newBinder(&Runtime{store: backend, reminderStore: reminders, clock: sourceClock}, id) +func newTestGrainContext(id GrainId, backend store.Store, reminders store.ReminderStore, sourceClock clock.Clock) *GrainContext { + return newGrainContext(&Runtime{store: backend, reminderStore: reminders, clock: sourceClock}, id) } func TestState_PersistsAllRegisteredValuesAsOneRecord(t *testing.T) { backend := store.NewMemory() - binder := newTestBinder(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) - balance := NewState[int64](binder, "balance") - name := NewState[string](binder, "name") + grain := newTestGrainContext(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) + balance := NewState[int64](grain, "balance") + name := NewState[string](grain, "name") if err := balance.Set(context.Background(), 42); err != nil { t.Fatalf("balance Set: %v", err) @@ -39,24 +39,24 @@ func TestState_PersistsAllRegisteredValuesAsOneRecord(t *testing.T) { } } -func TestSelf_ReturnsBinderIdentity(t *testing.T) { +func TestSelf_ReturnsBinderIdgrain(t *testing.T) { want := GrainId{GrainType: "account", GrainKey: "alice"} - binder := newTestBinder(want, store.NewMemory(), nil, clock.Real{}) + grain := newTestGrainContext(want, store.NewMemory(), nil, clock.Real{}) - if got := Self(binder); got != want { + if got := Self(grain); got != want { t.Fatalf("Self = %#v, want %#v", got, want) } } func TestState_LoadsValuesAndETagFromStore(t *testing.T) { backend := store.NewMemory() - first := newTestBinder(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) + first := newTestGrainContext(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) firstBalance := NewState[int64](first, "balance") if err := firstBalance.Set(context.Background(), 42); err != nil { t.Fatalf("first Set: %v", err) } - second := newTestBinder(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) + second := newTestGrainContext(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) secondBalance := NewState[int64](second, "balance") if err := second.load(context.Background()); err != nil { t.Fatalf("load: %v", err) @@ -79,13 +79,13 @@ func TestState_LoadsValuesAndETagFromStore(t *testing.T) { func TestState_ConflictLeavesValueAndMarksBinder(t *testing.T) { backend := store.NewMemory() - first := newTestBinder(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) + first := newTestGrainContext(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) firstBalance := NewState[int64](first, "balance") if err := firstBalance.Set(context.Background(), 1); err != nil { t.Fatalf("first Set: %v", err) } - second := newTestBinder(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) + second := newTestGrainContext(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) secondBalance := NewState[int64](second, "balance") if err := second.load(context.Background()); err != nil { t.Fatalf("load: %v", err) @@ -108,8 +108,8 @@ func TestState_ConflictLeavesValueAndMarksBinder(t *testing.T) { func TestState_WriteErrorLeavesValueAndMarksBinder(t *testing.T) { writeErr := errors.New("store unavailable") - binder := newTestBinder(GrainId{GrainType: "account", GrainKey: "alice"}, failingWriteStore{err: writeErr}, nil, clock.Real{}) - balance := NewState[int64](binder, "balance") + grain := newTestGrainContext(GrainId{GrainType: "account", GrainKey: "alice"}, failingWriteStore{err: writeErr}, nil, clock.Real{}) + balance := NewState[int64](grain, "balance") balance.cell.value = 1 err := balance.Set(context.Background(), 2) @@ -119,8 +119,8 @@ func TestState_WriteErrorLeavesValueAndMarksBinder(t *testing.T) { if balance.Get() != 1 { t.Fatalf("value after write error = %d, want 1", balance.Get()) } - if !errors.Is(binder.discardError(), writeErr) { - t.Fatalf("discard marker = %v, want %v", binder.discardError(), writeErr) + if !errors.Is(grain.discardError(), writeErr) { + t.Fatalf("discard marker = %v, want %v", grain.discardError(), writeErr) } } @@ -129,6 +129,10 @@ type failingWriteStore struct { record store.Record } +func (f failingWriteStore) Check(ctx context.Context) error { + return ctx.Err() +} + func (f failingWriteStore) Read(context.Context, store.GrainId) (store.Record, error) { return f.record, nil } @@ -138,22 +142,22 @@ func (s failingWriteStore) Write(context.Context, store.GrainId, []byte, store.E } func TestNewState_PanicsOnDuplicateName(t *testing.T) { - binder := newTestBinder(GrainId{GrainType: "account", GrainKey: "alice"}, store.NewMemory(), nil, clock.Real{}) - NewState[int64](binder, "balance") + grain := newTestGrainContext(GrainId{GrainType: "account", GrainKey: "alice"}, store.NewMemory(), nil, clock.Real{}) + NewState[int64](grain, "balance") defer func() { if recover() == nil { t.Fatal("duplicate state name did not panic") } }() - NewState[string](binder, "balance") + NewState[string](grain, "balance") } func TestState_NewValueIsAbsentAndNotPersisted(t *testing.T) { forEachStateBackend(t, func(t *testing.T, backend store.Store) { - binder := newTestBinder(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) - absent := NewState[int](binder, "absent") - other := NewState[int](binder, "other") + grain := newTestGrainContext(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) + absent := NewState[int](grain, "absent") + other := NewState[int](grain, "other") if absent.Exists() { t.Fatal("new State Exists = true, want false") @@ -165,7 +169,7 @@ func TestState_NewValueIsAbsentAndNotPersisted(t *testing.T) { t.Fatalf("other Set: %v", err) } - record, err := backend.Read(context.Background(), binder.identity) + record, err := backend.Read(context.Background(), grain.identity) if err != nil { t.Fatalf("Read: %v", err) } @@ -178,8 +182,8 @@ func TestState_NewValueIsAbsentAndNotPersisted(t *testing.T) { func TestState_PresentZeroValueIsDistinctFromAbsent(t *testing.T) { forEachStateBackend(t, func(t *testing.T, backend store.Store) { id := GrainId{GrainType: "account", GrainKey: "alice"} - binder := newTestBinder(id, backend, nil, clock.Real{}) - value := NewState[int](binder, "value") + grain := newTestGrainContext(id, backend, nil, clock.Real{}) + value := NewState[int](grain, "value") if err := value.Set(context.Background(), 0); err != nil { t.Fatalf("Set zero: %v", err) @@ -191,7 +195,7 @@ func TestState_PresentZeroValueIsDistinctFromAbsent(t *testing.T) { t.Fatalf("present zero State Get = %d, want zero", value.Get()) } - restarted := newTestBinder(id, backend, nil, clock.Real{}) + restarted := newTestGrainContext(id, backend, nil, clock.Real{}) restartedValue := NewState[int](restarted, "value") if err := restarted.load(context.Background()); err != nil { t.Fatalf("load: %v", err) @@ -205,8 +209,8 @@ func TestState_PresentZeroValueIsDistinctFromAbsent(t *testing.T) { func TestState_ClearKeepsEmptyRecordAndAbsenceAfterRestart(t *testing.T) { forEachStateBackend(t, func(t *testing.T, backend store.Store) { id := GrainId{GrainType: "account", GrainKey: "alice"} - binder := newTestBinder(id, backend, nil, clock.Real{}) - value := NewState[int](binder, "value") + grain := newTestGrainContext(id, backend, nil, clock.Real{}) + value := NewState[int](grain, "value") if err := value.Set(context.Background(), 9); err != nil { t.Fatalf("Set: %v", err) } @@ -220,7 +224,7 @@ func TestState_ClearKeepsEmptyRecordAndAbsenceAfterRestart(t *testing.T) { t.Fatalf("cleared State Get = %d, want zero", value.Get()) } - record, err := backend.Read(context.Background(), store.GrainId(id)) + record, err := backend.Read(context.Background(), toStoreGrainID(id)) if err != nil { t.Fatalf("Read after Clear: %v", err) } @@ -228,7 +232,7 @@ func TestState_ClearKeepsEmptyRecordAndAbsenceAfterRestart(t *testing.T) { t.Fatalf("record after Clear = %#v, want empty record with ETag 2", record) } - restarted := newTestBinder(id, backend, nil, clock.Real{}) + restarted := newTestGrainContext(id, backend, nil, clock.Real{}) restartedValue := NewState[int](restarted, "value") if err := restarted.load(context.Background()); err != nil { t.Fatalf("restart load: %v", err) @@ -241,9 +245,9 @@ func TestState_ClearKeepsEmptyRecordAndAbsenceAfterRestart(t *testing.T) { func TestState_ClearPreservesOtherPresentValues(t *testing.T) { forEachStateBackend(t, func(t *testing.T, backend store.Store) { - binder := newTestBinder(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) - first := NewState[int](binder, "first") - second := NewState[string](binder, "second") + grain := newTestGrainContext(GrainId{GrainType: "account", GrainKey: "alice"}, backend, nil, clock.Real{}) + first := NewState[int](grain, "first") + second := NewState[string](grain, "second") if err := first.Set(context.Background(), 1); err != nil { t.Fatalf("first Set: %v", err) } @@ -263,7 +267,7 @@ func TestState_ClearPreservesOtherPresentValues(t *testing.T) { t.Fatalf("second State = (exists=%v, value=%q), want (true, updated)", second.Exists(), second.Get()) } - record, err := backend.Read(context.Background(), binder.identity) + record, err := backend.Read(context.Background(), grain.identity) if err != nil { t.Fatalf("Read: %v", err) } @@ -276,22 +280,22 @@ func TestState_ClearPreservesOtherPresentValues(t *testing.T) { func TestState_ClearConflictLeavesPresenceAndValueUnchanged(t *testing.T) { forEachStateBackend(t, func(t *testing.T, backend store.Store) { id := GrainId{GrainType: "account", GrainKey: "alice"} - first := newTestBinder(id, backend, nil, clock.Real{}) + first := newTestGrainContext(id, backend, nil, clock.Real{}) firstValue := NewState[int](first, "value") if err := firstValue.Set(context.Background(), 1); err != nil { t.Fatalf("first Set: %v", err) } - second := newTestBinder(id, backend, nil, clock.Real{}) + second := newTestGrainContext(id, backend, nil, clock.Real{}) secondValue := NewState[int](second, "value") if err := second.load(context.Background()); err != nil { t.Fatalf("second load: %v", err) } - record, err := backend.Read(context.Background(), store.GrainId(id)) + record, err := backend.Read(context.Background(), toStoreGrainID(id)) if err != nil { t.Fatalf("Read before external update: %v", err) } - if _, err := backend.Write(context.Background(), store.GrainId(id), []byte(`{"value":2}`), record.ETag); err != nil { + if _, err := backend.Write(context.Background(), toStoreGrainID(id), []byte(`{"value":2}`), record.ETag); err != nil { t.Fatalf("external Write: %v", err) } @@ -310,14 +314,14 @@ func TestState_ClearConflictLeavesPresenceAndValueUnchanged(t *testing.T) { func TestState_ClearStoreFailureLeavesPresenceAndValueUnchanged(t *testing.T) { writeErr := errors.New("store unavailable") - binder := newTestBinder( + grain := newTestGrainContext( GrainId{GrainType: "account", GrainKey: "alice"}, failingWriteStore{err: writeErr, record: store.Record{Data: []byte(`{"value":1}`), ETag: 1}}, nil, clock.Real{}, ) - value := NewState[int](binder, "value") - if err := binder.load(context.Background()); err != nil { + value := NewState[int](grain, "value") + if err := grain.load(context.Background()); err != nil { t.Fatalf("load: %v", err) } @@ -328,8 +332,8 @@ func TestState_ClearStoreFailureLeavesPresenceAndValueUnchanged(t *testing.T) { if !value.Exists() || value.Get() != 1 { t.Fatalf("State after store failure = (exists=%v, value=%d), want (true, 1)", value.Exists(), value.Get()) } - if !errors.Is(binder.discardError(), writeErr) { - t.Fatalf("discard marker = %v, want %v", binder.discardError(), writeErr) + if !errors.Is(grain.discardError(), writeErr) { + t.Fatalf("discard marker = %v, want %v", grain.discardError(), writeErr) } } diff --git a/stop_coordination_test.go b/stop_coordination_test.go index 77a8569..b2bf0a5 100644 --- a/stop_coordination_test.go +++ b/stop_coordination_test.go @@ -30,13 +30,14 @@ func TestRootLifecycle_KillDuringCloseEscalatesAndCancels(t *testing.T) { } return dispatchAccount(ctx, instance, method, args, reply) }) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) - id := GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"} admittedDone := make(chan error, 1) go func() { admittedDone <- rt.Invoke(context.Background(), id, "Block", nil, nil) @@ -46,7 +47,7 @@ func TestRootLifecycle_KillDuringCloseEscalatesAndCancels(t *testing.T) { closeDone := make(chan struct{}) go func() { - rt.Close() + closeRuntime(rt) close(closeDone) }() synctest.Wait() @@ -59,7 +60,7 @@ func TestRootLifecycle_KillDuringCloseEscalatesAndCancels(t *testing.T) { killDone := make(chan struct{}) go func() { - rt.Kill() + killRuntime(rt) close(killDone) }() synctest.Wait() @@ -98,13 +99,14 @@ func TestRootLifecycle_KillDoesNotWaitForUserMethod(t *testing.T) { } return dispatchAccount(ctx, instance, method, args, reply) }) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) - id := GrainId{GrainType: TypeName[Account](), GrainKey: "alice"} + id := GrainId{GrainType: GrainType("gor.Account"), GrainKey: "alice"} go func() { _ = rt.Invoke(context.Background(), id, "Block", nil, nil) }() @@ -113,7 +115,7 @@ func TestRootLifecycle_KillDoesNotWaitForUserMethod(t *testing.T) { killDone := make(chan struct{}) go func() { - rt.Kill() + killRuntime(rt) close(killDone) }() synctest.Wait() @@ -136,7 +138,7 @@ func TestRootLifecycle_KillDoesNotWaitForUserMethod(t *testing.T) { } // TestScenario_ConcurrentCloseDrainsAllAdmitted is the concurrent-close -// scenario: several calls admitted across several entities before Close, more +// scenario: several calls admitted across several grains before Close, more // calls arriving during Close, and Close returns only after every admitted call // has released. Late calls are rejected at the admission gate. func TestScenario_ConcurrentCloseDrainsAllAdmitted(t *testing.T) { @@ -152,16 +154,17 @@ func TestScenario_ConcurrentCloseDrainsAllAdmitted(t *testing.T) { } return dispatchAccount(ctx, instance, method, args, reply) }) - if err := Register[Account](rt, func(b *Binder) Account { + if err := Register[Account](rt, func(b *GrainContext) Account { return &account{value: NewState[int64](b, "value")} }); err != nil { t.Fatal(err) } + mustStart(t, rt) ids := []GrainId{ - {GrainType: TypeName[Account](), GrainKey: "a"}, - {GrainType: TypeName[Account](), GrainKey: "b"}, - {GrainType: TypeName[Account](), GrainKey: "c"}, + {GrainType: GrainType("gor.Account"), GrainKey: "a"}, + {GrainType: GrainType("gor.Account"), GrainKey: "b"}, + {GrainType: GrainType("gor.Account"), GrainKey: "c"}, } admitted := make([]chan error, len(ids)) for i := range ids { @@ -177,7 +180,7 @@ func TestScenario_ConcurrentCloseDrainsAllAdmitted(t *testing.T) { closeDone := make(chan struct{}) go func() { - rt.Close() + closeRuntime(rt) close(closeDone) }() synctest.Wait() diff --git a/store/benchmark_filesystem_linux_test.go b/store/benchmark_filesystem_linux_test.go new file mode 100644 index 0000000..815cb5e --- /dev/null +++ b/store/benchmark_filesystem_linux_test.go @@ -0,0 +1,13 @@ +//go:build linux + +package store + +import "syscall" + +func benchmarkFileSystemMagic(path string) (uint64, bool, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(path, &stat); err != nil { + return 0, false, err + } + return uint64(stat.Type), true, nil +} diff --git a/store/benchmark_filesystem_other_test.go b/store/benchmark_filesystem_other_test.go new file mode 100644 index 0000000..59c6691 --- /dev/null +++ b/store/benchmark_filesystem_other_test.go @@ -0,0 +1,7 @@ +//go:build !linux + +package store + +func benchmarkFileSystemMagic(string) (uint64, bool, error) { + return 0, false, nil +} diff --git a/store/benchmark_test.go b/store/benchmark_test.go index f796089..f6a6798 100644 --- a/store/benchmark_test.go +++ b/store/benchmark_test.go @@ -4,7 +4,6 @@ import ( "context" "os" "path/filepath" - "syscall" "testing" ) @@ -66,15 +65,18 @@ func benchmarkRealDiskDir(b *testing.B) string { } }) - var stat syscall.Statfs_t - if err := syscall.Statfs(dir, &stat); err != nil { + magic, known, err := benchmarkFileSystemMagic(dir) + if err != nil { b.Fatal(err) } + if !known { + b.Log("benchmark data path: file-system type check is not available on this system") + return dir + } const ( tmpfsSuperMagic = 0x01021994 ramfsSuperMagic = 0x858458f6 ) - magic := uint64(stat.Type) b.Logf("benchmark data path: statfs magic %#x", magic) if magic == tmpfsSuperMagic || magic == ramfsSuperMagic { b.Fatalf("benchmark data path %q is on an in-memory filesystem (statfs magic %#x); use real disk storage", dir, magic) diff --git a/store/check_test.go b/store/check_test.go new file mode 100644 index 0000000..1879dbd --- /dev/null +++ b/store/check_test.go @@ -0,0 +1,254 @@ +package store + +import ( + "context" + "errors" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestMemoryStore_CheckHonorsContext(t *testing.T) { + memory := NewMemory() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if err := memory.Check(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("Check error = %v, want context.Canceled", err) + } + if err := memory.Check(context.Background()); err != nil { + t.Fatalf("Check with live context: %v", err) + } +} + +func TestMemoryStore_CheckDoesNotChangeData(t *testing.T) { + memory := NewMemory() + ctx := context.Background() + id := GrainId{GrainType: "account", GrainKey: "alice"} + if _, err := memory.Write(ctx, id, []byte("state"), 0); err != nil { + t.Fatalf("Write: %v", err) + } + reminder := Reminder{ + GrainId: id, + Name: "wake", + Method: "Wake", + DueAt: time.Unix(10, 0).UTC(), + } + if err := memory.Put(ctx, reminder); err != nil { + t.Fatalf("Put: %v", err) + } + + if err := memory.Check(ctx); err != nil { + t.Fatalf("Check: %v", err) + } + state, err := memory.Read(ctx, id) + if err != nil { + t.Fatalf("Read after Check: %v", err) + } + if string(state.Data) != "state" || state.ETag != 1 { + t.Fatalf("state after Check = %#v, want unchanged state", state) + } + reminders := listReminderRows(t, memory, reminder.DueAt) + if len(reminders) != 1 || reminders[0].ETag != 1 { + t.Fatalf("Reminders after Check = %#v, want one unchanged row", reminders) + } +} + +func TestSQLiteStore_CheckHonorsContext(t *testing.T) { + sqlite := newSQLiteTestStore(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if err := sqlite.Check(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("Check error = %v, want context.Canceled", err) + } +} + +func TestSQLiteStore_CheckDoesNotChangeData(t *testing.T) { + sqlite := newSQLiteTestStore(t) + ctx := context.Background() + id := GrainId{GrainType: "account", GrainKey: "alice"} + if _, err := sqlite.Write(ctx, id, []byte("state"), 0); err != nil { + t.Fatalf("Write: %v", err) + } + reminder := Reminder{ + GrainId: id, + Name: "wake", + Method: "Wake", + DueAt: time.Unix(10, 0).UTC(), + } + if err := sqlite.Put(ctx, reminder); err != nil { + t.Fatalf("Put: %v", err) + } + + if err := sqlite.Check(ctx); err != nil { + t.Fatalf("Check: %v", err) + } + state, err := sqlite.Read(ctx, id) + if err != nil { + t.Fatalf("Read after Check: %v", err) + } + if string(state.Data) != "state" || state.ETag != 1 { + t.Fatalf("state after Check = %#v, want unchanged state", state) + } + reminders := listReminderRows(t, sqlite, reminder.DueAt) + if len(reminders) != 1 || reminders[0].ETag != 1 { + t.Fatalf("Reminders after Check = %#v, want one unchanged row", reminders) + } +} + +func TestSQLiteStore_CheckRejectsMissingStateSchema(t *testing.T) { + sqlite := newSQLiteTestStore(t) + if _, err := sqlite.stateWriteDB.Exec("DROP TABLE records"); err != nil { + t.Fatalf("drop records table: %v", err) + } + + err := sqlite.Check(context.Background()) + if err == nil || !strings.Contains(err.Error(), "records") { + t.Fatalf("Check error = %v, want missing records schema error", err) + } +} + +func TestSQLiteStore_CheckRejectsMissingReminderSchema(t *testing.T) { + sqlite := newSQLiteTestStore(t) + if _, err := sqlite.writeDB.Exec("DROP TABLE schedule"); err != nil { + t.Fatalf("drop schedule table: %v", err) + } + + err := sqlite.Check(context.Background()) + if err == nil || !strings.Contains(err.Error(), "schedule") { + t.Fatalf("Check error = %v, want missing schedule schema error", err) + } +} + +func TestSQLiteStore_CheckRejectsMissingReminderDueIndex(t *testing.T) { + sqlite := newSQLiteTestStore(t) + if _, err := sqlite.writeDB.Exec("DROP INDEX schedule_due_idx"); err != nil { + t.Fatalf("drop Reminder due index: %v", err) + } + + err := sqlite.Check(context.Background()) + if err == nil || !strings.Contains(err.Error(), "Reminder due index") { + t.Fatalf("Check error = %v, want missing Reminder due index error", err) + } +} + +func TestSQLiteStore_CheckRejectsInvalidReminderDueIndex(t *testing.T) { + sqlite := newSQLiteTestStore(t) + if _, err := sqlite.writeDB.Exec(` + DROP INDEX schedule_due_idx; + CREATE INDEX schedule_due_idx + ON schedule (grain_type, due_at, grain_key, name)`); err != nil { + t.Fatalf("replace Reminder due index: %v", err) + } + + err := sqlite.Check(context.Background()) + if err == nil || !strings.Contains(err.Error(), "Reminder due index") { + t.Fatalf("Check error = %v, want invalid Reminder due index error", err) + } +} + +func TestSQLiteStore_CheckRejectsPartialReminderDueIndex(t *testing.T) { + sqlite := newSQLiteTestStore(t) + if _, err := sqlite.writeDB.Exec(` + DROP INDEX schedule_due_idx; + CREATE INDEX schedule_due_idx + ON schedule (due_at, grain_type, grain_key, name) + WHERE due_at < 0`); err != nil { + t.Fatalf("replace Reminder due index: %v", err) + } + + err := sqlite.Check(context.Background()) + if err == nil || !strings.Contains(err.Error(), "Reminder due index") { + t.Fatalf("Check error = %v, want partial Reminder due index error", err) + } +} + +func TestSQLiteStore_CheckRejectsMissingReminderVersion(t *testing.T) { + sqlite := newSQLiteTestStore(t) + if _, err := sqlite.writeDB.Exec("DROP TABLE schedule_version"); err != nil { + t.Fatalf("drop Reminder version table: %v", err) + } + + err := sqlite.Check(context.Background()) + if err == nil || !strings.Contains(err.Error(), "Reminder version") { + t.Fatalf("Check error = %v, want missing Reminder version error", err) + } +} + +func TestSQLiteStore_CheckRejectsOldReminderVersion(t *testing.T) { + sqlite := newSQLiteTestStore(t) + if err := sqlite.Put(context.Background(), Reminder{ + GrainId: GrainId{GrainType: "account", GrainKey: "alice"}, + Name: "wake", + Method: "Wake", + DueAt: time.Unix(10, 0).UTC(), + }); err != nil { + t.Fatalf("Put Reminder: %v", err) + } + if _, err := sqlite.writeDB.Exec("UPDATE schedule_version SET etag = 0 WHERE id = 1"); err != nil { + t.Fatalf("make Reminder version old: %v", err) + } + + err := sqlite.Check(context.Background()) + if err == nil || !strings.Contains(err.Error(), "Reminder version") { + t.Fatalf("Check error = %v, want old Reminder version error", err) + } +} + +func TestSQLiteStore_CheckRejectsInvalidStatePrimaryKey(t *testing.T) { + sqlite := newSQLiteTestStore(t) + if _, err := sqlite.stateWriteDB.Exec(` + DROP TABLE records; + CREATE TABLE records ( + identity_type TEXT NOT NULL, + identity_key TEXT NOT NULL, + data BLOB NOT NULL, + etag INTEGER NOT NULL + )`); err != nil { + t.Fatalf("replace records table: %v", err) + } + + err := sqlite.Check(context.Background()) + if err == nil || !strings.Contains(err.Error(), "primary key") { + t.Fatalf("Check error = %v, want invalid records primary key error", err) + } +} + +func TestSQLiteStore_CheckRejectsInvalidReminderPrimaryKey(t *testing.T) { + sqlite := newSQLiteTestStore(t) + if _, err := sqlite.writeDB.Exec(` + DROP TABLE schedule; + CREATE TABLE schedule ( + grain_type TEXT NOT NULL, + grain_key TEXT NOT NULL, + name TEXT NOT NULL, + method TEXT NOT NULL, + first_tick_time INTEGER NOT NULL, + due_at INTEGER NOT NULL, + interval INTEGER NOT NULL, + etag INTEGER NOT NULL + )`); err != nil { + t.Fatalf("replace schedule table: %v", err) + } + + err := sqlite.Check(context.Background()) + if err == nil || !strings.Contains(err.Error(), "primary key") { + t.Fatalf("Check error = %v, want invalid schedule primary key error", err) + } +} + +func TestSQLiteStore_CheckRejectsClosedDatabase(t *testing.T) { + sqlite, err := OpenSQLite(filepath.Join(t.TempDir(), "store.db")) + if err != nil { + t.Fatalf("OpenSQLite: %v", err) + } + if err := sqlite.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + if err := sqlite.Check(context.Background()); err == nil { + t.Fatal("Check after Close returned nil, want availability error") + } +} diff --git a/store/contract_test.go b/store/contract_test.go new file mode 100644 index 0000000..c8d470b --- /dev/null +++ b/store/contract_test.go @@ -0,0 +1,252 @@ +package store + +import ( + "context" + "errors" + "path/filepath" + "testing" +) + +type storeContractFactory func(*testing.T) Store + +func TestStoreContract(t *testing.T) { + for _, backend := range []struct { + name string + open storeContractFactory + }{ + {name: "memory", open: func(*testing.T) Store { return NewMemory() }}, + {name: "sqlite", open: func(t *testing.T) Store { + s, err := OpenSQLite(filepath.Join(t.TempDir(), "store.db")) + if err != nil { + t.Fatalf("OpenSQLite: %v", err) + } + t.Cleanup(func() { + if err := s.Close(); err != nil { + t.Errorf("Close SQLite: %v", err) + } + }) + return s + }}, + } { + t.Run(backend.name, func(t *testing.T) { + runStoreContract(t, backend.open) + }) + } +} + +func runStoreContract(t *testing.T, open storeContractFactory) { + t.Helper() + t.Run("write_and_read", func(t *testing.T) { + backend := open(t) + id := GrainId{GrainType: "account", GrainKey: "alice"} + etag, err := backend.Write(context.Background(), id, []byte("first"), 0) + if err != nil { + t.Fatalf("Write: %v", err) + } + if etag != 1 { + t.Fatalf("ETag = %d, want 1", etag) + } + record, err := backend.Read(context.Background(), id) + if err != nil { + t.Fatalf("Read: %v", err) + } + if string(record.Data) != "first" || record.ETag != etag { + t.Fatalf("Record = %#v, want data first and ETag %d", record, etag) + } + nextETag, err := backend.Write(context.Background(), id, []byte("second"), etag) + if err != nil { + t.Fatalf("second Write: %v", err) + } + if nextETag != 2 { + t.Fatalf("second ETag = %d, want 2", nextETag) + } + }) + + t.Run("conflict_leaves_record", func(t *testing.T) { + backend := open(t) + id := GrainId{GrainType: "account", GrainKey: "alice"} + etag, err := backend.Write(context.Background(), id, []byte("original"), 0) + if err != nil { + t.Fatalf("seed Write: %v", err) + } + newETag, err := backend.Write(context.Background(), id, []byte("replacement"), etag+1) + if !errors.Is(err, ErrConflict) { + t.Fatalf("Write error = %v, want ErrConflict", err) + } + if newETag != 0 { + t.Fatalf("conflicting ETag = %d, want 0", newETag) + } + record, err := backend.Read(context.Background(), id) + if err != nil { + t.Fatalf("Read: %v", err) + } + if string(record.Data) != "original" || record.ETag != etag { + t.Fatalf("Record after conflict = %#v, want original data and ETag %d", record, etag) + } + }) + + t.Run("zero_ETag_conflicts_with_existing", func(t *testing.T) { + backend := open(t) + id := GrainId{GrainType: "account", GrainKey: "alice"} + if _, err := backend.Write(context.Background(), id, []byte("existing"), 0); err != nil { + t.Fatalf("seed Write: %v", err) + } + if _, err := backend.Write(context.Background(), id, []byte("overwrite"), 0); !errors.Is(err, ErrConflict) { + t.Fatalf("Write error = %v, want ErrConflict", err) + } + }) + + t.Run("nonzero_ETag_conflicts_with_missing", func(t *testing.T) { + backend := open(t) + id := GrainId{GrainType: "account", GrainKey: "missing"} + if _, err := backend.Write(context.Background(), id, []byte("unexpected"), 5); !errors.Is(err, ErrConflict) { + t.Fatalf("Write error = %v, want ErrConflict", err) + } + }) + + t.Run("nil_data_is_rejected", func(t *testing.T) { + backend := open(t) + id := GrainId{GrainType: "account", GrainKey: "alice"} + if _, err := backend.Write(context.Background(), id, nil, 0); !errors.Is(err, ErrInvalidRecordData) { + t.Fatalf("Write error = %v, want ErrInvalidRecordData", err) + } + record, err := backend.Read(context.Background(), id) + if err != nil { + t.Fatalf("Read: %v", err) + } + if record.Data != nil || record.ETag != 0 { + t.Fatalf("Record after nil Write = %#v, want zero Record", record) + } + }) + + t.Run("empty_data_is_rejected", func(t *testing.T) { + backend := open(t) + id := GrainId{GrainType: "account", GrainKey: "alice"} + if _, err := backend.Write(context.Background(), id, []byte{}, 0); !errors.Is(err, ErrInvalidRecordData) { + t.Fatalf("Write error = %v, want ErrInvalidRecordData", err) + } + record, err := backend.Read(context.Background(), id) + if err != nil { + t.Fatalf("Read: %v", err) + } + if record.Data != nil || record.ETag != 0 { + t.Fatalf("Record after empty Write = %#v, want zero Record", record) + } + }) + + t.Run("concurrent_compare_and_swap", func(t *testing.T) { + backend := open(t) + id := GrainId{GrainType: "account", GrainKey: "alice"} + etag, err := backend.Write(context.Background(), id, []byte("seed"), 0) + if err != nil { + t.Fatalf("seed Write: %v", err) + } + start := make(chan struct{}) + results := make(chan error, 2) + for _, data := range [][]byte{[]byte("first"), []byte("second")} { + go func(data []byte) { + <-start + _, err := backend.Write(context.Background(), id, data, etag) + results <- err + }(data) + } + close(start) + + var successes, conflicts int + for range 2 { + err := <-results + switch { + case err == nil: + successes++ + case errors.Is(err, ErrConflict): + conflicts++ + default: + t.Fatalf("concurrent Write error = %v", err) + } + } + if successes != 1 || conflicts != 1 { + t.Fatalf("concurrent results = (success=%d, conflict=%d), want (1, 1)", successes, conflicts) + } + record, err := backend.Read(context.Background(), id) + if err != nil { + t.Fatalf("Read: %v", err) + } + if record.ETag != etag+1 || (string(record.Data) != "first" && string(record.Data) != "second") { + t.Fatalf("Record after concurrent Writes = %#v", record) + } + }) + + t.Run("missing_is_zero_record", func(t *testing.T) { + backend := open(t) + record, err := backend.Read(context.Background(), GrainId{GrainType: "account", GrainKey: "missing"}) + if err != nil { + t.Fatalf("Read: %v", err) + } + if record.Data != nil || record.ETag != 0 { + t.Fatalf("Record = %#v, want zero Record", record) + } + }) + + t.Run("GrainIds_are_independent", func(t *testing.T) { + backend := open(t) + alice := GrainId{GrainType: "account", GrainKey: "alice"} + bob := GrainId{GrainType: "account", GrainKey: "bob"} + if _, err := backend.Write(context.Background(), alice, []byte("alice"), 0); err != nil { + t.Fatalf("alice Write: %v", err) + } + if _, err := backend.Write(context.Background(), bob, []byte("bob"), 0); err != nil { + t.Fatalf("bob Write: %v", err) + } + aliceRecord, err := backend.Read(context.Background(), alice) + if err != nil { + t.Fatalf("alice Read: %v", err) + } + bobRecord, err := backend.Read(context.Background(), bob) + if err != nil { + t.Fatalf("bob Read: %v", err) + } + if string(aliceRecord.Data) != "alice" || aliceRecord.ETag != 1 { + t.Fatalf("alice Record = %#v", aliceRecord) + } + if string(bobRecord.Data) != "bob" || bobRecord.ETag != 1 { + t.Fatalf("bob Record = %#v", bobRecord) + } + }) + + t.Run("data_is_copied", func(t *testing.T) { + backend := open(t) + id := GrainId{GrainType: "account", GrainKey: "alice"} + data := []byte("original") + if _, err := backend.Write(context.Background(), id, data, 0); err != nil { + t.Fatalf("Write: %v", err) + } + data[0] = 'X' + record, err := backend.Read(context.Background(), id) + if err != nil { + t.Fatalf("Read: %v", err) + } + record.Data[0] = 'Y' + unchanged, err := backend.Read(context.Background(), id) + if err != nil { + t.Fatalf("second Read: %v", err) + } + if string(unchanged.Data) != "original" { + t.Fatalf("stored data = %q, want original", unchanged.Data) + } + }) + + t.Run("canceled_context", func(t *testing.T) { + backend := open(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := backend.Check(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("Check error = %v, want context.Canceled", err) + } + if _, err := backend.Read(ctx, GrainId{GrainType: "account", GrainKey: "alice"}); !errors.Is(err, context.Canceled) { + t.Fatalf("Read error = %v, want context.Canceled", err) + } + if _, err := backend.Write(ctx, GrainId{GrainType: "account", GrainKey: "alice"}, nil, 0); !errors.Is(err, context.Canceled) { + t.Fatalf("Write error = %v, want context.Canceled", err) + } + }) +} diff --git a/store/durability_test.go b/store/durability_test.go index a0f0bdd..868e40b 100644 --- a/store/durability_test.go +++ b/store/durability_test.go @@ -192,8 +192,10 @@ func TestStateFilePath(t *testing.T) { {"dir/foo.bar.db", "dir/foo.bar-state.db"}, } for _, c := range cases { - if got := stateFilePath(c.in); got != c.want { - t.Errorf("stateFilePath(%q) = %q, want %q", c.in, got, c.want) + input := filepath.FromSlash(c.in) + want := filepath.FromSlash(c.want) + if got := stateFilePath(input); got != want { + t.Errorf("stateFilePath(%q) = %q, want %q", input, got, want) } } } diff --git a/store/member.go b/store/member.go index 7c4321e..da87b1f 100644 --- a/store/member.go +++ b/store/member.go @@ -56,7 +56,8 @@ type MemberSnapshot struct { // Implementations must support concurrent calls and the same atomic ETag // compare-and-swap rule as Store. WriteMember must return an error matching // ErrConflict when the supplied row ETag is stale, and ListMembers must return -// independent snapshot data rather than mutable storage-owned maps. +// independent snapshot data rather than mutable storage-owned maps. Both +// methods must honor context cancellation. type MemberStore interface { // WriteMember creates or replaces one member row using member.ETag as the // expected version and returns the new ETag. diff --git a/store/migration_test.go b/store/migration_test.go index 7f904f4..896b9fb 100644 --- a/store/migration_test.go +++ b/store/migration_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "path/filepath" + "strconv" "strings" "testing" "time" @@ -174,10 +175,7 @@ func TestMigrate_OldDatabaseReadsBackEveryConfirmedState(t *testing.T) { } } - reminders, err := s.ListDue(context.Background(), time.Unix(0, 100000).UTC()) - if err != nil { - t.Fatalf("ListDue: %v", err) - } + reminders := listReminderRows(t, s, time.Unix(0, 100000).UTC()) if len(reminders) != 2 { t.Fatalf("ListDue returned %d reminders, want 2", len(reminders)) } @@ -195,21 +193,49 @@ func TestMigrate_OldDatabaseReadsBackEveryConfirmedState(t *testing.T) { if len(members.Members) != 2 { t.Fatalf("ListMembers returned %d members, want 2", len(members.Members)) } - if members.Members[0].NodeAddr != "10.0.0.1" || members.Members[0].Status != MemberActive || members.Members[0].ETag != 1 { - t.Fatalf("member[0] = %#v, want 10.0.0.1 active etag 1", members.Members[0]) + firstMember := members.Members[0] + if firstMember.NodeAddr != "10.0.0.1" || firstMember.Generation != "gen-1" || + firstMember.Status != MemberActive || !firstMember.IamAliveAt.Equal(time.Unix(0, 2000).UTC()) || + len(firstMember.SuspectVotes) != 0 || firstMember.ETag != 1 { + t.Fatalf("member[0] = %#v, want exact active gen-1 row", firstMember) } - if members.Members[1].NodeAddr != "10.0.0.2" || members.Members[1].Status != MemberJoining || members.Members[1].ETag != 2 || len(members.Members[1].SuspectVotes) != 1 { - t.Fatalf("member[1] = %#v, want 10.0.0.2 joining etag 2 with one vote", members.Members[1]) + secondMember := members.Members[1] + vote, found := secondMember.SuspectVotes[MemberID{NodeAddr: "10.0.0.1", Generation: "gen-1"}] + if secondMember.NodeAddr != "10.0.0.2" || secondMember.Generation != "gen-2" || + secondMember.Status != MemberJoining || !secondMember.IamAliveAt.Equal(time.Unix(0, 3000).UTC()) || + len(secondMember.SuspectVotes) != 1 || !found || !vote.ExpiresAt.Equal(time.Unix(0, 4000).UTC()) || secondMember.ETag != 2 { + t.Fatalf("member[1] = %#v, want exact joining gen-2 row with gen-1 vote", secondMember) } main := openRawSQLite(t, path) state := openRawSQLite(t, stateFilePath(path)) + if err := checkSQLiteIndex(context.Background(), main, "schedule", "schedule_due_idx", reminderDueIndexColumns()); err != nil { + t.Fatalf("migrated Reminder due index: %v", err) + } if has, err := tableExists(main, "records"); err != nil || has { t.Fatalf("main file records table after migration: has=%v err=%v, want absent", has, err) } if count := tableRowCount(t, state, "records"); count != records { t.Fatalf("state file records count = %d, want %d", count, records) } + columns := sqliteTableColumns(t, main, "schedule") + for _, column := range []string{"grain_type", "grain_key", "first_tick_time"} { + if !columns[column] { + t.Fatalf("migrated schedule is missing %s: %#v", column, columns) + } + } + for _, column := range []string{"entity_type", "entity_key"} { + if columns[column] { + t.Fatalf("migrated schedule still has %s: %#v", column, columns) + } + } + var scheduleVersion int64 + if err := main.QueryRow(`SELECT etag FROM schedule_version WHERE id = 1`).Scan(&scheduleVersion); err != nil { + t.Fatalf("read migrated schedule version: %v", err) + } + if scheduleVersion != 3 { + t.Fatalf("migrated schedule version = %d, want 3", scheduleVersion) + } reopened, err := OpenSQLite(path, WithDurability(tier)) if err != nil { @@ -223,10 +249,7 @@ func TestMigrate_OldDatabaseReadsBackEveryConfirmedState(t *testing.T) { if string(record.Data) != "data-0" || record.ETag != 5 { t.Fatalf("record after reopen = %#v, want data-0 etag 5", record) } - reopenedReminders, err := reopened.ListDue(context.Background(), time.Unix(0, 100000).UTC()) - if err != nil { - t.Fatalf("ListDue after reopen: %v", err) - } + reopenedReminders := listReminderRows(t, reopened, time.Unix(0, 100000).UTC()) if len(reopenedReminders) != 2 || !reopenedReminders[0].FirstTickTime.Equal(reopenedReminders[0].DueAt) || !reopenedReminders[1].FirstTickTime.Equal(reopenedReminders[1].DueAt) { t.Fatalf("reminders after reopen = %#v, want fallback FirstTickTime values", reopenedReminders) } @@ -234,6 +257,201 @@ func TestMigrate_OldDatabaseReadsBackEveryConfirmedState(t *testing.T) { } } +func sqliteTableColumns(t *testing.T, db *sql.DB, table string) map[string]bool { + t.Helper() + rows, err := db.Query(`PRAGMA table_info(` + table + `)`) + if err != nil { + t.Fatalf("read %s columns: %v", table, err) + } + defer rows.Close() + columns := make(map[string]bool) + for rows.Next() { + var ( + cid int + name string + columnType string + notNull int + defaultVal any + primaryKey int + ) + if err := rows.Scan(&cid, &name, &columnType, ¬Null, &defaultVal, &primaryKey); err != nil { + t.Fatalf("scan %s columns: %v", table, err) + } + columns[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("read %s columns: %v", table, err) + } + return columns +} + +func TestMigrate_ReminderIdentityColumnsRejectIncompleteOrMixedSchema(t *testing.T) { + for _, test := range []struct { + name string + columns string + }{ + {name: "mixed", columns: "entity_type TEXT NOT NULL, entity_key TEXT NOT NULL, grain_type TEXT NOT NULL, grain_key TEXT NOT NULL"}, + {name: "crossed", columns: "entity_type TEXT NOT NULL, grain_key TEXT NOT NULL"}, + {name: "missing key", columns: "entity_type TEXT NOT NULL"}, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "gor.db") + db := openRawSQLite(t, path) + if _, err := db.Exec(`CREATE TABLE schedule (` + test.columns + `, + name TEXT NOT NULL, + method TEXT NOT NULL, + due_at INTEGER NOT NULL, + interval INTEGER NOT NULL, + etag INTEGER NOT NULL + )`); err != nil { + t.Fatalf("create unsupported schedule schema: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close unsupported schedule schema: %v", err) + } + + _, err := OpenSQLite(path) + if err == nil || !strings.Contains(err.Error(), "unsupported Reminder identity columns") { + t.Fatalf("OpenSQLite error = %v, want unsupported Reminder identity columns", err) + } + + check := openRawSQLite(t, path) + if columns := sqliteTableColumns(t, check, "schedule"); columns["first_tick_time"] { + t.Fatalf("failed migration added first_tick_time: %#v", columns) + } + for _, table := range []string{"schedule_version", "member"} { + found, err := tableExists(check, table) + if err != nil { + t.Fatalf("check %s after failed migration: %v", table, err) + } + if found { + t.Fatalf("failed migration created %s", table) + } + } + }) + } +} + +func TestMigrate_PartialReminderSchemaCompletesOnOpen(t *testing.T) { + path := filepath.Join(t.TempDir(), "gor.db") + db := openRawSQLite(t, path) + if _, err := db.Exec(` + CREATE TABLE schedule ( + grain_type TEXT NOT NULL, + grain_key TEXT NOT NULL, + name TEXT NOT NULL, + method TEXT NOT NULL, + first_tick_time INTEGER NOT NULL, + due_at INTEGER NOT NULL, + interval INTEGER NOT NULL, + etag INTEGER NOT NULL, + PRIMARY KEY (grain_type, grain_key, name) + ); + CREATE INDEX schedule_due_idx + ON schedule (due_at, grain_type, grain_key, name) + WHERE due_at < 0; + INSERT INTO schedule VALUES ('account', 'alice', 'wake', 'Wake', 10, 10, 0, 7) + `); err != nil { + t.Fatalf("create partial Reminder schema: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close partial database: %v", err) + } + + first, err := OpenSQLite(path) + if err != nil { + t.Fatalf("OpenSQLite partial schema: %v", err) + } + if err := first.Check(context.Background()); err != nil { + first.Close() + t.Fatalf("Check migrated schema: %v", err) + } + rows := listReminderRows(t, first, time.Unix(0, 10).UTC()) + if len(rows) != 1 || rows[0].ETag != 7 { + first.Close() + t.Fatalf("migrated Reminder = %#v, want active ETag 7", rows) + } + if err := first.Close(); err != nil { + t.Fatalf("close first Store: %v", err) + } + + second, err := OpenSQLite(path) + if err != nil { + t.Fatalf("reopen migrated schema: %v", err) + } + defer second.Close() + if err := second.Check(context.Background()); err != nil { + t.Fatalf("Check reopened schema: %v", err) + } +} + +func TestMigrate_ReminderSchemaFailureRollsBackCoordinationSchema(t *testing.T) { + path := filepath.Join(t.TempDir(), "gor.db") + db := openRawSQLite(t, path) + if _, err := db.Exec(` + CREATE TABLE schedule ( + grain_type TEXT NOT NULL, + grain_key TEXT NOT NULL, + name TEXT NOT NULL, + method TEXT NOT NULL, + first_tick_time INTEGER NOT NULL, + due_at INTEGER NOT NULL, + interval INTEGER NOT NULL, + etag INTEGER NOT NULL, + PRIMARY KEY (grain_type, grain_key, name) + ); + INSERT INTO schedule VALUES ('account', 'alice', 'wake', 'Wake', 10, 10, 0, 7); + CREATE VIEW schedule_due_idx AS SELECT due_at FROM schedule + `); err != nil { + t.Fatalf("create interrupted Reminder schema: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close interrupted database: %v", err) + } + + opened, err := OpenSQLite(path) + if err == nil { + opened.Close() + t.Fatal("OpenSQLite error = nil, want due index replacement failure") + } + + db = openRawSQLite(t, path) + for _, table := range []string{"schedule_version", "member"} { + found, err := tableExists(db, table) + if err != nil { + db.Close() + t.Fatalf("check %s after failed migration: %v", table, err) + } + if found { + db.Close() + t.Fatalf("failed migration created %s", table) + } + } + if _, err := db.Exec(`DROP VIEW schedule_due_idx`); err != nil { + db.Close() + t.Fatalf("remove migration blocker: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close repaired database: %v", err) + } + + reopened, err := OpenSQLite(path) + if err != nil { + t.Fatalf("OpenSQLite after repair: %v", err) + } + defer reopened.Close() + if err := reopened.Check(context.Background()); err != nil { + t.Fatalf("Check repaired schema: %v", err) + } + var version int64 + if err := reopened.readDB.QueryRow(`SELECT etag FROM schedule_version WHERE id = 1`).Scan(&version); err != nil { + t.Fatalf("read repaired Reminder version: %v", err) + } + if version != 7 { + t.Fatalf("Reminder version after retry = %d, want 7", version) + } +} + func TestMigrate_InterruptedMigrationCompletesOnReopen(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "gor.db") @@ -334,6 +552,59 @@ func TestMigrate_CountMismatchAbortsLeavingOldDatabaseIntact(t *testing.T) { } } +func TestMigrate_TargetIntegrityFailureLeavesOldDatabaseIntact(t *testing.T) { + path := filepath.Join(t.TempDir(), "gor.db") + const records = 10 + buildOldLayoutDB(t, path, records) + + statePath := stateFilePath(path) + state, err := sql.Open("sqlite", sqliteDSN(statePath, DurabilityFull)) + if err != nil { + t.Fatalf("open State database: %v", err) + } + if err := createStateSchema(state); err != nil { + t.Fatalf("create State schema: %v", err) + } + if _, err := state.Exec(`CREATE TABLE integrity_probe(value INTEGER CHECK(value > 0))`); err != nil { + t.Fatalf("create integrity probe: %v", err) + } + if _, err := state.Exec(`PRAGMA ignore_check_constraints = ON`); err != nil { + t.Fatalf("disable check constraints: %v", err) + } + if _, err := state.Exec(`INSERT INTO integrity_probe VALUES (-1)`); err != nil { + t.Fatalf("insert invalid integrity probe: %v", err) + } + if err := state.Close(); err != nil { + t.Fatalf("close State database: %v", err) + } + + if _, err := OpenSQLite(path); err == nil || !strings.Contains(err.Error(), "integrity") { + t.Fatalf("OpenSQLite error = %v, want integrity failure", err) + } + main := openRawSQLite(t, path) + if count := tableRowCount(t, main, "records"); count != records { + t.Fatalf("old State rows after failed migration = %d, want %d", count, records) + } +} + +func TestMigrate_PathWithQuote(t *testing.T) { + path := filepath.Join(t.TempDir(), "gor's.db") + buildOldLayoutDB(t, path, 1) + + s, err := OpenSQLite(path) + if err != nil { + t.Fatalf("OpenSQLite: %v", err) + } + defer s.Close() + record, err := s.Read(context.Background(), GrainId{GrainType: "type-0", GrainKey: "key-0"}) + if err != nil { + t.Fatalf("Read: %v", err) + } + if string(record.Data) != "data-0" || record.ETag != 5 { + t.Fatalf("migrated Record = %#v, want data-0 and ETag 5", record) + } +} + func TestMigrate_OldDatabaseWithoutStateRows(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "gor.db") @@ -345,10 +616,7 @@ func TestMigrate_OldDatabaseWithoutStateRows(t *testing.T) { } defer s.Close() - reminders, err := s.ListDue(context.Background(), time.Unix(0, 100000).UTC()) - if err != nil { - t.Fatalf("ListDue: %v", err) - } + reminders := listReminderRows(t, s, time.Unix(0, 100000).UTC()) if len(reminders) != 2 { t.Fatalf("ListDue returned %d reminders, want 2", len(reminders)) } @@ -424,7 +692,7 @@ func TestMigrate_ErrorMentionsDatabasePath(t *testing.T) { if err == nil { t.Fatalf("OpenSQLite: expected error for broken state database") } - if !strings.Contains(err.Error(), path) { + if !strings.Contains(err.Error(), path) && !strings.Contains(err.Error(), strconv.Quote(path)) { t.Fatalf("OpenSQLite error %q does not mention database path %q", err, path) } diff --git a/store/schedule.go b/store/schedule.go index e680c92..ccb54b4 100644 --- a/store/schedule.go +++ b/store/schedule.go @@ -2,6 +2,7 @@ package store import ( "context" + "errors" "sort" "time" ) @@ -10,7 +11,7 @@ import ( // // GrainId and Name identify the row. DueAt is inclusive when queried by // ListDue. FirstTickTime is the first due time for the current setting. -// Interval is retained for the poller, and ETag is the version used by Claim. +// Interval is retained for the poller. ETag is the version used by Claim. type Reminder struct { GrainId GrainId Name string @@ -21,24 +22,45 @@ type Reminder struct { ETag ETag } +// ReminderCursor identifies the last row returned by one Reminder page. +type ReminderCursor struct { + DueAt time.Time + GrainId GrainId + Name string +} + +// ReminderPage contains one bounded page of due Reminders. Next identifies the +// last row in Rows. It is nil when the page is the last page for the query. +type ReminderPage struct { + Rows []Reminder + Next *ReminderCursor +} + // ReminderStore persists Reminders and atomically claims due rows. // // Implementations must support concurrent calls. Claim must compare the row's -// identity, name, and ETag atomically so concurrent claimers using one -// snapshot produce at most one winner. Put and Delete are unconditional +// identity, name, and ETag atomically. Concurrent callers that use one +// snapshot must produce at most one winner. Put and Delete are unconditional // changes by design. type ReminderStore interface { - // ListDue returns every Reminder whose DueAt is no later than now. - ListDue(context.Context, time.Time) ([]Reminder, error) + // Check verifies that the store can serve Reminders without changing stored + // data. + Check(context.Context) error + // ListDue returns at most limit Reminders whose DueAt is no later than now. + // Rows after a non-nil cursor use DueAt, GrainType, GrainKey, and name order. + // Next must identify the last returned row and move forward. The limit must + // be positive. + ListDue(context.Context, time.Time, *ReminderCursor, int) (ReminderPage, error) // Claim compares the Reminder identity, name, and ETag atomically. When - // they match, a non-zero nextDueAt replaces DueAt and increments the stored - // ETag; a zero nextDueAt deletes the Reminder. It returns true only when the - // update or deletion succeeds, and returns false with a nil error when the - // row is absent or its ETag is stale. + // they match, a non-zero nextDueAt replaces DueAt and assigns a newer ETag; + // a zero nextDueAt deletes the Reminder. It returns true only when the update + // or deletion succeeds. It returns false with a nil error when the row is + // absent or its ETag is stale. Claim(context.Context, Reminder, time.Time) (bool, error) - // Put inserts or replaces a Reminder without an ETag precondition. A new - // row receives ETag 1; replacing an existing row increments its current - // ETag. The input ETag is ignored. + // Put inserts or replaces a Reminder without an ETag precondition. The row + // gets a Store-assigned ETag that is newer than all prior Reminder ETags + // from that Store. The input ETag is ignored. An implementation must not + // reuse an ETag after Delete. Put(context.Context, Reminder) error // Delete unconditionally removes the named Reminder, if it exists. Delete(context.Context, GrainId, string) error @@ -53,38 +75,87 @@ func keyForReminder(reminder Reminder) reminderKey { return reminderKey{identity: reminder.GrainId, name: reminder.Name} } -func sortReminders(reminders []Reminder) { - sort.Slice(reminders, func(i, j int) bool { - if reminders[i].DueAt != reminders[j].DueAt { - return reminders[i].DueAt.Before(reminders[j].DueAt) - } - if reminders[i].GrainId.GrainType != reminders[j].GrainId.GrainType { - return reminders[i].GrainId.GrainType < reminders[j].GrainId.GrainType - } - if reminders[i].GrainId.GrainKey != reminders[j].GrainId.GrainKey { - return reminders[i].GrainId.GrainKey < reminders[j].GrainId.GrainKey - } - return reminders[i].Name < reminders[j].Name +var errInvalidReminderPageLimit = errors.New("store: Reminder page limit must be positive") + +func reminderLess(left Reminder, right Reminder) bool { + if !left.DueAt.Equal(right.DueAt) { + return left.DueAt.Before(right.DueAt) + } + if left.GrainId.GrainType != right.GrainId.GrainType { + return left.GrainId.GrainType < right.GrainId.GrainType + } + if left.GrainId.GrainKey != right.GrainId.GrainKey { + return left.GrainId.GrainKey < right.GrainId.GrainKey + } + return left.Name < right.Name +} + +func reminderAfterCursor(reminder Reminder, cursor *ReminderCursor) bool { + if cursor == nil { + return true + } + return reminderLess(Reminder{ + DueAt: cursor.DueAt, + GrainId: cursor.GrainId, + Name: cursor.Name, + }, reminder) +} + +func insertBoundedReminder(rows []Reminder, reminder Reminder, maximum int) []Reminder { + index := sort.Search(len(rows), func(index int) bool { + return !reminderLess(rows[index], reminder) }) + if index >= maximum { + return rows + } + if len(rows) < maximum { + rows = append(rows, Reminder{}) + } + copy(rows[index+1:], rows[index:len(rows)-1]) + rows[index] = reminder + return rows +} + +func reminderPage(rows []Reminder, limit int) ReminderPage { + page := ReminderPage{Rows: rows} + if len(rows) <= limit { + return page + } + page.Rows = rows[:limit] + last := page.Rows[len(page.Rows)-1] + page.Next = &ReminderCursor{DueAt: last.DueAt, GrainId: last.GrainId, Name: last.Name} + return page +} + +func reminderReadLimit(limit int) int { + maximumInt := int(^uint(0) >> 1) + if limit == maximumInt { + return limit + } + return limit + 1 } -// ListDue returns due Reminders in deterministic DueAt, identity, and name +// ListDue returns one bounded page in deterministic DueAt, GrainId, and name // order. -func (m *Memory) ListDue(ctx context.Context, now time.Time) ([]Reminder, error) { +func (m *Memory) ListDue(ctx context.Context, now time.Time, after *ReminderCursor, limit int) (ReminderPage, error) { if err := ctx.Err(); err != nil { - return nil, err + return ReminderPage{}, err + } + if limit <= 0 { + return ReminderPage{}, errInvalidReminderPageLimit } m.mu.RLock() defer m.mu.RUnlock() - result := make([]Reminder, 0) + readLimit := reminderReadLimit(limit) + result := make([]Reminder, 0, min(readLimit, len(m.reminders))) for _, reminder := range m.reminders { - if !reminder.DueAt.After(now) { - result = append(result, reminder) + if reminder.DueAt.After(now) || !reminderAfterCursor(reminder, after) { + continue } + result = insertBoundedReminder(result, reminder, readLimit) } - sortReminders(result) - return result, nil + return reminderPage(result, limit), nil } // Claim atomically checks the Reminder identity, name, and ETag. It returns @@ -108,7 +179,7 @@ func (m *Memory) Claim(ctx context.Context, reminder Reminder, nextDueAt time.Ti return true, nil } current.DueAt = nextDueAt - current.ETag++ + current.ETag = m.nextReminderETag() m.reminders[key] = current return true, nil } @@ -122,15 +193,16 @@ func (m *Memory) Put(ctx context.Context, reminder Reminder) error { defer m.mu.Unlock() key := keyForReminder(reminder) - current, ok := m.reminders[key] - reminder.ETag = 1 - if ok { - reminder.ETag = current.ETag + 1 - } + reminder.ETag = m.nextReminderETag() m.reminders[key] = reminder return nil } +func (m *Memory) nextReminderETag() ETag { + m.reminderETag++ + return m.reminderETag +} + // Delete unconditionally removes the Reminder identified by id and name. func (m *Memory) Delete(ctx context.Context, id GrainId, name string) error { if err := ctx.Err(); err != nil { @@ -139,6 +211,7 @@ func (m *Memory) Delete(ctx context.Context, id GrainId, name string) error { m.mu.Lock() defer m.mu.Unlock() - delete(m.reminders, reminderKey{identity: id, name: name}) + key := reminderKey{identity: id, name: name} + delete(m.reminders, key) return nil } diff --git a/store/schedule_sqlite.go b/store/schedule_sqlite.go index 78249e6..2799b6e 100644 --- a/store/schedule_sqlite.go +++ b/store/schedule_sqlite.go @@ -8,36 +8,49 @@ import ( var _ ReminderStore = (*SQLite)(nil) -// ListDue returns due Reminders in deterministic DueAt, identity, and name +// ListDue returns one bounded page in deterministic DueAt, GrainId, and name // order. -func (s *SQLite) ListDue(ctx context.Context, now time.Time) ([]Reminder, error) { - rows, err := s.readDB.QueryContext(ctx, ` -SELECT entity_type, entity_key, name, method, first_tick_time, due_at, interval, etag +func (s *SQLite) ListDue(ctx context.Context, now time.Time, after *ReminderCursor, limit int) (ReminderPage, error) { + if limit <= 0 { + return ReminderPage{}, errInvalidReminderPageLimit + } + query := ` +SELECT grain_type, grain_key, name, method, first_tick_time, due_at, interval, etag FROM schedule WHERE due_at <= ? -ORDER BY due_at, entity_type, entity_key, name`, timeValue(now)) + ` + arguments := []any{timeValue(now)} + if after != nil { + query += `AND (due_at, grain_type, grain_key, name) > (?, ?, ?, ?) + ` + arguments = append(arguments, timeValue(after.DueAt), after.GrainId.GrainType, after.GrainId.GrainKey, after.Name) + } + query += `ORDER BY due_at, grain_type, grain_key, name +LIMIT ?` + arguments = append(arguments, reminderReadLimit(limit)) + rows, err := s.readDB.QueryContext(ctx, query, arguments...) if err != nil { - return nil, err + return ReminderPage{}, err } defer rows.Close() result := make([]Reminder, 0) for rows.Next() { var ( - entityType string - entityKey string - name string - method string - firstTick int64 - dueAt int64 - interval int64 - etag int64 + grainType string + grainKey string + name string + method string + firstTick int64 + dueAt int64 + interval int64 + etag int64 ) - if err := rows.Scan(&entityType, &entityKey, &name, &method, &firstTick, &dueAt, &interval, &etag); err != nil { - return nil, err + if err := rows.Scan(&grainType, &grainKey, &name, &method, &firstTick, &dueAt, &interval, &etag); err != nil { + return ReminderPage{}, err } result = append(result, Reminder{ - GrainId: GrainId{GrainType: entityType, GrainKey: entityKey}, + GrainId: GrainId{GrainType: grainType, GrainKey: grainKey}, Name: name, Method: method, FirstTickTime: timeFromValue(firstTick), @@ -47,9 +60,9 @@ ORDER BY due_at, entity_type, entity_key, name`, timeValue(now)) }) } if err := rows.Err(); err != nil { - return nil, err + return ReminderPage{}, err } - return result, nil + return reminderPage(result, limit), nil } // Claim atomically checks the Reminder identity, name, and ETag. It returns @@ -57,25 +70,33 @@ ORDER BY due_at, entity_type, entity_key, name`, timeValue(now)) // when nextDueAt is zero. It returns false and nil when the row is absent or // stale. func (s *SQLite) Claim(ctx context.Context, reminder Reminder, nextDueAt time.Time) (bool, error) { - var ( - result sql.Result - err error - ) + tx, err := s.writeDB.BeginTx(ctx, nil) + if err != nil { + return false, err + } + defer tx.Rollback() + + var result sql.Result if nextDueAt.IsZero() { - result, err = s.writeDB.ExecContext(ctx, ` + result, err = tx.ExecContext(ctx, ` DELETE FROM schedule -WHERE entity_type = ? AND entity_key = ? AND name = ? AND etag = ?`, +WHERE grain_type = ? AND grain_key = ? AND name = ? AND etag = ?`, reminder.GrainId.GrainType, reminder.GrainId.GrainKey, reminder.Name, int64(reminder.ETag), ) } else { - result, err = s.writeDB.ExecContext(ctx, ` + nextETag, nextErr := nextSQLiteReminderETag(ctx, tx) + if nextErr != nil { + return false, nextErr + } + result, err = tx.ExecContext(ctx, ` UPDATE schedule -SET due_at = ?, etag = etag + 1 -WHERE entity_type = ? AND entity_key = ? AND name = ? AND etag = ?`, +SET due_at = ?, etag = ? +WHERE grain_type = ? AND grain_key = ? AND name = ? AND etag = ?`, timeValue(nextDueAt), + int64(nextETag), reminder.GrainId.GrainType, reminder.GrainId.GrainKey, reminder.Name, @@ -89,20 +110,35 @@ WHERE entity_type = ? AND entity_key = ? AND name = ? AND etag = ?`, if err != nil { return false, err } - return rows == 1, nil + if rows != 1 { + return false, nil + } + if err := tx.Commit(); err != nil { + return false, err + } + return true, nil } // Put unconditionally inserts or replaces a Reminder and assigns a new ETag. func (s *SQLite) Put(ctx context.Context, reminder Reminder) error { - _, err := s.writeDB.ExecContext(ctx, ` -INSERT INTO schedule (entity_type, entity_key, name, method, first_tick_time, due_at, interval, etag) -VALUES (?, ?, ?, ?, ?, ?, ?, 1) -ON CONFLICT (entity_type, entity_key, name) DO UPDATE SET + tx, err := s.writeDB.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + nextETag, err := nextSQLiteReminderETag(ctx, tx) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, ` +INSERT INTO schedule (grain_type, grain_key, name, method, first_tick_time, due_at, interval, etag) +VALUES (?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT (grain_type, grain_key, name) DO UPDATE SET method = excluded.method, first_tick_time = excluded.first_tick_time, due_at = excluded.due_at, interval = excluded.interval, -etag = schedule.etag + 1`, +etag = excluded.etag`, reminder.GrainId.GrainType, reminder.GrainId.GrainKey, reminder.Name, @@ -110,14 +146,28 @@ etag = schedule.etag + 1`, timeValue(reminder.FirstTickTime), timeValue(reminder.DueAt), int64(reminder.Interval), + int64(nextETag), ) - return err + if err != nil { + return err + } + return tx.Commit() +} + +func nextSQLiteReminderETag(ctx context.Context, tx *sql.Tx) (ETag, error) { + var etag int64 + err := tx.QueryRowContext(ctx, ` +UPDATE schedule_version +SET etag = etag + 1 +WHERE id = 1 +RETURNING etag`).Scan(&etag) + return ETag(etag), err } // Delete unconditionally removes the Reminder identified by id and name. func (s *SQLite) Delete(ctx context.Context, id GrainId, name string) error { _, err := s.writeDB.ExecContext(ctx, ` DELETE FROM schedule -WHERE entity_type = ? AND entity_key = ? AND name = ?`, id.GrainType, id.GrainKey, name) +WHERE grain_type = ? AND grain_key = ? AND name = ?`, id.GrainType, id.GrainKey, name) return err } diff --git a/store/schedule_test.go b/store/schedule_test.go index d92622c..fb9e59a 100644 --- a/store/schedule_test.go +++ b/store/schedule_test.go @@ -2,6 +2,7 @@ package store import ( "context" + "fmt" "path/filepath" "sync" "testing" @@ -16,6 +17,74 @@ func TestSQLiteReminderStore(t *testing.T) { runReminderStoreTests(t, newSQLiteTestStore(t)) } +func TestReminderReadLimitDoesNotOverflow(t *testing.T) { + maximumInt := int(^uint(0) >> 1) + if got := reminderReadLimit(maximumInt); got != maximumInt { + t.Fatalf("read limit at maximum int = %d, want %d", got, maximumInt) + } + if got := reminderReadLimit(7); got != 8 { + t.Fatalf("read limit = %d, want 8", got) + } +} + +func TestMemoryReminderStore_LargeDueSetUsesBoundedPages(t *testing.T) { + const ( + total = 1025 + pageSize = 17 + ) + ctx := context.Background() + backend := NewMemory() + dueAt := time.Unix(10, 0).UTC() + for index := total - 1; index >= 0; index-- { + reminder := Reminder{ + GrainId: GrainId{GrainType: "bulk", GrainKey: fmt.Sprintf("%04d", index)}, + Name: "wake", + Method: "Wake", + DueAt: dueAt, + } + if err := backend.Put(ctx, reminder); err != nil { + t.Fatalf("Put row %d: %v", index, err) + } + } + + var cursor *ReminderCursor + read := 0 + pages := 0 + for { + page, err := backend.ListDue(ctx, dueAt, cursor, pageSize) + if err != nil { + t.Fatalf("ListDue page %d: %v", pages, err) + } + if len(page.Rows) > pageSize { + t.Fatalf("page %d has %d rows, want at most %d", pages, len(page.Rows), pageSize) + } + for _, reminder := range page.Rows { + wantKey := fmt.Sprintf("%04d", read) + if reminder.GrainId.GrainKey != wantKey { + t.Fatalf("row %d key = %q, want %q", read, reminder.GrainId.GrainKey, wantKey) + } + read++ + } + pages++ + if page.Next == nil { + break + } + last := page.Rows[len(page.Rows)-1] + if !page.Next.DueAt.Equal(last.DueAt) || page.Next.GrainId != last.GrainId || page.Next.Name != last.Name { + t.Fatalf("page %d cursor = %#v, want last row %#v", pages-1, page.Next, last) + } + cursor = page.Next + } + + if read != total { + t.Fatalf("read %d rows, want %d", read, total) + } + wantPages := (total + pageSize - 1) / pageSize + if pages != wantPages { + t.Fatalf("read %d pages, want %d", pages, wantPages) + } +} + func TestSQLiteReminderStore_PreservesFirstTickAfterReopen(t *testing.T) { path := filepath.Join(t.TempDir(), "reminders.db") first, err := OpenSQLite(path) @@ -43,15 +112,120 @@ func TestSQLiteReminderStore_PreservesFirstTickAfterReopen(t *testing.T) { t.Fatalf("OpenSQLite second: %v", err) } defer second.Close() - got, err := second.ListDue(context.Background(), row.DueAt) - if err != nil { - t.Fatalf("ListDue after reopen: %v", err) - } + got := listReminderRows(t, second, row.DueAt) if len(got) != 1 || !got[0].FirstTickTime.Equal(row.FirstTickTime) || !got[0].DueAt.Equal(row.DueAt) { t.Fatalf("row after reopen = %#v, want FirstTickTime %s and DueAt %s", got, row.FirstTickTime, row.DueAt) } } +func TestSQLiteReminderStore_DoesNotReuseETagAfterReopen(t *testing.T) { + path := filepath.Join(t.TempDir(), "reminders.db") + now := time.Unix(30, 0).UTC() + setting := Reminder{ + GrainId: GrainId{GrainType: "account", GrainKey: "recreated"}, + Name: "wake", + Method: "Wake", + FirstTickTime: now, + DueAt: now, + Interval: time.Hour, + } + + first, err := OpenSQLite(path) + if err != nil { + t.Fatalf("OpenSQLite first: %v", err) + } + if err := first.Put(context.Background(), setting); err != nil { + first.Close() + t.Fatalf("Put: %v", err) + } + old := findReminder(listReminderRows(t, first, now), setting.GrainId, setting.Name) + if err := first.Delete(context.Background(), setting.GrainId, setting.Name); err != nil { + first.Close() + t.Fatalf("Delete: %v", err) + } + if err := first.Close(); err != nil { + t.Fatalf("Close first: %v", err) + } + + second, err := OpenSQLite(path) + if err != nil { + t.Fatalf("OpenSQLite second: %v", err) + } + defer second.Close() + if err := second.Put(context.Background(), setting); err != nil { + t.Fatalf("Put recreated setting: %v", err) + } + got := findReminder(listReminderRows(t, second, now), setting.GrainId, setting.Name) + if got.ETag <= old.ETag { + t.Fatalf("recreated Reminder = %#v, want ETag newer than %d", got, old.ETag) + } + if won, err := second.Claim(context.Background(), old, now.Add(time.Hour)); err != nil || won { + t.Fatalf("Claim old generation = (%v, %v), want (false, nil)", won, err) + } +} + +func TestSQLiteReminderStore_ConcurrentPutAndTerminalClaimKeepNewSetting(t *testing.T) { + path := filepath.Join(t.TempDir(), "reminders.db") + first, err := OpenSQLite(path) + if err != nil { + t.Fatalf("OpenSQLite first: %v", err) + } + defer first.Close() + second, err := OpenSQLite(path) + if err != nil { + t.Fatalf("OpenSQLite second: %v", err) + } + defer second.Close() + + ctx := context.Background() + now := time.Unix(35, 0).UTC() + for attempt := range 32 { + setting := Reminder{ + GrainId: GrainId{GrainType: "account", GrainKey: fmt.Sprintf("race-%d", attempt)}, + Name: "wake", + Method: "Missing", + DueAt: now, + } + if err := first.Put(ctx, setting); err != nil { + t.Fatalf("attempt %d Put old setting: %v", attempt, err) + } + old := findReminder(listReminderRows(t, first, now), setting.GrainId, setting.Name) + replacement := setting + replacement.Method = "Wake" + + start := make(chan struct{}) + var ( + wait sync.WaitGroup + claimErr error + putErr error + ) + wait.Add(2) + go func() { + defer wait.Done() + <-start + _, claimErr = first.Claim(ctx, old, time.Time{}) + }() + go func() { + defer wait.Done() + <-start + putErr = second.Put(ctx, replacement) + }() + close(start) + wait.Wait() + if claimErr != nil || putErr != nil { + t.Fatalf("attempt %d concurrent results = (Claim %v, Put %v)", attempt, claimErr, putErr) + } + + current := findReminder(listReminderRows(t, first, now), setting.GrainId, setting.Name) + if current.Method != "Wake" || current.ETag <= old.ETag { + t.Fatalf("attempt %d current Reminder = %#v, want Wake with ETag newer than %d", attempt, current, old.ETag) + } + if won, err := first.Claim(ctx, old, time.Time{}); err != nil || won { + t.Fatalf("attempt %d old Claim after race = (%v, %v), want (false, nil)", attempt, won, err) + } + } +} + func runReminderStoreTests(t *testing.T, backend ReminderStore) { t.Helper() t.Run("WriteAndListDue", func(t *testing.T) { @@ -75,10 +249,7 @@ func runReminderStoreTests(t *testing.T, backend ReminderStore) { t.Fatalf("Put future: %v", err) } - got, err := backend.ListDue(ctx, now) - if err != nil { - t.Fatalf("ListDue: %v", err) - } + got := listReminderRows(t, backend, now) if len(got) != 1 { t.Fatalf("ListDue returned %d rows, want 1", len(got)) } @@ -100,19 +271,13 @@ func runReminderStoreTests(t *testing.T, backend ReminderStore) { if won { t.Fatal("Claim with stale ETag won, want false") } - got, err = backend.ListDue(ctx, now) - if err != nil { - t.Fatalf("ListDue after replacement: %v", err) - } + got = listReminderRows(t, backend, now) if len(got) != 0 { t.Fatalf("ListDue after replacement returned %d rows, want 0", len(got)) } - got, err = backend.ListDue(ctx, now.Add(time.Hour)) - if err != nil { - t.Fatalf("ListDue at replacement: %v", err) - } - if len(got) != 1 || got[0].Method != replacement.Method || !got[0].FirstTickTime.Equal(replacement.FirstTickTime) || got[0].ETag != 2 { - t.Fatalf("replacement rows = %#v, want method %q and ETag 2", got, replacement.Method) + got = listReminderRows(t, backend, now.Add(time.Hour)) + if len(got) != 1 || got[0].Method != replacement.Method || !got[0].FirstTickTime.Equal(replacement.FirstTickTime) || got[0].ETag <= due.ETag { + t.Fatalf("replacement rows = %#v, want method %q and ETag newer than %d", got, replacement.Method, due.ETag) } }) @@ -129,10 +294,7 @@ func runReminderStoreTests(t *testing.T, backend ReminderStore) { if err := backend.Put(ctx, task); err != nil { t.Fatalf("Put: %v", err) } - due, err := backend.ListDue(ctx, now) - if err != nil { - t.Fatalf("ListDue: %v", err) - } + due := listReminderRows(t, backend, now) if len(due) != 1 { t.Fatalf("ListDue returned %d rows, want 1", len(due)) } @@ -172,10 +334,7 @@ func runReminderStoreTests(t *testing.T, backend ReminderStore) { t.Fatalf("Claim winners = %d, want 1", wins) } - got, err := backend.ListDue(ctx, nextDueAt) - if err != nil { - t.Fatalf("ListDue after Claim: %v", err) - } + got := listReminderRows(t, backend, nextDueAt) var claimed Reminder found := false for _, candidate := range got { @@ -185,8 +344,47 @@ func runReminderStoreTests(t *testing.T, backend ReminderStore) { break } } - if !found || claimed.ETag != task.ETag+1 || !claimed.FirstTickTime.Equal(task.FirstTickTime) || !claimed.DueAt.Equal(nextDueAt) { - t.Fatalf("claimed row = %#v, want %s/%s at next due time and ETag %d", got, task.GrainId.GrainType, task.Name, task.ETag+1) + if !found || claimed.ETag <= task.ETag || !claimed.FirstTickTime.Equal(task.FirstTickTime) || !claimed.DueAt.Equal(nextDueAt) { + t.Fatalf("claimed row = %#v, want %s/%s at next due time and ETag newer than %d", got, task.GrainId.GrainType, task.Name, task.ETag) + } + }) + + t.Run("DeleteThenPutDoesNotReuseETag", func(t *testing.T) { + ctx := context.Background() + now := time.Unix(275, 0).UTC() + setting := Reminder{ + GrainId: GrainId{GrainType: "account", GrainKey: "recreated"}, + Name: "wake", + Method: "Wake", + FirstTickTime: now, + DueAt: now, + Interval: time.Hour, + } + if err := backend.Put(ctx, setting); err != nil { + t.Fatalf("Put old setting: %v", err) + } + old := findReminder(listReminderRows(t, backend, now), setting.GrainId, setting.Name) + if old.ETag == 0 { + t.Fatal("old ETag = 0, want a Store-assigned value") + } + if err := backend.Delete(ctx, setting.GrainId, setting.Name); err != nil { + t.Fatalf("Delete old setting: %v", err) + } + if won, err := backend.Claim(ctx, old, now.Add(time.Hour)); err != nil || won { + t.Fatalf("Claim after Delete = (%v, %v), want (false, nil)", won, err) + } + if err := backend.Put(ctx, setting); err != nil { + t.Fatalf("Put recreated setting: %v", err) + } + recreated := findReminder(listReminderRows(t, backend, now), setting.GrainId, setting.Name) + if recreated.ETag <= old.ETag { + t.Fatalf("recreated ETag = %d, want newer than %d", recreated.ETag, old.ETag) + } + if won, err := backend.Claim(ctx, old, now.Add(time.Hour)); err != nil || won { + t.Fatalf("Claim old generation = (%v, %v), want (false, nil)", won, err) + } + if err := backend.Delete(ctx, setting.GrainId, setting.Name); err != nil { + t.Fatalf("Delete recreated setting: %v", err) } }) @@ -202,10 +400,7 @@ func runReminderStoreTests(t *testing.T, backend ReminderStore) { if err := backend.Put(ctx, task); err != nil { t.Fatalf("Put: %v", err) } - due, err := backend.ListDue(ctx, now) - if err != nil { - t.Fatalf("ListDue: %v", err) - } + due := listReminderRows(t, backend, now) won, err := backend.Claim(ctx, due[0], time.Time{}) if err != nil { t.Fatalf("Claim: %v", err) @@ -213,10 +408,7 @@ func runReminderStoreTests(t *testing.T, backend ReminderStore) { if !won { t.Fatal("Claim won = false, want true") } - remaining, err := backend.ListDue(ctx, now) - if err != nil { - t.Fatalf("ListDue after one-shot Claim: %v", err) - } + remaining := listReminderRows(t, backend, now) if len(remaining) != 0 { t.Fatalf("remaining rows = %#v, want none", remaining) } @@ -237,12 +429,126 @@ func runReminderStoreTests(t *testing.T, backend ReminderStore) { if err := backend.Delete(ctx, task.GrainId, task.Name); err != nil { t.Fatalf("Delete: %v", err) } - remaining, err := backend.ListDue(ctx, now) - if err != nil { - t.Fatalf("ListDue after Delete: %v", err) - } + remaining := listReminderRows(t, backend, now) if len(remaining) != 0 { t.Fatalf("remaining rows = %#v, want none", remaining) } }) + + t.Run("ListDueUsesBoundedCursorPages", func(t *testing.T) { + ctx := context.Background() + dueAt := time.Unix(40, 0).UTC() + now := time.Unix(50, 0).UTC() + want := []Reminder{ + {GrainId: GrainId{GrainType: "page-a", GrainKey: "a"}, Name: "one", Method: "Wake", DueAt: dueAt}, + {GrainId: GrainId{GrainType: "page-a", GrainKey: "a"}, Name: "two", Method: "Wake", DueAt: dueAt}, + {GrainId: GrainId{GrainType: "page-a", GrainKey: "b"}, Name: "one", Method: "Wake", DueAt: dueAt}, + {GrainId: GrainId{GrainType: "page-b", GrainKey: "a"}, Name: "one", Method: "Wake", DueAt: dueAt}, + {GrainId: GrainId{GrainType: "page-b", GrainKey: "a"}, Name: "later", Method: "Wake", DueAt: dueAt.Add(time.Second)}, + } + for _, reminder := range want { + if err := backend.Put(ctx, reminder); err != nil { + t.Fatalf("Put %#v: %v", reminder, err) + } + } + + var ( + cursor *ReminderCursor + got []Reminder + ) + for pageNumber := 0; ; pageNumber++ { + page, err := backend.ListDue(ctx, now, cursor, 2) + if err != nil { + t.Fatalf("ListDue page %d: %v", pageNumber, err) + } + if len(page.Rows) > 2 { + t.Fatalf("page %d has %d rows, want at most 2", pageNumber, len(page.Rows)) + } + got = append(got, page.Rows...) + if page.Next == nil { + break + } + cursor = page.Next + } + if len(got) != len(want) { + t.Fatalf("paged rows = %#v, want %d rows", got, len(want)) + } + for index := range want { + if got[index].GrainId != want[index].GrainId || got[index].Name != want[index].Name || !got[index].DueAt.Equal(want[index].DueAt) { + t.Fatalf("row %d = %#v, want %#v", index, got[index], want[index]) + } + } + + for _, limit := range []int{0, -1} { + if _, err := backend.ListDue(ctx, now, nil, limit); err == nil { + t.Fatalf("ListDue accepted limit %d", limit) + } + } + }) + + t.Run("ListDueCursorIsStableAcrossChanges", func(t *testing.T) { + ctx := context.Background() + dueAt := time.Unix(20, 0).UTC() + now := time.Unix(30, 0).UTC() + rows := []Reminder{ + {GrainId: GrainId{GrainType: "mutation", GrainKey: "b"}, Name: "wake", Method: "Wake", DueAt: dueAt}, + {GrainId: GrainId{GrainType: "mutation", GrainKey: "c"}, Name: "wake", Method: "Wake", DueAt: dueAt}, + {GrainId: GrainId{GrainType: "mutation", GrainKey: "d"}, Name: "wake", Method: "Wake", DueAt: dueAt}, + {GrainId: GrainId{GrainType: "mutation", GrainKey: "e"}, Name: "wake", Method: "Wake", DueAt: dueAt}, + } + for _, reminder := range rows { + if err := backend.Put(ctx, reminder); err != nil { + t.Fatalf("Put %#v: %v", reminder, err) + } + } + + first, err := backend.ListDue(ctx, now, nil, 2) + if err != nil { + t.Fatalf("ListDue first page: %v", err) + } + if len(first.Rows) != 2 || first.Rows[0].GrainId.GrainKey != "b" || first.Rows[1].GrainId.GrainKey != "c" || first.Next == nil { + t.Fatalf("first page = %#v, want b, c, and a cursor", first) + } + + before := Reminder{GrainId: GrainId{GrainType: "mutation", GrainKey: "a"}, Name: "wake", Method: "Wake", DueAt: dueAt} + after := Reminder{GrainId: GrainId{GrainType: "mutation", GrainKey: "f"}, Name: "wake", Method: "Wake", DueAt: dueAt} + if err := backend.Put(ctx, before); err != nil { + t.Fatalf("Put before cursor: %v", err) + } + if err := backend.Put(ctx, after); err != nil { + t.Fatalf("Put after cursor: %v", err) + } + if err := backend.Delete(ctx, rows[2].GrainId, rows[2].Name); err != nil { + t.Fatalf("Delete after cursor: %v", err) + } + + second, err := backend.ListDue(ctx, now, first.Next, 2) + if err != nil { + t.Fatalf("ListDue second page: %v", err) + } + if len(second.Rows) != 2 || second.Rows[0].GrainId.GrainKey != "e" || second.Rows[1].GrainId.GrainKey != "f" || second.Next != nil { + t.Fatalf("second page = %#v, want e, f, and no cursor", second) + } + }) +} + +func findReminder(rows []Reminder, id GrainId, name string) Reminder { + for _, reminder := range rows { + if reminder.GrainId == id && reminder.Name == name { + return reminder + } + } + return Reminder{} +} + +func listReminderRows(t *testing.T, backend ReminderStore, now time.Time) []Reminder { + t.Helper() + page, err := backend.ListDue(context.Background(), now, nil, 1024) + if err != nil { + t.Fatalf("ListDue: %v", err) + } + if page.Next != nil { + t.Fatal("ListDue test helper limit is too small") + } + return page.Rows } diff --git a/store/sqlite.go b/store/sqlite.go index f2db86b..bb7699e 100644 --- a/store/sqlite.go +++ b/store/sqlite.go @@ -7,6 +7,7 @@ import ( "fmt" "net/url" "path/filepath" + "slices" "strings" "github.com/suraciii/gor/clock" @@ -15,6 +16,10 @@ import ( const sqliteBusyTimeout = 5000 +func reminderDueIndexColumns() []string { + return []string{"due_at", "grain_type", "grain_key", "name"} +} + // Durability is the guarantee a confirmed state write keeps across a hard // crash (power loss, operating-system crash, hard reset). A clean restart // loses nothing at either tier. @@ -41,7 +46,7 @@ type options struct { durability Durability } -// WithDurability sets the durability tier for entity-state writes. The +// WithDurability sets the durability tier for State writes. The // default is DurabilityFull; DurabilityRelaxed must be chosen explicitly. func WithDurability(d Durability) Option { return func(o *options) { @@ -52,7 +57,7 @@ func WithDurability(d Durability) Option { // SQLite is a SQLite-backed implementation of Store, MemberStore, and // ReminderStore. // -// Entity state lives in a database file derived from the named path by +// Grain State lives in a database file derived from the named path by // inserting "-state" before the file extension; the schedule and membership // tables live in the named file. Backups must cover both files and their // -wal sidecars. @@ -106,11 +111,7 @@ func openSQLite(path string, memberClock clock.Clock, opts ...Option) (*SQLite, writeDB.Close() return nil, fmt.Errorf("open sqlite database %q: %w", path, err) } - if err := createCoordinationSchema(writeDB); err != nil { - writeDB.Close() - return nil, err - } - if err := migrateReminderSchema(writeDB); err != nil { + if err := initializeCoordinationSchema(writeDB); err != nil { writeDB.Close() return nil, err } @@ -120,6 +121,7 @@ func openSQLite(path string, memberClock clock.Clock, opts ...Option) (*SQLite, writeDB.Close() return nil, err } + readDB.SetMaxOpenConns(16) readDB.SetMaxIdleConns(16) if err := readDB.Ping(); err != nil { readDB.Close() @@ -155,6 +157,7 @@ func openSQLite(path string, memberClock clock.Clock, opts ...Option) (*SQLite, writeDB.Close() return nil, err } + stateReadDB.SetMaxOpenConns(16) stateReadDB.SetMaxIdleConns(16) if err := stateReadDB.Ping(); err != nil { stateReadDB.Close() @@ -164,7 +167,7 @@ func openSQLite(path string, memberClock clock.Clock, opts ...Option) (*SQLite, return nil, fmt.Errorf("open sqlite database %q: %w", statePath, err) } - if err := migrateOldLayout(writeDB, statePath); err != nil { + if err := migrateOldLayout(writeDB, stateWriteDB, statePath); err != nil { stateReadDB.Close() stateWriteDB.Close() readDB.Close() @@ -182,6 +185,252 @@ func openSQLite(path string, memberClock clock.Clock, opts ...Option) (*SQLite, }, nil } +// Check verifies the SQLite connections, database integrity, and the State and +// Reminder tables. It does not change stored data. +func (s *SQLite) Check(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + + connections := []struct { + name string + db *sql.DB + }{ + {name: "coordination read", db: s.readDB}, + {name: "coordination write", db: s.writeDB}, + {name: "state read", db: s.stateReadDB}, + {name: "state write", db: s.stateWriteDB}, + } + for _, connection := range connections { + if connection.db == nil { + return fmt.Errorf("store: %s database is nil", connection.name) + } + if err := connection.db.PingContext(ctx); err != nil { + return fmt.Errorf("store: check %s database: %w", connection.name, err) + } + } + for _, database := range []struct { + name string + db *sql.DB + }{ + {name: "coordination", db: s.readDB}, + {name: "State", db: s.stateReadDB}, + } { + if err := checkSQLiteIntegrity(ctx, database.db); err != nil { + return fmt.Errorf("store: check %s database integrity: %w", database.name, err) + } + } + if err := checkSQLiteTable(ctx, s.stateReadDB, "records", []string{ + "identity_type", + "identity_key", + "data", + "etag", + }, []string{"identity_type", "identity_key"}); err != nil { + return fmt.Errorf("store: check State schema: %w", err) + } + if err := checkSQLiteTable(ctx, s.readDB, "schedule", []string{ + "grain_type", + "grain_key", + "name", + "method", + "first_tick_time", + "due_at", + "interval", + "etag", + }, []string{"grain_type", "grain_key", "name"}); err != nil { + return fmt.Errorf("store: check Reminder schema: %w", err) + } + if err := checkSQLiteTable(ctx, s.readDB, "schedule_version", []string{ + "id", + "etag", + }, []string{"id"}); err != nil { + return fmt.Errorf("store: check Reminder version schema: %w", err) + } + if err := checkReminderVersion(ctx, s.readDB); err != nil { + return fmt.Errorf("store: check Reminder version: %w", err) + } + if err := checkSQLiteIndex(ctx, s.readDB, "schedule", "schedule_due_idx", reminderDueIndexColumns()); err != nil { + return fmt.Errorf("store: check Reminder due index: %w", err) + } + return ctx.Err() +} + +func checkReminderVersion(ctx context.Context, db *sql.DB) error { + var valid int + err := db.QueryRowContext(ctx, ` +SELECT COUNT(*) = 1 + AND MIN(id) = 1 + AND MAX(id) = 1 + AND MIN(etag) >= COALESCE((SELECT MAX(etag) FROM schedule), 0) +FROM schedule_version`).Scan(&valid) + if err != nil { + return err + } + if valid == 0 { + return errors.New("version is older than a Reminder ETag") + } + return nil +} + +func checkSQLiteIndex(ctx context.Context, db *sql.DB, table, name string, columns []string) error { + definition, err := sqliteIndex(ctx, db, table, name) + if err != nil { + return err + } + if !definition.found || definition.unique || definition.partial || definition.origin != "c" || !slices.Equal(definition.columns, columns) { + return fmt.Errorf("index %q has columns %v, unique %t, partial %t, and origin %q", name, definition.columns, definition.unique, definition.partial, definition.origin) + } + return nil +} + +type sqliteQueryer interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +} + +type sqliteIndexDefinition struct { + columns []string + origin string + found bool + unique bool + partial bool +} + +func sqliteIndex(ctx context.Context, db sqliteQueryer, table, name string) (sqliteIndexDefinition, error) { + rows, err := db.QueryContext(ctx, "PRAGMA index_list("+table+")") + if err != nil { + return sqliteIndexDefinition{}, err + } + definition := sqliteIndexDefinition{} + for rows.Next() { + var ( + sequence int + indexName string + unique int + origin string + partial int + ) + if err := rows.Scan(&sequence, &indexName, &unique, &origin, &partial); err != nil { + rows.Close() + return sqliteIndexDefinition{}, err + } + if indexName == name { + definition = sqliteIndexDefinition{ + origin: origin, + found: true, + unique: unique != 0, + partial: partial != 0, + } + } + } + if err := rows.Err(); err != nil { + rows.Close() + return sqliteIndexDefinition{}, err + } + if err := rows.Close(); err != nil { + return sqliteIndexDefinition{}, err + } + if !definition.found { + return definition, nil + } + + rows, err = db.QueryContext(ctx, "PRAGMA index_info("+name+")") + if err != nil { + return sqliteIndexDefinition{}, err + } + defer rows.Close() + + for rows.Next() { + var ( + sequence int + columnID int + column string + ) + if err := rows.Scan(&sequence, &columnID, &column); err != nil { + return sqliteIndexDefinition{}, err + } + definition.columns = append(definition.columns, column) + } + return definition, rows.Err() +} + +func checkSQLiteIntegrity(ctx context.Context, db *sql.DB) error { + rows, err := db.QueryContext(ctx, "PRAGMA integrity_check") + if err != nil { + return err + } + defer rows.Close() + + checked := false + for rows.Next() { + var result string + if err := rows.Scan(&result); err != nil { + return err + } + checked = true + if result != "ok" { + return errors.New(result) + } + } + if err := rows.Err(); err != nil { + return err + } + if !checked { + return errors.New("integrity check returned no result") + } + return nil +} + +func checkSQLiteTable(ctx context.Context, db *sql.DB, table string, requiredColumns, requiredPrimaryKey []string) error { + if err := ctx.Err(); err != nil { + return err + } + rows, err := db.QueryContext(ctx, "PRAGMA table_info("+table+")") + if err != nil { + return err + } + defer rows.Close() + + columns := make(map[string]struct{}, len(requiredColumns)) + primaryKeyColumns := make(map[int]string, len(requiredPrimaryKey)) + for rows.Next() { + var ( + cid int + name string + columnType string + notNull int + defaultVal any + primaryKey int + ) + if err := rows.Scan(&cid, &name, &columnType, ¬Null, &defaultVal, &primaryKey); err != nil { + return err + } + columns[name] = struct{}{} + if primaryKey > 0 { + primaryKeyColumns[primaryKey] = name + } + if err := ctx.Err(); err != nil { + return err + } + } + if err := rows.Err(); err != nil { + return err + } + for _, column := range requiredColumns { + if _, ok := columns[column]; !ok { + return fmt.Errorf("table %q is missing column %q", table, column) + } + } + if len(primaryKeyColumns) != len(requiredPrimaryKey) { + return fmt.Errorf("table %q has an invalid primary key", table) + } + for index, column := range requiredPrimaryKey { + if primaryKeyColumns[index+1] != column { + return fmt.Errorf("table %q has an invalid primary key", table) + } + } + return nil +} + // Read returns a copy of the stored record for id, or a zero Record and nil // when id has not been written. func (s *SQLite) Read(ctx context.Context, id GrainId) (Record, error) { @@ -207,6 +456,12 @@ func (s *SQLite) Read(ctx context.Context, id GrainId) (Record, error) { // It returns the incremented ETag, or an error matching ErrConflict when the // comparison fails. func (s *SQLite) Write(ctx context.Context, id GrainId, data []byte, expect ETag) (ETag, error) { + if err := ctx.Err(); err != nil { + return 0, err + } + if len(data) == 0 { + return 0, ErrInvalidRecordData + } var ( result sql.Result err error @@ -280,20 +535,44 @@ func stateFilePath(path string) string { return filepath.Join(dir, strings.TrimSuffix(base, ext)+"-state"+ext) } -func createCoordinationSchema(db *sql.DB) error { - _, err := db.Exec(` +func initializeCoordinationSchema(db *sql.DB) error { + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if err := createCoordinationSchema(tx); err != nil { + return err + } + if err := migrateReminderSchema(tx); err != nil { + return err + } + return tx.Commit() +} + +func createCoordinationSchema(tx *sql.Tx) error { + _, err := tx.Exec(` CREATE TABLE IF NOT EXISTS schedule ( - entity_type TEXT NOT NULL, - entity_key TEXT NOT NULL, + grain_type TEXT NOT NULL, + grain_key TEXT NOT NULL, name TEXT NOT NULL, method TEXT NOT NULL, first_tick_time INTEGER NOT NULL, due_at INTEGER NOT NULL, interval INTEGER NOT NULL, etag INTEGER NOT NULL, - PRIMARY KEY (entity_type, entity_key, name) + PRIMARY KEY (grain_type, grain_key, name) ); +CREATE TABLE IF NOT EXISTS schedule_version ( + id INTEGER NOT NULL, + etag INTEGER NOT NULL, + PRIMARY KEY (id), + CHECK (id = 1) +); + +INSERT OR IGNORE INTO schedule_version (id, etag) VALUES (1, 0); + CREATE TABLE IF NOT EXISTS member ( node_addr TEXT NOT NULL, generation TEXT NOT NULL, @@ -306,14 +585,14 @@ CREATE TABLE IF NOT EXISTS member ( return err } -func migrateReminderSchema(db *sql.DB) error { - rows, err := db.Query(`PRAGMA table_info(schedule)`) +func migrateReminderSchema(tx *sql.Tx) error { + rows, err := tx.Query(`PRAGMA table_info(schedule)`) if err != nil { return err } defer rows.Close() - found := false + columns := make(map[string]bool) for rows.Next() { var ( cid int @@ -326,20 +605,58 @@ func migrateReminderSchema(db *sql.DB) error { if err := rows.Scan(&cid, &name, &columnType, ¬Null, &defaultVal, &primaryKey); err != nil { return err } - if name == "first_tick_time" { - found = true - } + columns[name] = true } if err := rows.Err(); err != nil { + rows.Close() return err } - if !found { - if _, err := db.Exec(`ALTER TABLE schedule ADD COLUMN first_tick_time INTEGER`); err != nil { + if err := rows.Close(); err != nil { + return err + } + legacyIdentity := columns["entity_type"] && columns["entity_key"] + grainIdentity := columns["grain_type"] && columns["grain_key"] + legacyIdentityColumn := columns["entity_type"] || columns["entity_key"] + grainIdentityColumn := columns["grain_type"] || columns["grain_key"] + switch { + case legacyIdentity && !grainIdentityColumn: + if _, err := tx.Exec(` + ALTER TABLE schedule RENAME COLUMN entity_type TO grain_type; + ALTER TABLE schedule RENAME COLUMN entity_key TO grain_key`); err != nil { + return fmt.Errorf("migrate Reminder identity columns: %w", err) + } + case grainIdentity && !legacyIdentityColumn: + default: + return fmt.Errorf("unsupported Reminder identity columns: entity_type=%t entity_key=%t grain_type=%t grain_key=%t", + columns["entity_type"], columns["entity_key"], columns["grain_type"], columns["grain_key"]) + } + if !columns["first_tick_time"] { + if _, err := tx.Exec(`ALTER TABLE schedule ADD COLUMN first_tick_time INTEGER`); err != nil { return err } } - _, err = db.Exec(`UPDATE schedule SET first_tick_time = due_at WHERE first_tick_time IS NULL`) - return err + if _, err := tx.Exec(`UPDATE schedule SET first_tick_time = due_at WHERE first_tick_time IS NULL`); err != nil { + return err + } + if _, err := tx.Exec(` + UPDATE schedule_version + SET etag = MAX(etag, COALESCE((SELECT MAX(etag) FROM schedule), 0)) + WHERE id = 1`); err != nil { + return err + } + definition, err := sqliteIndex(context.Background(), tx, "schedule", "schedule_due_idx") + if err != nil { + return err + } + if !definition.found || definition.unique || definition.partial || definition.origin != "c" || !slices.Equal(definition.columns, reminderDueIndexColumns()) { + if _, err := tx.Exec(` + DROP INDEX IF EXISTS schedule_due_idx; + CREATE INDEX schedule_due_idx + ON schedule (due_at, grain_type, grain_key, name)`); err != nil { + return err + } + } + return nil } func createStateSchema(db *sql.DB) error { @@ -358,14 +675,20 @@ CREATE TABLE IF NOT EXISTS records ( // still keeps the state, schedule, and membership tables together. The copy // commits before the old rows are dropped, so an interruption between the two // leaves the old database intact and the next open redoes the copy. -func migrateOldLayout(writeDB *sql.DB, statePath string) error { +func migrateOldLayout(writeDB *sql.DB, stateWriteDB *sql.DB, statePath string) error { hasRecords, err := tableExists(writeDB, "records") if err != nil || !hasRecords { return err } + if err := checkSQLiteIntegrity(context.Background(), writeDB); err != nil { + return fmt.Errorf("check old database integrity: %w", err) + } if err := copyStateRows(writeDB, statePath); err != nil { return err } + if err := checkSQLiteIntegrity(context.Background(), stateWriteDB); err != nil { + return fmt.Errorf("check copied State database integrity: %w", err) + } _, err = writeDB.Exec(`DROP TABLE records`) return err } @@ -395,8 +718,7 @@ func copyStateRows(writeDB *sql.DB, statePath string) error { } defer tx.Rollback() - // ATTACH takes no bound parameters; the URI form percent-encodes the path. - if _, err := tx.Exec(`ATTACH DATABASE '` + sqliteFileURI(statePath) + `' AS state`); err != nil { + if _, err := tx.Exec(`ATTACH DATABASE ? AS state`, sqliteFileURI(statePath)); err != nil { return fmt.Errorf("attach state database: %w", err) } if _, err := tx.Exec(`INSERT OR REPLACE INTO state.records (identity_type, identity_key, data, etag) @@ -424,5 +746,9 @@ SELECT identity_type, identity_key, data, etag FROM records`); err != nil { func sqliteFileURI(path string) string { abs, _ := filepath.Abs(path) - return (&url.URL{Scheme: "file", Path: filepath.ToSlash(abs)}).String() + slashPath := filepath.ToSlash(abs) + if filepath.VolumeName(abs) != "" && !strings.HasPrefix(slashPath, "/") { + slashPath = "/" + slashPath + } + return (&url.URL{Scheme: "file", Path: slashPath}).String() } diff --git a/store/sqlite_recovery_test.go b/store/sqlite_recovery_test.go new file mode 100644 index 0000000..3e38148 --- /dev/null +++ b/store/sqlite_recovery_test.go @@ -0,0 +1,194 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + moderncsqlite "modernc.org/sqlite" +) + +func TestOpenSQLite_ConfiguresBoundedConnectionPools(t *testing.T) { + s := newSQLiteTestStore(t) + assertSQLitePoolLimit(t, "coordination read", s.readDB, 16) + assertSQLitePoolLimit(t, "coordination write", s.writeDB, 1) + assertSQLitePoolLimit(t, "State read", s.stateReadDB, 16) + assertSQLitePoolLimit(t, "State write", s.stateWriteDB, 1) +} + +func assertSQLitePoolLimit(t *testing.T, name string, db *sql.DB, limit int) { + t.Helper() + if got := db.Stats().MaxOpenConnections; got != limit { + t.Fatalf("%s MaxOpenConnections = %d, want %d", name, got, limit) + } + + connections := make([]*sql.Conn, 0, limit) + for range limit { + connection, err := db.Conn(context.Background()) + if err != nil { + for _, open := range connections { + open.Close() + } + t.Fatalf("%s acquire connection: %v", name, err) + } + connections = append(connections, connection) + } + for _, connection := range connections { + if err := connection.Close(); err != nil { + t.Fatalf("%s release connection: %v", name, err) + } + } + if got := db.Stats().Idle; got != limit { + t.Fatalf("%s idle connections = %d, want %d", name, got, limit) + } +} + +func TestSQLiteCheckRejectsIntegrityFailure(t *testing.T) { + for _, test := range []struct { + name string + db func(*SQLite) *sql.DB + }{ + {name: "coordination", db: func(s *SQLite) *sql.DB { return s.writeDB }}, + {name: "State", db: func(s *SQLite) *sql.DB { return s.stateWriteDB }}, + } { + t.Run(test.name, func(t *testing.T) { + s := newSQLiteTestStore(t) + db := test.db(s) + if _, err := db.Exec(`CREATE TABLE integrity_probe(value INTEGER CHECK(value > 0))`); err != nil { + t.Fatalf("create integrity probe: %v", err) + } + if _, err := db.Exec(`PRAGMA ignore_check_constraints = ON`); err != nil { + t.Fatalf("disable check constraints: %v", err) + } + if _, err := db.Exec(`INSERT INTO integrity_probe VALUES (-1)`); err != nil { + t.Fatalf("insert invalid integrity probe: %v", err) + } + if _, err := db.Exec(`PRAGMA ignore_check_constraints = OFF`); err != nil { + t.Fatalf("enable check constraints: %v", err) + } + + err := s.Check(context.Background()) + if err == nil { + t.Fatal("Check error = nil, want integrity failure") + } + for _, part := range []string{test.name, "integrity", "CHECK constraint failed"} { + if !strings.Contains(err.Error(), part) { + t.Fatalf("Check error %q does not contain %q", err, part) + } + } + }) + } +} + +func TestSQLiteColdBackupRestoreIncludesStateReminderAndWAL(t *testing.T) { + sourcePath := filepath.Join(t.TempDir(), "source", "gor.db") + backupPath := filepath.Join(t.TempDir(), "backup", "gor.db") + if err := os.MkdirAll(filepath.Dir(sourcePath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(backupPath), 0o755); err != nil { + t.Fatal(err) + } + + source, err := OpenSQLite(sourcePath) + if err != nil { + t.Fatalf("OpenSQLite source: %v", err) + } + id := GrainId{GrainType: "account", GrainKey: "alice"} + if etag, err := source.Write(context.Background(), id, []byte(`{"balance":7}`), 0); err != nil || etag != 1 { + t.Fatalf("write State = (%d, %v), want (1, nil)", etag, err) + } + firstTick := time.Unix(100, 0).UTC() + dueAt := firstTick.Add(time.Minute) + wantReminder := Reminder{ + GrainId: id, + Name: "daily", + Method: "Apply", + FirstTickTime: firstTick, + DueAt: dueAt, + Interval: time.Hour, + } + if err := source.Put(context.Background(), wantReminder); err != nil { + t.Fatalf("put Reminder: %v", err) + } + + keepSQLiteWALAfterClose(t, source.writeDB) + keepSQLiteWALAfterClose(t, source.stateWriteDB) + if err := source.Close(); err != nil { + t.Fatalf("close source: %v", err) + } + if size := walSize(t, sourcePath); size == 0 { + t.Fatal("coordination WAL is empty before backup") + } + if size := walSize(t, stateFilePath(sourcePath)); size == 0 { + t.Fatal("State WAL is empty before backup") + } + + copySQLiteBackupFile(t, sourcePath, backupPath) + copySQLiteBackupFile(t, sourcePath+"-wal", backupPath+"-wal") + copySQLiteBackupFile(t, stateFilePath(sourcePath), stateFilePath(backupPath)) + copySQLiteBackupFile(t, stateFilePath(sourcePath)+"-wal", stateFilePath(backupPath)+"-wal") + + restored, err := OpenSQLite(backupPath) + if err != nil { + t.Fatalf("OpenSQLite restored backup: %v", err) + } + defer restored.Close() + if err := restored.Check(context.Background()); err != nil { + t.Fatalf("Check restored backup: %v", err) + } + record, err := restored.Read(context.Background(), id) + if err != nil { + t.Fatalf("read restored State: %v", err) + } + if string(record.Data) != `{"balance":7}` || record.ETag != 1 { + t.Fatalf("restored State = %#v, want balance 7 and ETag 1", record) + } + reminders := listReminderRows(t, restored, dueAt) + if len(reminders) != 1 { + t.Fatalf("restored Reminders = %#v, want one", reminders) + } + gotReminder := reminders[0] + wantReminder.ETag = 1 + if gotReminder != wantReminder { + t.Fatalf("restored Reminder = %#v, want %#v", gotReminder, wantReminder) + } +} + +func keepSQLiteWALAfterClose(t *testing.T, db *sql.DB) { + t.Helper() + connection, err := db.Conn(context.Background()) + if err != nil { + t.Fatalf("get SQLite connection: %v", err) + } + if err := connection.Raw(func(driverConnection any) error { + control, ok := driverConnection.(moderncsqlite.FileControl) + if !ok { + return errors.New("SQLite driver does not support file control") + } + _, err := control.FileControlPersistWAL("main", 1) + return err + }); err != nil { + connection.Close() + t.Fatalf("keep SQLite WAL after close: %v", err) + } + if err := connection.Close(); err != nil { + t.Fatalf("release SQLite connection: %v", err) + } +} + +func copySQLiteBackupFile(t *testing.T, source string, target string) { + t.Helper() + data, err := os.ReadFile(source) + if err != nil { + t.Fatalf("read backup source %q: %v", source, err) + } + if err := os.WriteFile(target, data, 0o600); err != nil { + t.Fatalf("write backup target %q: %v", target, err) + } +} diff --git a/store/sqlite_test.go b/store/sqlite_test.go index 7ee0162..e8f936c 100644 --- a/store/sqlite_test.go +++ b/store/sqlite_test.go @@ -2,155 +2,14 @@ package store import ( "context" - "errors" "os" "path/filepath" + "runtime" + "strconv" "strings" "testing" ) -func TestSQLiteStore_WriteWithMatchingETagReturnsNewETag(t *testing.T) { - store := newSQLiteTestStore(t) - id := GrainId{GrainType: "account", GrainKey: "alice"} - - etag, err := store.Write(context.Background(), id, []byte("first"), 0) - if err != nil { - t.Fatalf("Write: %v", err) - } - if etag != 1 { - t.Fatalf("ETag = %d, want 1", etag) - } - - record, err := store.Read(context.Background(), id) - if err != nil { - t.Fatalf("Read: %v", err) - } - if string(record.Data) != "first" || record.ETag != etag { - t.Fatalf("Record = %#v, want data first and ETag %d", record, etag) - } - - nextETag, err := store.Write(context.Background(), id, []byte("second"), etag) - if err != nil { - t.Fatalf("second Write: %v", err) - } - if nextETag != 2 { - t.Fatalf("second ETag = %d, want 2", nextETag) - } -} - -func TestSQLiteStore_ConflictLeavesRecordUnchanged(t *testing.T) { - store := newSQLiteTestStore(t) - id := GrainId{GrainType: "account", GrainKey: "alice"} - - etag, err := store.Write(context.Background(), id, []byte("original"), 0) - if err != nil { - t.Fatalf("seed Write: %v", err) - } - - newETag, err := store.Write(context.Background(), id, []byte("replacement"), etag+1) - if !errors.Is(err, ErrConflict) { - t.Fatalf("Write error = %v, want ErrConflict", err) - } - if newETag != 0 { - t.Fatalf("conflicting ETag = %d, want 0", newETag) - } - - record, err := store.Read(context.Background(), id) - if err != nil { - t.Fatalf("Read: %v", err) - } - if string(record.Data) != "original" || record.ETag != etag { - t.Fatalf("Record after conflict = %#v, want original data and ETag %d", record, etag) - } -} - -func TestSQLiteStore_ZeroETagConflictsWithExistingRecord(t *testing.T) { - store := newSQLiteTestStore(t) - id := GrainId{GrainType: "account", GrainKey: "alice"} - - if _, err := store.Write(context.Background(), id, []byte("existing"), 0); err != nil { - t.Fatalf("seed Write: %v", err) - } - - if _, err := store.Write(context.Background(), id, []byte("overwrite"), 0); !errors.Is(err, ErrConflict) { - t.Fatalf("Write error = %v, want ErrConflict", err) - } -} - -func TestSQLiteStore_NonzeroETagConflictsWithMissingRecord(t *testing.T) { - store := newSQLiteTestStore(t) - id := GrainId{GrainType: "account", GrainKey: "missing"} - - if _, err := store.Write(context.Background(), id, []byte("unexpected"), 5); !errors.Is(err, ErrConflict) { - t.Fatalf("Write error = %v, want ErrConflict", err) - } -} - -func TestSQLiteStore_ReadMissingReturnsZeroRecord(t *testing.T) { - store := newSQLiteTestStore(t) - - record, err := store.Read(context.Background(), GrainId{GrainType: "account", GrainKey: "missing"}) - if err != nil { - t.Fatalf("Read: %v", err) - } - if record.Data != nil || record.ETag != 0 { - t.Fatalf("Record = %#v, want zero Record", record) - } -} - -func TestSQLiteStore_DifferentIdentitiesAreIndependent(t *testing.T) { - store := newSQLiteTestStore(t) - alice := GrainId{GrainType: "account", GrainKey: "alice"} - bob := GrainId{GrainType: "account", GrainKey: "bob"} - - if _, err := store.Write(context.Background(), alice, []byte("alice"), 0); err != nil { - t.Fatalf("alice Write: %v", err) - } - if _, err := store.Write(context.Background(), bob, []byte("bob"), 0); err != nil { - t.Fatalf("bob Write: %v", err) - } - - aliceRecord, err := store.Read(context.Background(), alice) - if err != nil { - t.Fatalf("alice Read: %v", err) - } - bobRecord, err := store.Read(context.Background(), bob) - if err != nil { - t.Fatalf("bob Read: %v", err) - } - if string(aliceRecord.Data) != "alice" || aliceRecord.ETag != 1 { - t.Fatalf("alice Record = %#v", aliceRecord) - } - if string(bobRecord.Data) != "bob" || bobRecord.ETag != 1 { - t.Fatalf("bob Record = %#v", bobRecord) - } -} - -func TestSQLiteStore_ReadAndWriteCopyData(t *testing.T) { - store := newSQLiteTestStore(t) - id := GrainId{GrainType: "account", GrainKey: "alice"} - data := []byte("original") - - if _, err := store.Write(context.Background(), id, data, 0); err != nil { - t.Fatalf("Write: %v", err) - } - data[0] = 'X' - - record, err := store.Read(context.Background(), id) - if err != nil { - t.Fatalf("Read: %v", err) - } - record.Data[0] = 'Y' - - unchanged, err := store.Read(context.Background(), id) - if err != nil { - t.Fatalf("second Read: %v", err) - } - if string(unchanged.Data) != "original" { - t.Fatalf("stored data = %q, want original", unchanged.Data) - } -} - func TestSQLiteStore_PersistsAcrossReopen(t *testing.T) { path := filepath.Join(t.TempDir(), "store.db") id := GrainId{GrainType: "account", GrainKey: "alice"} @@ -182,7 +41,11 @@ func TestSQLiteStore_PersistsAcrossReopen(t *testing.T) { } func TestOpenSQLite_PathWithURICharactersUsesRequestedFile(t *testing.T) { - path := filepath.Join(t.TempDir(), "store?#.db") + name := "store?#.db" + if runtime.GOOS == "windows" { + name = "store%.db" + } + path := filepath.Join(t.TempDir(), name) id := GrainId{GrainType: "account", GrainKey: "uri"} first, err := OpenSQLite(path) @@ -222,9 +85,9 @@ func TestOpenSQLite_MissingParentDirErrorNamesPath(t *testing.T) { _, err := OpenSQLite(path) if err == nil { - t.Fatalf("OpenSQLite: expected error for missing parent dir") + t.Fatal("OpenSQLite: expected error for missing parent dir") } - if !strings.Contains(err.Error(), path) { + if !strings.Contains(err.Error(), path) && !strings.Contains(err.Error(), strconv.Quote(path)) { t.Fatalf("OpenSQLite error %q does not mention path %q", err, path) } } diff --git a/store/sqlite_windows_test.go b/store/sqlite_windows_test.go new file mode 100644 index 0000000..cb3738d --- /dev/null +++ b/store/sqlite_windows_test.go @@ -0,0 +1,13 @@ +//go:build windows + +package store + +import "testing" + +func TestSQLiteFileURI_WindowsDriveIsURIPath(t *testing.T) { + const path = `C:\data\store?#.db` + const want = "file:///C:/data/store%3F%23.db" + if got := sqliteFileURI(path); got != want { + t.Fatalf("sqliteFileURI(%q) = %q, want %q", path, got, want) + } +} diff --git a/store/store.go b/store/store.go index 32742be..b70916e 100644 --- a/store/store.go +++ b/store/store.go @@ -30,7 +30,12 @@ type ETag int64 // and implementations may wrap it when returning the error. var ErrConflict = errors.New("store: etag conflict") -// Record is the data and ETag returned for one entity identity. +// ErrInvalidRecordData reports a Write with empty record data. A missing +// record is the zero Record returned by Read. It is not a confirmed empty +// record. +var ErrInvalidRecordData = errors.New("store: record data must not be empty") + +// Record is the data and ETag returned for one GrainId. // // A missing record is returned as the zero Record with a nil error. Store // implementations must not retain or expose the caller's mutable Data slice. @@ -39,16 +44,21 @@ type Record struct { ETag ETag } -// Store persists one entity-state record per GrainId. +// Store persists one State record per GrainId. // // Implementations must support concurrent calls. Write must atomically compare // the current ETag with expect and commit only on an exact match. A missing // record has ETag zero; a successful write stores a new value and returns the // incremented ETag. A failed comparison must leave the record unchanged and -// return an error matching ErrConflict with errors.Is. Methods must honor the -// context and return its error when it is canceled before the operation can -// complete. +// return an error matching ErrConflict with errors.Is. After storage work +// starts, another Write error can have an unknown result. The new record can be +// committed when the caller receives an error. Methods must honor the context +// and return its error when it is canceled before storage work starts. Write +// must reject empty data with ErrInvalidRecordData and must not create a record. type Store interface { + // Check verifies that the store can serve State without changing stored + // data. + Check(context.Context) error // Read returns the record for id, or a zero Record and nil when it is absent. Read(context.Context, GrainId) (Record, error) // Write replaces id's data when its current ETag equals expect. @@ -66,11 +76,12 @@ func timeFromValue(value int64) time.Time { // Memory is an in-memory implementation of Store, MemberStore, and // ReminderStore. type Memory struct { - mu sync.RWMutex - records map[GrainId]Record - reminders map[reminderKey]Reminder - members map[memberKey]Member - memberClock clock.Clock + mu sync.RWMutex + records map[GrainId]Record + reminders map[reminderKey]Reminder + reminderETag ETag + members map[memberKey]Member + memberClock clock.Clock } var _ Store = (*Memory)(nil) @@ -94,6 +105,12 @@ func NewMemory(memberClocks ...clock.Clock) *Memory { } } +// Check returns the context error when the context is done. Memory has no +// external resource to probe. +func (m *Memory) Check(ctx context.Context) error { + return ctx.Err() +} + // Read returns a copy of the stored record for id, or a zero Record and nil // when id has not been written. func (m *Memory) Read(ctx context.Context, id GrainId) (Record, error) { @@ -102,6 +119,9 @@ func (m *Memory) Read(ctx context.Context, id GrainId) (Record, error) { } m.mu.RLock() defer m.mu.RUnlock() + if err := ctx.Err(); err != nil { + return Record{}, err + } record, ok := m.records[id] if !ok { @@ -118,8 +138,14 @@ func (m *Memory) Write(ctx context.Context, id GrainId, data []byte, expect ETag if err := ctx.Err(); err != nil { return 0, err } + if len(data) == 0 { + return 0, ErrInvalidRecordData + } m.mu.Lock() defer m.mu.Unlock() + if err := ctx.Err(); err != nil { + return 0, err + } current := m.records[id] if current.ETag != expect { @@ -132,5 +158,10 @@ func (m *Memory) Write(ctx context.Context, id GrainId, data []byte, expect ETag } func clone(data []byte) []byte { - return append([]byte(nil), data...) + if data == nil { + return nil + } + result := make([]byte, len(data)) + copy(result, data) + return result } diff --git a/store/store_test.go b/store/store_test.go index cc6bff2..3b6506d 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -7,148 +7,6 @@ import ( "time" ) -func TestMemoryStore_WriteWithMatchingETagReturnsNewETag(t *testing.T) { - memory := NewMemory() - id := GrainId{GrainType: "account", GrainKey: "alice"} - - etag, err := memory.Write(context.Background(), id, []byte("first"), 0) - if err != nil { - t.Fatalf("Write: %v", err) - } - if etag != 1 { - t.Fatalf("ETag = %d, want 1", etag) - } - - record, err := memory.Read(context.Background(), id) - if err != nil { - t.Fatalf("Read: %v", err) - } - if string(record.Data) != "first" || record.ETag != etag { - t.Fatalf("Record = %#v, want data first and ETag %d", record, etag) - } - - nextETag, err := memory.Write(context.Background(), id, []byte("second"), etag) - if err != nil { - t.Fatalf("second Write: %v", err) - } - if nextETag != 2 { - t.Fatalf("second ETag = %d, want 2", nextETag) - } -} - -func TestMemoryStore_ConflictLeavesRecordUnchanged(t *testing.T) { - memory := NewMemory() - id := GrainId{GrainType: "account", GrainKey: "alice"} - - etag, err := memory.Write(context.Background(), id, []byte("original"), 0) - if err != nil { - t.Fatalf("seed Write: %v", err) - } - - newETag, err := memory.Write(context.Background(), id, []byte("replacement"), etag+1) - if !errors.Is(err, ErrConflict) { - t.Fatalf("Write error = %v, want ErrConflict", err) - } - if newETag != 0 { - t.Fatalf("conflicting ETag = %d, want 0", newETag) - } - - record, err := memory.Read(context.Background(), id) - if err != nil { - t.Fatalf("Read: %v", err) - } - if string(record.Data) != "original" || record.ETag != etag { - t.Fatalf("Record after conflict = %#v, want original data and ETag %d", record, etag) - } -} - -func TestMemoryStore_ZeroETagConflictsWithExistingRecord(t *testing.T) { - memory := NewMemory() - id := GrainId{GrainType: "account", GrainKey: "alice"} - - if _, err := memory.Write(context.Background(), id, []byte("existing"), 0); err != nil { - t.Fatalf("seed Write: %v", err) - } - - if _, err := memory.Write(context.Background(), id, []byte("overwrite"), 0); !errors.Is(err, ErrConflict) { - t.Fatalf("Write error = %v, want ErrConflict", err) - } -} - -func TestMemoryStore_NonzeroETagConflictsWithMissingRecord(t *testing.T) { - memory := NewMemory() - id := GrainId{GrainType: "account", GrainKey: "missing"} - - if _, err := memory.Write(context.Background(), id, []byte("unexpected"), 5); !errors.Is(err, ErrConflict) { - t.Fatalf("Write error = %v, want ErrConflict", err) - } -} - -func TestMemoryStore_ReadMissingReturnsZeroRecord(t *testing.T) { - memory := NewMemory() - - record, err := memory.Read(context.Background(), GrainId{GrainType: "account", GrainKey: "missing"}) - if err != nil { - t.Fatalf("Read: %v", err) - } - if record.Data != nil || record.ETag != 0 { - t.Fatalf("Record = %#v, want zero Record", record) - } -} - -func TestMemoryStore_DifferentIdentitiesAreIndependent(t *testing.T) { - memory := NewMemory() - alice := GrainId{GrainType: "account", GrainKey: "alice"} - bob := GrainId{GrainType: "account", GrainKey: "bob"} - - if _, err := memory.Write(context.Background(), alice, []byte("alice"), 0); err != nil { - t.Fatalf("alice Write: %v", err) - } - if _, err := memory.Write(context.Background(), bob, []byte("bob"), 0); err != nil { - t.Fatalf("bob Write: %v", err) - } - - aliceRecord, err := memory.Read(context.Background(), alice) - if err != nil { - t.Fatalf("alice Read: %v", err) - } - bobRecord, err := memory.Read(context.Background(), bob) - if err != nil { - t.Fatalf("bob Read: %v", err) - } - if string(aliceRecord.Data) != "alice" || aliceRecord.ETag != 1 { - t.Fatalf("alice Record = %#v", aliceRecord) - } - if string(bobRecord.Data) != "bob" || bobRecord.ETag != 1 { - t.Fatalf("bob Record = %#v", bobRecord) - } -} - -func TestMemoryStore_ReadAndWriteCopyData(t *testing.T) { - memory := NewMemory() - id := GrainId{GrainType: "account", GrainKey: "alice"} - data := []byte("original") - - if _, err := memory.Write(context.Background(), id, data, 0); err != nil { - t.Fatalf("Write: %v", err) - } - data[0] = 'X' - - record, err := memory.Read(context.Background(), id) - if err != nil { - t.Fatalf("Read: %v", err) - } - record.Data[0] = 'Y' - - unchanged, err := memory.Read(context.Background(), id) - if err != nil { - t.Fatalf("second Read: %v", err) - } - if string(unchanged.Data) != "original" { - t.Fatalf("stored data = %q, want original", unchanged.Data) - } -} - func TestMemoryStore_MethodsHonorCanceledContext(t *testing.T) { memory := NewMemory() ctx, cancel := context.WithCancel(context.Background()) @@ -164,7 +22,7 @@ func TestMemoryStore_MethodsHonorCanceledContext(t *testing.T) { wantCanceled(err) _, err = memory.Write(ctx, GrainId{GrainType: "account", GrainKey: "alice"}, nil, 0) wantCanceled(err) - _, err = memory.ListDue(ctx, time.Time{}) + _, err = memory.ListDue(ctx, time.Time{}, nil, 1) wantCanceled(err) _, err = memory.Claim(ctx, Reminder{}, time.Time{}) wantCanceled(err) diff --git a/timer/timer.go b/timer/timer.go deleted file mode 100644 index 926e321..0000000 --- a/timer/timer.go +++ /dev/null @@ -1,141 +0,0 @@ -// Package timer polls persisted Reminders and delivers due Grain Calls for -// gor. -// -// It is an implementation package, not an application dependency. Create and -// manage Reminders through the root gor package's Reminder APIs instead of -// importing timer directly. -package timer - -import ( - "context" - "time" - - "github.com/suraciii/gor/clock" - "github.com/suraciii/gor/store" -) - -type Table interface { - ListDue(context.Context, time.Time) ([]store.Reminder, error) - Claim(context.Context, store.Reminder, time.Time) (bool, error) -} - -type Invoker interface { - Owns(store.GrainId) bool - Invoke(context.Context, store.GrainId, string, any, any) error -} - -// ReminderCallFactory creates the normal typed request and reply values for a -// claimed Reminder. The root package converts these time values into its -// public TickStatus before calling generated code. -type ReminderCallFactory func(store.GrainId, string, time.Time, time.Duration, time.Time) (any, any) - -type Poller struct { - table Table - clock clock.Clock - interval time.Duration - invoker Invoker - newCall ReminderCallFactory - - ctx context.Context - cancel context.CancelFunc - done chan struct{} -} - -func New(table Table, clock clock.Clock, interval time.Duration, invoker Invoker, newCall ReminderCallFactory) *Poller { - ctx, cancel := context.WithCancel(context.Background()) - poller := &Poller{ - table: table, - clock: clock, - interval: interval, - invoker: invoker, - newCall: newCall, - ctx: ctx, - cancel: cancel, - done: make(chan struct{}), - } - go poller.run(poller.clock.NewTicker(poller.interval)) - return poller -} - -func (p *Poller) Close() { - p.cancel() - <-p.done -} - -func (p *Poller) run(ticker clock.Ticker) { - defer func() { - ticker.Stop() - close(p.done) - }() - - for { - select { - case <-ticker.C(): - p.poll() - case <-p.ctx.Done(): - return - } - } -} - -func (p *Poller) poll() { - now := p.clock.Now() - reminders, err := p.table.ListDue(p.ctx, now) - if err != nil { - return - } - for _, reminder := range reminders { - if p.ctx.Err() != nil { - return - } - if !p.invoker.Owns(reminder.GrainId) { - continue - } - nextDueAt := nextDueAt(reminder, now) - claimed, err := p.table.Claim(p.ctx, reminder, nextDueAt) - if err != nil || !claimed { - continue - } - if p.newCall == nil { - continue - } - args, reply := p.newCall(reminder.GrainId, reminder.Method, reminder.FirstTickTime, reminder.Interval, reminder.DueAt) - _ = p.invoker.Invoke(p.ctx, reminder.GrainId, reminder.Method, args, reply) - } -} - -const maxDuration = time.Duration(1<<63 - 1) - -func nextDueAt(reminder store.Reminder, now time.Time) time.Time { - period := reminder.Interval - if period <= 0 { - return time.Time{} - } - if reminder.DueAt.After(now) { - return reminder.DueAt - } - - elapsed := now.Sub(reminder.DueAt) - missed := elapsed / period - if missed == maxDuration { - return futureDueAt(now, period) - } - missed++ - if missed > maxDuration/period { - return futureDueAt(now, period) - } - - candidate := reminder.DueAt.Add(period * missed) - if !candidate.After(now) { - return futureDueAt(now, period) - } - return candidate -} - -func futureDueAt(now time.Time, period time.Duration) time.Time { - fallback := now.Add(period) - if fallback.After(now) { - return fallback - } - return now.Add(time.Nanosecond) -} diff --git a/timer/timer_test.go b/timer/timer_test.go deleted file mode 100644 index c8a5ae6..0000000 --- a/timer/timer_test.go +++ /dev/null @@ -1,375 +0,0 @@ -package timer - -import ( - "context" - "slices" - "sync" - "sync/atomic" - "testing" - "testing/synctest" - "time" - - "github.com/suraciii/gor/clock" - "github.com/suraciii/gor/store" -) - -type fakeTable struct { - rows []store.Reminder - claimWon bool - - recorder *stepRecorder - nextDueAt []time.Time -} - -func (t *fakeTable) ListDue(context.Context, time.Time) ([]store.Reminder, error) { - t.recorder.record("list") - return append([]store.Reminder(nil), t.rows...), nil -} - -func (t *fakeTable) Claim(_ context.Context, schedule store.Reminder, nextDueAt time.Time) (bool, error) { - t.recorder.mu.Lock() - t.recorder.steps = append(t.recorder.steps, "claim") - t.nextDueAt = append(t.nextDueAt, nextDueAt) - t.recorder.mu.Unlock() - return t.claimWon, nil -} - -type recordingInvoker struct { - recorder *stepRecorder - calls []store.GrainId -} - -func testReminderCall(store.GrainId, string, time.Time, time.Duration, time.Time) (any, any) { - return &struct{}{}, &struct{}{} -} - -func (i *recordingInvoker) Invoke(_ context.Context, id store.GrainId, method string, _, _ any) error { - i.recorder.mu.Lock() - i.recorder.steps = append(i.recorder.steps, "invoke") - i.calls = append(i.calls, id) - i.recorder.mu.Unlock() - return nil -} - -func (i *recordingInvoker) Owns(store.GrainId) bool { - return true -} - -type blockingInvoker struct { - started chan struct{} - finished chan struct{} -} - -type stepRecorder struct { - mu sync.Mutex - steps []string -} - -func (r *stepRecorder) record(step string) { - r.mu.Lock() - r.steps = append(r.steps, step) - r.mu.Unlock() -} - -func (i *blockingInvoker) Invoke(ctx context.Context, _ store.GrainId, _ string, _, _ any) error { - close(i.started) - <-ctx.Done() - close(i.finished) - return ctx.Err() -} - -func (i *blockingInvoker) Owns(store.GrainId) bool { - return true -} - -type ownershipInvoker struct { - owns bool - calls atomic.Int32 -} - -func (i *ownershipInvoker) Owns(store.GrainId) bool { - return i.owns -} - -func (i *ownershipInvoker) Invoke(context.Context, store.GrainId, string, any, any) error { - i.calls.Add(1) - return nil -} - -func TestPoller_ClaimsBeforeInvoking(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - start := time.Unix(100, 0).UTC() - recorder := &stepRecorder{} - backend := &fakeTable{ - rows: []store.Reminder{{ - GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, - Name: "wake", - Method: "Wake", - DueAt: start.Add(-time.Second), - Interval: time.Hour, - ETag: 1, - }}, - claimWon: true, - recorder: recorder, - } - fakeClock := clock.NewFake(start) - invoker := &recordingInvoker{recorder: recorder} - poller := New(backend, fakeClock, time.Second, invoker, testReminderCall) - synctest.Wait() - fakeClock.Advance(time.Second) - synctest.Wait() - poller.Close() - - if got, want := recorder.steps, []string{"list", "claim", "invoke"}; !slices.Equal(got, want) { - t.Fatalf("steps = %v, want %v", got, want) - } - }) -} - -func TestPoller_AdvancesToFirstFutureTime(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - start := time.Unix(200, 0).UTC() - interval := time.Hour - backend := &fakeTable{ - rows: []store.Reminder{{ - GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, - Name: "wake", - Method: "Wake", - DueAt: start.Add(-3 * interval), - Interval: interval, - ETag: 1, - }}, - claimWon: true, - recorder: &stepRecorder{}, - } - fakeClock := clock.NewFake(start) - invoker := &recordingInvoker{recorder: backend.recorder} - poller := New(backend, fakeClock, time.Second, invoker, testReminderCall) - synctest.Wait() - fakeClock.Advance(time.Second) - synctest.Wait() - poller.Close() - - if len(backend.nextDueAt) != 1 { - t.Fatalf("next due times = %v, want one claim", backend.nextDueAt) - } - want := start.Add(interval) - if !backend.nextDueAt[0].Equal(want) { - t.Fatalf("next due time = %s, want %s", backend.nextDueAt[0], want) - } - }) -} - -func TestNextDueAt_LargeDowntimeReturnsPromptly(t *testing.T) { - period := time.Nanosecond - dueAt := time.Unix(0, 0).UTC() - now := dueAt.Add(time.Hour) - reminder := store.Reminder{DueAt: dueAt, Interval: period} - - got := nextDueAt(reminder, now) - want := now.Add(period) - if !got.After(now) || !got.Equal(want) { - t.Fatalf("next due time = %s, want %s strictly after now", got, want) - } - - future := now.Add(time.Hour) - reminder.DueAt = future - if got := nextDueAt(reminder, now); !got.Equal(future) { - t.Fatalf("future due time = %s, want unchanged %s", got, future) - } - - reminder.DueAt = now - reminder.Interval = 2 * time.Nanosecond - if got := nextDueAt(reminder, now); !got.Equal(now.Add(reminder.Interval)) { - t.Fatalf("due-now next time = %s, want %s", got, now.Add(reminder.Interval)) - } - - for _, interval := range []time.Duration{0, -time.Nanosecond} { - reminder.Interval = interval - if got := nextDueAt(reminder, now); !got.IsZero() { - t.Fatalf("interval %s next time = %s, want zero", interval, got) - } - } - - const maxElapsed = time.Duration(1<<63 - 1) - overflowDueAt := time.Unix(0, 0).UTC() - overflowNow := overflowDueAt.Add(maxElapsed) - reminder.DueAt = overflowDueAt - reminder.Interval = 2 * time.Nanosecond - if got := nextDueAt(reminder, overflowNow); !got.After(overflowNow) { - t.Fatalf("overflow fallback = %s, want strictly after %s", got, overflowNow) - } -} - -func TestPoller_PassesPeriodicTickStatus(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - start := time.Unix(250, 0).UTC() - period := time.Hour - first := start.Add(-3 * period) - due := start.Add(-period) - backend := &fakeTable{ - rows: []store.Reminder{{ - GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, - Name: "wake", - Method: "Wake", - FirstTickTime: first, - DueAt: due, - Interval: period, - ETag: 1, - }}, - claimWon: true, - recorder: &stepRecorder{}, - } - fakeClock := clock.NewFake(start) - var gotFirst, gotCurrent time.Time - var gotPeriod time.Duration - factory := func(_ store.GrainId, _ string, firstTick time.Time, tickPeriod time.Duration, current time.Time) (any, any) { - gotFirst = firstTick - gotPeriod = tickPeriod - gotCurrent = current - return &struct{}{}, &struct{}{} - } - poller := New(backend, fakeClock, time.Second, &recordingInvoker{recorder: backend.recorder}, factory) - synctest.Wait() - fakeClock.Advance(time.Second) - synctest.Wait() - poller.Close() - - if !gotFirst.Equal(first) || gotPeriod != period || !gotCurrent.Equal(due) { - t.Fatalf("TickStatus = first %s period %s current %s, want %s %s %s", gotFirst, gotPeriod, gotCurrent, first, period, due) - } - }) -} - -func TestPoller_PassesZeroPeriodForOneShot(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - start := time.Unix(275, 0).UTC() - backend := &fakeTable{ - rows: []store.Reminder{{ - GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, - Name: "wake", - Method: "Wake", - FirstTickTime: start, - DueAt: start, - ETag: 1, - }}, - claimWon: true, - recorder: &stepRecorder{}, - } - fakeClock := clock.NewFake(start) - var gotPeriod time.Duration - factory := func(_ store.GrainId, _ string, _ time.Time, period time.Duration, _ time.Time) (any, any) { - gotPeriod = period - return &struct{}{}, &struct{}{} - } - poller := New(backend, fakeClock, time.Second, &recordingInvoker{recorder: backend.recorder}, factory) - synctest.Wait() - fakeClock.Advance(time.Second) - synctest.Wait() - poller.Close() - - if gotPeriod != 0 { - t.Fatalf("one-shot Period = %s, want 0", gotPeriod) - } - }) -} - -func TestPoller_ClaimFailureDoesNotInvoke(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - start := time.Unix(300, 0).UTC() - backend := &fakeTable{ - rows: []store.Reminder{{ - GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, - Name: "wake", - Method: "Wake", - DueAt: start.Add(-time.Second), - }}, - recorder: &stepRecorder{}, - } - fakeClock := clock.NewFake(start) - invoker := &recordingInvoker{recorder: backend.recorder} - poller := New(backend, fakeClock, time.Second, invoker, testReminderCall) - synctest.Wait() - fakeClock.Advance(time.Second) - synctest.Wait() - poller.Close() - - if len(invoker.calls) != 0 { - t.Fatalf("invocations = %v, want none", invoker.calls) - } - }) -} - -func TestPoller_SkipsSchedulesNotOwnedByInvoker(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - start := time.Unix(350, 0).UTC() - backend := store.NewMemory() - schedule := store.Reminder{ - GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, - Name: "wake", - Method: "Wake", - DueAt: start.Add(-time.Second), - } - if err := backend.Put(context.Background(), schedule); err != nil { - t.Fatalf("Put: %v", err) - } - - nonOwner := &ownershipInvoker{} - owner := &ownershipInvoker{owns: true} - nonOwnerClock := clock.NewFake(start) - ownerClock := clock.NewFake(start) - nonOwnerPoller := New(backend, nonOwnerClock, time.Second, nonOwner, testReminderCall) - ownerPoller := New(backend, ownerClock, time.Second, owner, testReminderCall) - synctest.Wait() - - nonOwnerClock.Advance(time.Second) - synctest.Wait() - if got := nonOwner.calls.Load(); got != 0 { - t.Fatalf("non-owner invocations = %d, want 0", got) - } - - ownerClock.Advance(time.Second) - synctest.Wait() - if got := owner.calls.Load(); got != 1 { - t.Fatalf("owner invocations = %d, want 1", got) - } - if got := nonOwner.calls.Load(); got != 0 { - t.Fatalf("non-owner invocations after owner poll = %d, want 0", got) - } - nonOwnerPoller.Close() - ownerPoller.Close() - }) -} - -func TestPoller_CloseStopsTheGoroutine(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - start := time.Unix(400, 0).UTC() - backend := &fakeTable{ - rows: []store.Reminder{{ - GrainId: store.GrainId{GrainType: "account", GrainKey: "alice"}, - Name: "wake", - Method: "Wake", - DueAt: start.Add(-time.Second), - }}, - claimWon: true, - recorder: &stepRecorder{}, - } - fakeClock := clock.NewFake(start) - invoker := &blockingInvoker{started: make(chan struct{}), finished: make(chan struct{})} - poller := New(backend, fakeClock, time.Second, invoker, testReminderCall) - synctest.Wait() - fakeClock.Advance(time.Second) - synctest.Wait() - poller.Close() - poller.Close() - select { - case <-invoker.finished: - default: - t.Fatal("invocation is still running after Close") - } - }) -} - -var _ Table = (*fakeTable)(nil) -var _ Invoker = (*recordingInvoker)(nil) -var _ Invoker = (*blockingInvoker)(nil) diff --git a/transport/frame.go b/transport/frame.go index f7ad0e1..3d4c915 100644 --- a/transport/frame.go +++ b/transport/frame.go @@ -2,7 +2,7 @@ // runtimes. // // It defines the extension boundary for custom transports and provides TCP -// and frame helpers. Transport does not interpret entity identities, methods, +// and frame helpers. Transport does not interpret GrainIds, methods, // or application payloads. package transport diff --git a/transport/frame_test.go b/transport/frame_test.go index 6de58df..439e564 100644 --- a/transport/frame_test.go +++ b/transport/frame_test.go @@ -157,6 +157,46 @@ func TestWriteFrameRejectsInvalidTypeAndOversizedPayload(t *testing.T) { } } +func FuzzReadFrame(f *testing.F) { + for _, frame := range []Frame{ + {ID: 1, Type: FrameRequest}, + {ID: 2, Type: FrameResponse, Payload: []byte("reply")}, + {ID: 3, Type: FrameError, Payload: []byte("failed")}, + } { + var wire bytes.Buffer + if err := WriteFrame(&wire, frame); err != nil { + f.Fatal(err) + } + f.Add(wire.Bytes()) + } + f.Add([]byte{}) + f.Add([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99}) + f.Add([]byte{0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 1, byte(FrameRequest)}) + f.Add([]byte{0, 0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, byte(FrameRequest)}) + + f.Fuzz(func(t *testing.T, wire []byte) { + reader := bytes.NewReader(wire) + frame, err := ReadFrame(reader) + if err != nil { + if frame.ID != 0 || frame.Type != 0 || frame.Payload != nil { + t.Fatalf("ReadFrame error returned nonzero Frame: %#v", frame) + } + return + } + if len(frame.Payload) > MaxPayloadSize || !frame.Type.valid() { + t.Fatalf("ReadFrame returned invalid Frame: %#v", frame) + } + var encoded bytes.Buffer + if err := WriteFrame(&encoded, frame); err != nil { + t.Fatalf("WriteFrame after successful decode: %v", err) + } + consumed := len(wire) - reader.Len() + if consumed != encoded.Len() || !bytes.Equal(wire[:consumed], encoded.Bytes()) { + t.Fatalf("decoded Frame did not preserve its wire prefix: consumed %d, encoded %d", consumed, encoded.Len()) + } + }) +} + type headerOnlyReader struct { header []byte read bool diff --git a/transport/main_test.go b/transport/main_test.go deleted file mode 100644 index acdc351..0000000 --- a/transport/main_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package transport - -import ( - "fmt" - "os" - "runtime" - "testing" -) - -func TestMain(m *testing.M) { - before := runtime.NumGoroutine() - code := m.Run() - after := runtime.NumGoroutine() - if after != before { - fmt.Fprintf(os.Stderr, "goroutine leak: before=%d after=%d\n", before, after) - if code == 0 { - code = 1 - } - } - os.Exit(code) -}