diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 967b934be..00c57a639 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -86,3 +86,29 @@ jobs: SKIP_CLOUDSTORAGE_TESTS: "1" SKIP_DOCKER_TESTS: "1" run: make test + + test-sqlite-cdc: + name: Test SQLite CDC (Linux, CGO, race) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version: '1.26.5' + cache: true + + - name: Install dependencies + run: go mod download + + - name: Run tagged SQLite CDC tests + env: + CGO_ENABLED: 1 + GORACE: "halt_on_error=1" + SKIP_TEMPORAL_TESTS: "1" + SKIP_CLOUDSTORAGE_TESTS: "1" + SKIP_DOCKER_TESTS: "1" + run: make test-cdc-sqlite diff --git a/Makefile b/Makefile index 4c37d07fe..ffd0bf184 100644 --- a/Makefile +++ b/Makefile @@ -13,9 +13,17 @@ test: go test ./system/... -v -race -short go test ./service/... -v -race -short go test ./cluster/... -v -race -short - go test --tags "fts5 sqlite_vec treesitter" ./runtime/... -v -race -short + go test --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" ./runtime/... -v -race -short go test ./boot/... -v -race -short - go test --tags "fts5 sqlite_vec treesitter" ./cmd/... -v -race -short + go test --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" ./cmd/... -v -race -short + +# The default service test intentionally stays untagged so the SQLite stub and +# the non-CGO portability path remain covered. The real preupdate-hook source +# is exercised by the Linux CGO CI job through this target. +.PHONY: test-cdc-sqlite +test-cdc-sqlite: + CGO_ENABLED=1 go test ./service/sql/... ./service/cdc/sqlite -v -race -tags sqlite_preupdate_hook + CGO_ENABLED=1 go test ./service/cdc/sqlite -v -race -timeout 300s -tags "integration sqlite_preupdate_hook" test-system: go test ./internal/... -v -race @@ -25,7 +33,7 @@ test-system: test-runtime: go test ./internal/... -v -race go test ./api/... -v -race - go test --tags "fts5 sqlite_vec treesitter" ./runtime/... -v -race + go test --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" ./runtime/... -v -race test-service: go test ./internal/... -v -race @@ -45,7 +53,7 @@ test-network: .PHONY: lint lint: - go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.8.0 run --timeout=10m --build-tags=race ./... + go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.8.0 run --timeout=10m --build-tags=race,sqlite_preupdate_hook ./... # Mutation testing with gremlins. Coverage is scoped to the directory gremlins # runs from, so target a package subtree via MUTATE_DIR. workers=1 keeps per- @@ -92,7 +100,7 @@ build-wippy: build-wippy-local .PHONY: build-wippy-local build-wippy-local: mkdir -p ./dist - CGO_ENABLED=1 go build --tags "fts5 sqlite_vec treesitter" \ + CGO_ENABLED=1 go build --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" \ -ldflags="$(WIPPY_LDFLAGS)" \ -trimpath \ -o ./dist/wippy-$(shell go env GOOS)-$(shell go env GOARCH) \ @@ -104,7 +112,7 @@ build-wippy-all: build-wippy-linux-amd64 build-wippy-linux-arm64 build-wippy-dar .PHONY: build-wippy-linux-amd64 build-wippy-linux-amd64: mkdir -p ./dist - CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build --tags "fts5 sqlite_vec treesitter" \ + CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" \ -ldflags="$(WIPPY_LDFLAGS)" \ -trimpath \ -o ./dist/wippy-linux-amd64 \ @@ -114,7 +122,7 @@ build-wippy-linux-amd64: build-wippy-linux-arm64: mkdir -p ./dist CGO_LDFLAGS="" CGO_CFLAGS="" CC=aarch64-linux-gnu-gcc \ - CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build --tags "fts5 sqlite_vec treesitter" \ + CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" \ -ldflags="$(WIPPY_LDFLAGS)" \ -trimpath \ -o ./dist/wippy-linux-arm64 \ @@ -123,7 +131,7 @@ build-wippy-linux-arm64: .PHONY: build-wippy-darwin-amd64 build-wippy-darwin-amd64: mkdir -p ./dist - CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build --tags "fts5 sqlite_vec treesitter" \ + CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" \ -ldflags="$(WIPPY_LDFLAGS)" \ -trimpath \ -o ./dist/wippy-darwin-amd64 \ @@ -132,7 +140,7 @@ build-wippy-darwin-amd64: .PHONY: build-wippy-darwin-arm64 build-wippy-darwin-arm64: mkdir -p ./dist - CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build --tags "fts5 sqlite_vec treesitter" \ + CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" \ -ldflags="$(WIPPY_LDFLAGS)" \ -trimpath \ -o ./dist/wippy-darwin-arm64 \ @@ -142,7 +150,7 @@ build-wippy-darwin-arm64: build-wippy-windows-amd64: mkdir -p ./dist CGO_LDFLAGS="" CGO_CFLAGS="" CC=x86_64-w64-mingw32-gcc \ - CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build --tags "fts5 sqlite_vec treesitter" \ + CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" \ -ldflags="$(WIPPY_LDFLAGS)" \ -trimpath \ -o ./dist/wippy-windows-amd64.exe \ @@ -175,4 +183,4 @@ build-sign-wippy-windows: build-wippy-windows-amd64 sign-wippy-windows .PHONY: run-wippy run-wippy: - go run --tags "fts5 sqlite_vec treesitter" -ldflags="$(WIPPY_LDFLAGS)" ./cmd/wippy/ $(ARGS) + go run --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" -ldflags="$(WIPPY_LDFLAGS)" ./cmd/wippy/ $(ARGS) diff --git a/api/process/errors.go b/api/process/errors.go index aaf80ae07..2658d08be 100644 --- a/api/process/errors.go +++ b/api/process/errors.go @@ -27,6 +27,12 @@ var ( ErrProcessNotIdle = apierror.New(InvalidState, "process is not idle").WithRetryable(apierror.False) ErrSchedulerStopping = apierror.New(InvalidState, "scheduler is stopping").WithRetryable(apierror.False) + + // ErrMessageQueueOverflow is delivered as a terminal stream error when an + // explicitly bounded message subscription exhausts its retained backlog. + // It is intentionally an ordinary Go sentinel so consumers can use + // errors.Is on the error carried by the terminal payload. + ErrMessageQueueOverflow = errors.New("message queue limit exceeded") ) // ErrProcessReplacementRequested is an internal scheduler sentinel. A process diff --git a/api/process/queue.go b/api/process/queue.go index 12dec055a..6bd2199dd 100644 --- a/api/process/queue.go +++ b/api/process/queue.go @@ -5,6 +5,9 @@ package process import ( "sync" "sync/atomic" + + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/relay" ) // todo: move from api @@ -17,14 +20,70 @@ const defaultQueueCap = 16 // Generation counter ensures stale senders from previous executions // cannot push to a reused queue. type EventQueue struct { - signal chan struct{} - events []Event - drainBuf []Event - generation atomic.Uint64 - mu sync.Mutex - closed atomic.Bool + signal chan struct{} + // Message accounting is opt-in. Ordinary event traffic keeps the + // historical unbounded queue semantics; CDC messages carry MaxItems and/or + // MaxBytes and are admitted through PushMessage. + messageTopics map[string]*messageTopicState + events []Event + drainBuf []Event + generation atomic.Uint64 + mu sync.Mutex + closed atomic.Bool +} + +// messageTopicState is the accounting identity for one bounded topic +// incarnation. It must not be reused after a terminal is drained: a process +// may hold a message reservation past that point while a new stream with the +// same topic is admitted. Per-message leases retain this state until the +// consumer releases the message, so old traffic cannot debit a replacement +// stream's counters. +type messageTopicState struct { + items atomic.Int64 + bytes atomic.Int64 + maxItems int64 + maxBytes int64 + overflowed bool +} + +// messageRetentionLease transfers one EventQueue reservation to the process +// mailbox. Release is idempotent because either the process or relay's pooled +// package cleanup may be the first owner to finish the handoff. +type messageRetentionLease struct { + state *messageTopicState + items int64 + bytes int64 + once sync.Once +} + +func (l *messageRetentionLease) Release() { + if l == nil || l.state == nil { + return + } + l.once.Do(func() { + if l.items > 0 { + l.state.items.Add(-l.items) + } + if l.bytes > 0 { + l.state.bytes.Add(-l.bytes) + } + }) } +// MessageAdmission describes ownership after PushMessage. +// +// Accepted means the queue owns the package. Dropped means the queue emitted +// its overflow terminal but retained no part of the supplied package, so the +// caller must release it. Rejected means the queue did not admit the package +// (closed or stale generation), and the caller must release it as well. +type MessageAdmission uint8 + +const ( + MessageRejected MessageAdmission = iota + MessageDropped + MessageAccepted +) + // NewEventQueue creates a queue with default capacity. func NewEventQueue() *EventQueue { q := &EventQueue{ @@ -61,24 +120,244 @@ func (q *EventQueue) Push(e Event, gen uint64) bool { q.events = append(q.events, e) q.mu.Unlock() - // Non-blocking signal + q.signalPush() + return true +} + +func (q *EventQueue) signalPush() { select { case q.signal <- struct{}{}: default: } - return true } -// PushDirect adds an event without generation check (for scheduler's own use). -func (q *EventQueue) PushDirect(e Event) { +// PushMessage admits a relay package while enforcing the per-topic limits +// carried by its messages. A package can contain messages for more than one +// topic; each message is admitted independently and the package is compacted +// before ownership transfers to the queue. On the first overflow for a topic, +// one synthetic error+terminal message is appended in its position. Later +// traffic for that topic is discarded until Reset. +// +// The queue owns an accepted package and the scheduler releases it after +// processing. The caller owns rejected or fully dropped packages. +func (q *EventQueue) PushMessage(e Event, gen uint64) MessageAdmission { + if e.Type != EventMessage { + if q.Push(e, gen) { + return MessageAccepted + } + return MessageRejected + } + pkg, ok := e.Data.(*relay.Package) + if !ok || pkg == nil { + if q.Push(e, gen) { + return MessageAccepted + } + return MessageRejected + } + + if q.generation.Load() != gen || q.closed.Load() { + return MessageRejected + } + q.mu.Lock() + if q.generation.Load() != gen || q.closed.Load() { + q.mu.Unlock() + return MessageRejected + } + accepted := q.admitPackageLocked(pkg) + if !accepted { + q.mu.Unlock() + return MessageDropped + } q.events = append(q.events, e) q.mu.Unlock() - select { - case q.signal <- struct{}{}: - default: + q.signalPush() + return MessageAccepted +} + +func (q *EventQueue) admitPackageLocked(pkg *relay.Package) bool { + original := pkg.Messages + if len(original) == 0 { + return true + } + + accepted := make([]*relay.Message, 0, len(original)+1) + for _, msg := range original { + if msg == nil { + accepted = append(accepted, nil) + continue + } + + topic := msg.Topic + maxItems := msg.MaxItems + maxBytes := msg.MaxBytes + state := q.messageTopics[topic] + if state != nil { + if maxItems <= 0 { + maxItems = int(state.maxItems) + } else if state.maxItems > 0 && state.maxItems < int64(maxItems) { + maxItems = int(state.maxItems) + } + if maxBytes <= 0 { + maxBytes = state.maxBytes + } else if state.maxBytes > 0 && state.maxBytes < maxBytes { + maxBytes = state.maxBytes + } + } + if maxItems > 0 || maxBytes > 0 { + if state == nil { + if q.messageTopics == nil { + q.messageTopics = make(map[string]*messageTopicState) + } + state = &messageTopicState{} + q.messageTopics[topic] = state + } + if state.maxItems <= 0 || (maxItems > 0 && int64(maxItems) < state.maxItems) { + state.maxItems = int64(maxItems) + } + if state.maxBytes <= 0 || (maxBytes > 0 && maxBytes < state.maxBytes) { + state.maxBytes = maxBytes + } + maxItems = int(state.maxItems) + maxBytes = state.maxBytes + } + + if state != nil && state.overflowed { + relay.ReleaseMessage(msg) + continue + } + + // A terminal never consumes backlog capacity. This also makes the + // synthetic overflow terminal admissible after a full backlog. + if !messageHasData(msg) { + if maxItems > 0 { + msg.MaxItems = maxItems + } + if maxBytes > 0 { + msg.MaxBytes = maxBytes + } + accepted = append(accepted, msg) + continue + } + + payloadBytes := msg.PayloadBytes + if maxBytes > 0 && payloadBytes <= 0 { + // Missing size metadata must not bypass a byte budget. + payloadBytes = maxBytes + } + var items, bytes int64 + if state != nil { + items = state.items.Load() + bytes = state.bytes.Load() + } + if (maxItems > 0 && items >= int64(maxItems)) || + (maxBytes > 0 && (payloadBytes > maxBytes || bytes > maxBytes-payloadBytes)) { + accepted = q.messageOverflowedLocked(state, topic, accepted, maxItems, maxBytes) + relay.ReleaseMessage(msg) + continue + } + + msg.MaxItems = maxItems + msg.MaxBytes = maxBytes + msg.PayloadBytes = payloadBytes + if state != nil { + if maxItems > 0 { + state.items.Add(1) + } + if maxBytes > 0 && payloadBytes > 0 { + state.bytes.Add(payloadBytes) + } + reservationBytes := int64(0) + if maxBytes > 0 { + reservationBytes = payloadBytes + } + msg.SetRetentionLease(&messageRetentionLease{ + state: state, + items: boolInt64(maxItems > 0), + bytes: reservationBytes, + }) + } + accepted = append(accepted, msg) + } + + pkg.Messages = accepted + return len(accepted) > 0 +} + +func boolInt64(v bool) int64 { + if v { + return 1 } + return 0 +} + +func (q *EventQueue) messageOverflowedLocked(state *messageTopicState, topic string, accepted []*relay.Message, maxItems int, maxBytes int64) []*relay.Message { + if state == nil { + if q.messageTopics == nil { + q.messageTopics = make(map[string]*messageTopicState) + } + state = &messageTopicState{maxItems: int64(maxItems), maxBytes: maxBytes} + q.messageTopics[topic] = state + } + if state.overflowed { + return accepted + } + state.overflowed = true + msg := relay.AcquireMessage() + msg.Topic = topic + msg.Payloads = payload.Payloads{payload.NewError(ErrMessageQueueOverflow), payload.NewTerminal()} + msg.MaxItems = maxItems + msg.MaxBytes = maxBytes + msg.PayloadBytes = 0 + return append(accepted, msg) +} + +func messageHasTerminal(msg *relay.Message) bool { + if msg == nil { + return false + } + for _, pl := range msg.Payloads { + if pl != nil && payload.IsTerminal(pl) { + return true + } + } + return false +} + +func messageHasData(msg *relay.Message) bool { + if msg == nil { + return false + } + for _, pl := range msg.Payloads { + if pl == nil || payload.IsTerminal(pl) || pl.Format() == payload.GoError { + continue + } + return true + } + return false +} + +// PushDirect adds an event without generation check (for scheduler's own use). +// It returns false when the queue is closed. A rejected message package is +// released here because PushDirect is an ownership-taking scheduler path; a +// successful message remains owned by the queue/process as usual. +func (q *EventQueue) PushDirect(e Event) bool { + q.mu.Lock() + if q.closed.Load() { + q.mu.Unlock() + if e.Type == EventMessage { + if pkg, ok := e.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } + return false + } + q.events = append(q.events, e) + q.mu.Unlock() + + q.signalPush() + return true } // Drain returns all pending events and clears the queue. @@ -86,10 +365,20 @@ func (q *EventQueue) PushDirect(e Event) { // Single consumer only (scheduler). func (q *EventQueue) Drain() []Event { q.mu.Lock() + // The previous drain result is caller-owned. The single-consumer contract + // means the caller has finished with it before asking for the next batch; + // clear it now so the queue does not retain arbitrary Data values. + for i := range q.drainBuf { + q.drainBuf[i] = Event{} + } + q.drainBuf = q.drainBuf[:0] if len(q.events) == 0 { q.mu.Unlock() return nil } + for _, event := range q.events { + q.retireEventTopicsLocked(event) + } // Swap buffers to avoid allocation q.drainBuf, q.events = q.events, q.drainBuf[:0] @@ -116,7 +405,17 @@ func (q *EventQueue) Signal() <-chan struct{} { func (q *EventQueue) Close() { q.mu.Lock() q.closed.Store(true) + for i, event := range q.events { + q.retireEventTopicsLocked(event) + q.releaseEventPackageLocked(event) + q.events[i] = Event{} + } q.events = q.events[:0] + q.clearMessageAccountingLocked() + // A drained batch belongs to the scheduler. Drop the queue's reference + // without mutating the caller's slice, which may still be in flight while + // Close is called by a supervisor. + q.drainBuf = nil q.mu.Unlock() // Wake any waiters @@ -131,8 +430,16 @@ func (q *EventQueue) Reset() { q.mu.Lock() q.generation.Add(1) // Invalidate all existing senders q.closed.Store(false) + for i, event := range q.events { + q.retireEventTopicsLocked(event) + q.releaseEventPackageLocked(event) + q.events[i] = Event{} + } q.events = q.events[:0] - q.drainBuf = q.drainBuf[:0] + // See Close: the previous Drain result is consumer-owned. Detach it + // rather than touching a potentially concurrent scheduler slice. + q.drainBuf = nil + q.clearMessageAccountingLocked() q.mu.Unlock() // Drain signal channel @@ -142,6 +449,36 @@ func (q *EventQueue) Reset() { } } +func (q *EventQueue) retireEventTopicsLocked(event Event) { + if event.Type != EventMessage { + return + } + pkg, ok := event.Data.(*relay.Package) + if !ok || pkg == nil { + return + } + for _, msg := range pkg.Messages { + if !messageHasTerminal(msg) { + continue + } + topic := msg.Topic + delete(q.messageTopics, topic) + } +} + +func (q *EventQueue) releaseEventPackageLocked(event Event) { + if event.Type != EventMessage { + return + } + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } +} + +func (q *EventQueue) clearMessageAccountingLocked() { + clear(q.messageTopics) +} + // YieldScheduler is the subset of Scheduler needed for waking. type YieldScheduler interface { WakeProcessor(q *EventQueue, gen uint64) diff --git a/api/process/queue_limits_test.go b/api/process/queue_limits_test.go new file mode 100644 index 000000000..b0e07faa4 --- /dev/null +++ b/api/process/queue_limits_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MPL-2.0 + +package process + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/relay" +) + +func boundedPackage(topic string, items int, bytes int64, value string) *relay.Package { + pkg := relay.NewPackage(pid.PID{}, pid.PID{}, topic, payload.NewString(value)) + pkg.Messages[0].MaxItems = items + pkg.Messages[0].MaxBytes = bytes + pkg.Messages[0].PayloadBytes = bytes + return pkg +} + +func TestEventQueueMessageAdmissionEmitsOneTerminal(t *testing.T) { + q := NewEventQueue() + gen := q.Generation() + + first := boundedPackage("cdc:a", 1, 100, "first") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: first}, gen)) + + second := boundedPackage("cdc:a", 1, 100, "second") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: second}, gen), "overflow terminal remains admissible") + require.Len(t, second.Messages, 1) + require.True(t, payload.IsTerminal(second.Messages[0].Payloads[len(second.Messages[0].Payloads)-1])) + + late := boundedPackage("cdc:a", 1, 100, "late") + require.Equal(t, MessageDropped, q.PushMessage(Event{Type: EventMessage, Data: late}, gen)) + require.Empty(t, late.Messages, "caller owns and releases a fully dropped package") + relay.ReleasePackage(late) + + events := q.Drain() + require.Len(t, events, 2) + for _, event := range events { + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } + + // A reset clears the overflow tombstone and permits a new stream + // incarnation to reuse the same topic. + q.Reset() + reuse := boundedPackage("cdc:a", 1, 100, "reuse") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: reuse}, q.Generation())) + for _, event := range q.Drain() { + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } +} + +func TestEventQueueMessageLimitsArePerTopic(t *testing.T) { + q := NewEventQueue() + gen := q.Generation() + + for _, topic := range []string{"cdc:a", "cdc:b"} { + first := boundedPackage(topic, 1, 0, "first") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: first}, gen)) + second := boundedPackage(topic, 1, 0, "second") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: second}, gen)) + } + + events := q.Drain() + require.Len(t, events, 4) + for _, event := range events { + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } +} diff --git a/api/process/queue_retention_test.go b/api/process/queue_retention_test.go new file mode 100644 index 000000000..1599164f4 --- /dev/null +++ b/api/process/queue_retention_test.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: MPL-2.0 + +package process + +import ( + "strconv" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/relay" +) + +type countingRetentionLease struct { + releases atomic.Int32 +} + +func (l *countingRetentionLease) Release() { + l.releases.Add(1) +} + +func TestEventQueuePushDirectClosedReleasesPackage(t *testing.T) { + q := NewEventQueue() + q.Close() + + lease := &countingRetentionLease{} + msg := relay.AcquireMessage() + msg.Topic = "closed" + msg.Payloads = payload.Payloads{payload.NewString("value")} + msg.SetRetentionLease(lease) + pkg := relay.NewMessagePackage(pid.PID{}, pid.PID{}, msg) + + require.False(t, q.PushDirect(Event{Type: EventMessage, Data: pkg})) + require.Equal(t, int32(1), lease.releases.Load(), "closed PushDirect must consume package ownership") +} + +func TestEventQueueRetentionSurvivesDrainAndTopicReuse(t *testing.T) { + q := NewEventQueue() + gen := q.Generation() + + first := boundedPackage("reuse", 1, 100, "first") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: first}, gen)) + second := boundedPackage("reuse", 1, 100, "second") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: second}, gen)) + + events := q.Drain() + require.Len(t, events, 2) + firstPkg, ok := events[0].Data.(*relay.Package) + require.True(t, ok) + firstLease := firstPkg.Messages[0].TakeRetentionLease() + require.NotNil(t, firstLease) + require.Len(t, q.messageTopics, 0, "terminal drain retires the topic state") + + // Release the terminal package but keep the first data reservation alive. + secondPkg, ok := events[1].Data.(*relay.Package) + require.True(t, ok) + relay.ReleasePackage(secondPkg) + relay.ReleasePackage(firstPkg) + + // The new stream gets a new topic state. The old lease must not debit its + // counters when it is released after the replacement has admitted data. + replacement := boundedPackage("reuse", 1, 100, "replacement") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: replacement}, q.Generation())) + overflow := boundedPackage("reuse", 1, 100, "overflow") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: overflow}, q.Generation())) + require.Len(t, overflow.Messages, 1) + require.True(t, payload.IsTerminal(overflow.Messages[0].Payloads[len(overflow.Messages[0].Payloads)-1])) + + firstLease.Release() + for _, event := range q.Drain() { + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } +} + +func TestEventQueueRetiresUniqueTopicState(t *testing.T) { + q := NewEventQueue() + for i := 0; i < 256; i++ { + topic := "churn:" + strconv.Itoa(i) + data := boundedPackage(topic, 1, 32, "data") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: data}, q.Generation())) + terminal := relay.NewPackage(pid.PID{}, pid.PID{}, topic, payload.NewTerminal()) + terminal.Messages[0].MaxItems = 1 + terminal.Messages[0].MaxBytes = 32 + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: terminal}, q.Generation())) + for _, event := range q.Drain() { + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } + require.Empty(t, q.messageTopics, "terminal topic state must not grow with churn") + } +} + +func TestEventQueueDrainClearsPreviousData(t *testing.T) { + q := NewEventQueue() + gen := q.Generation() + require.True(t, q.Push(Event{Data: &struct{ Value string }{"stale"}}, gen)) + events := q.Drain() + require.Len(t, events, 1) + require.NotNil(t, events[0].Data) + + require.Nil(t, q.Drain()) + require.Nil(t, events[0].Data, "queue must clear the reusable drain buffer before reuse") +} + +func TestEventQueueCloseReleasesQueuedPackageOnce(t *testing.T) { + q := NewEventQueue() + lease := &countingRetentionLease{} + msg := relay.AcquireMessage() + msg.Topic = "close" + msg.Payloads = payload.Payloads{payload.NewString("value")} + msg.SetRetentionLease(lease) + pkg := relay.NewMessagePackage(pid.PID{}, pid.PID{}, msg) + require.True(t, q.PushDirect(Event{Type: EventMessage, Data: pkg})) + + q.Close() + q.Close() + require.Equal(t, int32(1), lease.releases.Load(), "repeated close must not release a queued package twice") +} + +func TestEventQueueConcurrentDirectCloseOwnsEachPackage(t *testing.T) { + q := NewEventQueue() + const n = 128 + leases := make([]*countingRetentionLease, n) + var wg sync.WaitGroup + for i := 0; i < n; i++ { + lease := &countingRetentionLease{} + leases[i] = lease + msg := relay.AcquireMessage() + msg.Topic = "race" + msg.Payloads = payload.Payloads{payload.NewString("value")} + msg.SetRetentionLease(lease) + pkg := relay.NewMessagePackage(pid.PID{}, pid.PID{}, msg) + wg.Add(1) + go func(pkg *relay.Package) { + defer wg.Done() + q.PushDirect(Event{Type: EventMessage, Data: pkg}) + }(pkg) + } + q.Close() + wg.Wait() + + for _, lease := range leases { + require.Equal(t, int32(1), lease.releases.Load()) + } + for _, event := range q.Drain() { + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } +} diff --git a/api/relay/pool.go b/api/relay/pool.go index 2ea0c2a4d..af6c1f17c 100644 --- a/api/relay/pool.go +++ b/api/relay/pool.go @@ -38,8 +38,18 @@ func ReleaseMessage(m *Message) { if m == nil { return } + // A package may be released by the scheduler before its consumer takes + // ownership of a bounded-retention reservation. Consume that handoff here + // so the reservation is released exactly once and pooled messages never + // retain a callback into a retired queue. + if lease := m.TakeRetentionLease(); lease != nil { + lease.Release() + } m.Topic = "" m.Payloads = nil + m.PayloadBytes = 0 + m.MaxBytes = 0 + m.MaxItems = 0 messagePool.Put(m) } diff --git a/api/relay/relay.go b/api/relay/relay.go index 67d5e2594..41d489527 100644 --- a/api/relay/relay.go +++ b/api/relay/relay.go @@ -5,6 +5,7 @@ package relay import ( "context" + "sync/atomic" "github.com/wippyai/runtime/api/event" "github.com/wippyai/runtime/api/payload" @@ -35,10 +36,41 @@ type ( // Topic represents a message channel identifier. Topic = string + // RetentionLease represents ownership of a bounded message-retention + // reservation. The producer of a bounded message attaches a lease before + // handing the message to a queue. A consumer takes the lease when it + // transfers the message into its own mailbox and releases it when the + // message is delivered or discarded. + // + // The interface deliberately lives in relay rather than process: relay + // packages can cross scheduler and internode boundaries without depending + // on any particular consumer implementation. + RetentionLease interface { + Release() + } + + messageRetentionLease struct { + lease RetentionLease + } + // Message represents a single message with topic and payload. Message struct { - Topic Topic - Payloads payload.Payloads + // retention is an atomic ownership handoff for the reservation charged + // by a bounded destination. It is intentionally not serialized: leases + // are local to a process handoff and must never cross the wire. + retention atomic.Pointer[messageRetentionLease] + Topic Topic + Payloads payload.Payloads + // PayloadBytes is the logical retained size of Payloads. It is + // optional metadata used by bounded subscribers; zero preserves the + // historical unbounded relay behavior. + PayloadBytes int64 + // MaxBytes is the per-destination backlog limit for this message's + // topic. Zero means that the destination applies no byte limit. + MaxBytes int64 + // MaxItems is the per-destination message backlog limit for this + // topic. Zero means that the destination applies no item limit. + MaxItems int } // Package combines source, target and messages for delivery. @@ -56,12 +88,54 @@ type ( } ) +// SetRetentionLease attaches a bounded-retention ownership token to m. +// +// A message has at most one lease. Replacing an existing lease releases the +// old token first, which keeps pooled messages leak-free even when a caller +// accidentally reuses a message that was already admitted elsewhere. +func (m *Message) SetRetentionLease(lease RetentionLease) { + if m == nil || lease == nil { + return + } + old := m.retention.Swap(&messageRetentionLease{lease: lease}) + if old != nil && old.lease != nil { + old.lease.Release() + } +} + +// TakeRetentionLease transfers the message's reservation to its consumer. +// It is safe for a concurrent package release; exactly one caller receives +// the token and the other observes nil. +func (m *Message) TakeRetentionLease() RetentionLease { + if m == nil { + return nil + } + entry := m.retention.Swap(nil) + if entry == nil { + return nil + } + return entry.lease +} + type ( // Receiver defines the interface for message delivery. Receiver interface { Send(*Package) error } + // ContextSender is the cancellable delivery capability. Implementations + // must stop waiting for delivery when ctx is canceled. It is optional so + // existing receivers keep the original Send contract; lifecycle-sensitive + // dispatchers can require this capability instead of detaching a blocked + // Send goroutine. + // + // Ownership is transactional: a nil error transfers the package to the + // receiver (or its accepted queue); a non-nil error means the receiver did + // not retain it and the caller must release it exactly once. + ContextSender interface { + SendContext(context.Context, *Package) error + } + // AttachableReceiver extends Receiver with channel-based message delivery. AttachableReceiver interface { Receiver diff --git a/api/service/cdc/command.go b/api/service/cdc/command.go index 23e964a25..3b1ec3f91 100644 --- a/api/service/cdc/command.go +++ b/api/service/cdc/command.go @@ -7,6 +7,7 @@ import ( "github.com/wippyai/runtime/api/dispatcher" "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/registry" ) func init() { @@ -18,22 +19,36 @@ const ( ) type StreamOptions struct { + // After is an opaque source cursor. A driver that cannot resume from a + // cursor must return ErrUnsupported rather than silently ignore it. + After string Tables []string Ops []string - Buffer int + // MaxBytes bounds the retained logical size of one subscriber backlog. + // Zero selects DefaultMaxStreamBytes; negative values are invalid. + MaxBytes int64 + Buffer int + Snapshot bool } type Change struct { - Before map[string]any `json:"before,omitempty"` - After map[string]any `json:"after,omitempty"` - Source string `json:"source"` - Op string `json:"op"` - Schema string `json:"schema"` - Table string `json:"table"` - Relation string `json:"relation"` - LSN string `json:"lsn"` - CommitLSN string `json:"commit_lsn,omitempty"` - XID uint32 `json:"xid,omitempty"` + Before map[string]any `json:"before,omitempty"` + After map[string]any `json:"after,omitempty"` + Source string `json:"source"` + // SourceID is the canonical registry identity. Source is retained as the + // legacy wire representation used by existing Lua consumers. + SourceID registry.ID `json:"source_id,omitempty"` + Op string `json:"op"` + Schema string `json:"schema"` + Table string `json:"table"` + Relation string `json:"relation"` + LSN string `json:"lsn"` + CommitLSN string `json:"commit_lsn,omitempty"` + Cursor string `json:"cursor,omitempty"` + Generation string `json:"generation,omitempty"` + Transaction string `json:"transaction,omitempty"` + Error string `json:"error,omitempty"` + XID uint32 `json:"xid,omitempty"` } type SubscribeCmd struct { diff --git a/api/service/cdc/composite.go b/api/service/cdc/composite.go new file mode 100644 index 000000000..a5f6adb6c --- /dev/null +++ b/api/service/cdc/composite.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import "context" + +type Engine interface { + SourceInspector + SourceStreamer +} + +type composite struct { + engines []Engine +} + +func NewComposite(engines ...Engine) *composite { + return &composite{engines: engines} +} + +func (c *composite) List() []SourceInfo { + out := make([]SourceInfo, 0) + for _, e := range c.engines { + out = append(out, e.List()...) + } + return out +} + +func (c *composite) Get(name string) (SourceInfo, bool) { + for _, e := range c.engines { + if info, ok := e.Get(name); ok { + return info, true + } + } + return SourceInfo{}, false +} + +func (c *composite) Stream(ctx context.Context, name string, opts StreamOptions) (ChangeStream, SourceInfo, error) { + for _, e := range c.engines { + if _, ok := e.Get(name); ok { + return e.Stream(ctx, name, opts) + } + } + return nil, SourceInfo{}, ErrSourceNotFound +} diff --git a/api/service/cdc/composite_test.go b/api/service/cdc/composite_test.go new file mode 100644 index 000000000..b06a4997e --- /dev/null +++ b/api/service/cdc/composite_test.go @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeStream struct{ ch chan Change } + +func (f *fakeStream) Changes() <-chan Change { return f.ch } +func (f *fakeStream) Close() {} + +type fakeEngine struct { + infos map[string]SourceInfo + opened []string +} + +func (e *fakeEngine) List() []SourceInfo { + out := make([]SourceInfo, 0, len(e.infos)) + for _, i := range e.infos { + out = append(out, i) + } + return out +} + +func (e *fakeEngine) Get(name string) (SourceInfo, bool) { + i, ok := e.infos[name] + return i, ok +} + +func (e *fakeEngine) Stream(_ context.Context, name string, _ StreamOptions) (ChangeStream, SourceInfo, error) { + e.opened = append(e.opened, name) + return &fakeStream{ch: make(chan Change)}, e.infos[name], nil +} + +func TestCompositeListAggregates(t *testing.T) { + a := &fakeEngine{infos: map[string]SourceInfo{"pg": {Name: "pg", Engine: "postgres"}}} + b := &fakeEngine{infos: map[string]SourceInfo{"lite": {Name: "lite", Engine: "sqlite"}}} + c := NewComposite(a, b) + + infos := c.List() + assert.Len(t, infos, 2) +} + +func TestCompositeGetAndStreamRouting(t *testing.T) { + a := &fakeEngine{infos: map[string]SourceInfo{"pg": {Name: "pg", Engine: "postgres"}}} + b := &fakeEngine{infos: map[string]SourceInfo{"lite": {Name: "lite", Engine: "sqlite"}}} + c := NewComposite(a, b) + + info, ok := c.Get("lite") + require.True(t, ok) + assert.Equal(t, "sqlite", info.Engine) + + _, _, err := c.Stream(context.Background(), "lite", StreamOptions{}) + require.NoError(t, err) + assert.Equal(t, []string{"lite"}, b.opened) + assert.Empty(t, a.opened) +} + +func TestCompositeStreamNotFound(t *testing.T) { + c := NewComposite(&fakeEngine{infos: map[string]SourceInfo{}}) + _, _, err := c.Stream(context.Background(), "missing", StreamOptions{}) + assert.ErrorIs(t, err, ErrSourceNotFound) +} diff --git a/api/service/cdc/config.go b/api/service/cdc/config.go index 51b59e33a..342f4b819 100644 --- a/api/service/cdc/config.go +++ b/api/service/cdc/config.go @@ -11,6 +11,7 @@ import ( const ( Postgres registry.Kind = "db.cdc.postgres" + SQLite registry.Kind = "db.cdc.sqlite" ) const ( @@ -18,26 +19,37 @@ const ( ProtocolVersion = 1 StreamingProtocolVersion = 2 + + // These defaults bound decoder memory when the corresponding entry fields + // are omitted. Zero in Config means "use this default", never unlimited. + DefaultPostgresMaxTransactionChanges = 1_000_000 + DefaultPostgresMaxTransactionBytes = 256 << 20 + DefaultPostgresMaxInflightChanges = 1_000_000 + DefaultPostgresMaxInflightBytes = 256 << 20 ) type Config struct { - Options map[string]string `json:"options"` - Database string `json:"database"` - Password string `json:"password"` - Host string `json:"host"` - Username string `json:"username"` - SlotName string `json:"slot_name"` - Publication string `json:"publication,omitempty"` - StandbyInterval string `json:"standby_interval,omitempty"` - StatusInterval string `json:"status_interval,omitempty"` - Tables []string `json:"tables,omitempty"` - Lifecycle supervisor.LifecycleConfig `json:"lifecycle"` - Port int `json:"port"` - SnapshotFetchSize int `json:"snapshot_fetch_size,omitempty"` - Temporary bool `json:"temporary,omitempty"` - Snapshot bool `json:"snapshot,omitempty"` - Streaming bool `json:"streaming,omitempty"` - Failover bool `json:"failover,omitempty"` + Options map[string]string `json:"options"` + Database string `json:"database"` + Password string `json:"password"` + Host string `json:"host"` + Username string `json:"username"` + SlotName string `json:"slot_name"` + Publication string `json:"publication,omitempty"` + StandbyInterval string `json:"standby_interval,omitempty"` + StatusInterval string `json:"status_interval,omitempty"` + Tables []string `json:"tables,omitempty"` + Lifecycle supervisor.LifecycleConfig `json:"lifecycle"` + Port int `json:"port"` + SnapshotFetchSize int `json:"snapshot_fetch_size,omitempty"` + MaxTransactionChanges int `json:"max_transaction_changes,omitempty"` + MaxTransactionBytes int64 `json:"max_transaction_bytes,omitempty"` + MaxInflightChanges int `json:"max_inflight_changes,omitempty"` + MaxInflightBytes int64 `json:"max_inflight_bytes,omitempty"` + Temporary bool `json:"temporary,omitempty"` + Snapshot bool `json:"snapshot,omitempty"` + Streaming bool `json:"streaming,omitempty"` + Failover bool `json:"failover,omitempty"` } func (c *Config) InitDefaults() { @@ -75,6 +87,18 @@ func (c *Config) Validate() error { if c.SnapshotFetchSize < 0 { return ErrInvalidSnapshotFetchSize } + if c.MaxTransactionChanges < 0 { + return ErrInvalidMaxTransactionChanges + } + if c.MaxTransactionBytes < 0 { + return ErrInvalidMaxTransactionBytes + } + if c.MaxInflightChanges < 0 { + return ErrInvalidMaxInflightChanges + } + if c.MaxInflightBytes < 0 { + return ErrInvalidMaxInflightBytes + } if _, err := c.StandbyDuration(); err != nil { return err } @@ -84,6 +108,34 @@ func (c *Config) Validate() error { return nil } +func (c *Config) EffectiveMaxTransactionChanges() int { + if c.MaxTransactionChanges > 0 { + return c.MaxTransactionChanges + } + return DefaultPostgresMaxTransactionChanges +} + +func (c *Config) EffectiveMaxTransactionBytes() int64 { + if c.MaxTransactionBytes > 0 { + return c.MaxTransactionBytes + } + return DefaultPostgresMaxTransactionBytes +} + +func (c *Config) EffectiveMaxInflightChanges() int { + if c.MaxInflightChanges > 0 { + return c.MaxInflightChanges + } + return DefaultPostgresMaxInflightChanges +} + +func (c *Config) EffectiveMaxInflightBytes() int64 { + if c.MaxInflightBytes > 0 { + return c.MaxInflightBytes + } + return DefaultPostgresMaxInflightBytes +} + func (c *Config) StandbyDuration() (time.Duration, error) { return parseInterval(c.StandbyInterval) } diff --git a/api/service/cdc/config_sqlite.go b/api/service/cdc/config_sqlite.go new file mode 100644 index 000000000..928192aab --- /dev/null +++ b/api/service/cdc/config_sqlite.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "time" + + "github.com/wippyai/runtime/api/supervisor" +) + +type SQLiteConfig struct { + DBResource string `json:"db_resource"` + Name string `json:"name,omitempty"` + StatusInterval string `json:"status_interval,omitempty"` + Tables []string `json:"tables,omitempty"` + Lifecycle supervisor.LifecycleConfig `json:"lifecycle"` + Snapshot bool `json:"snapshot,omitempty"` +} + +func (c *SQLiteConfig) InitDefaults() { + c.Lifecycle.InitDefaults() +} + +func (c *SQLiteConfig) Validate() error { + if c.DBResource == "" { + return ErrDBResourceRequired + } + if _, err := c.StatusDuration(); err != nil { + return err + } + return nil +} + +func (c *SQLiteConfig) StatusDuration() (time.Duration, error) { + return parseInterval(c.StatusInterval) +} diff --git a/api/service/cdc/config_sqlite_test.go b/api/service/cdc/config_sqlite_test.go new file mode 100644 index 000000000..b49f6a292 --- /dev/null +++ b/api/service/cdc/config_sqlite_test.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSQLiteConfigValidate(t *testing.T) { + missing := &SQLiteConfig{} + assert.ErrorIs(t, missing.Validate(), ErrDBResourceRequired) + + badInterval := &SQLiteConfig{DBResource: "app:db", StatusInterval: "nope"} + assert.ErrorIs(t, badInterval.Validate(), ErrInvalidInterval) + + negative := &SQLiteConfig{DBResource: "app:db", StatusInterval: "-5s"} + assert.ErrorIs(t, negative.Validate(), ErrInvalidInterval) + + ok := &SQLiteConfig{DBResource: "app:db", StatusInterval: "5s", Tables: []string{"users"}, Snapshot: true} + require.NoError(t, ok.Validate()) + + d, err := ok.StatusDuration() + require.NoError(t, err) + assert.Equal(t, "5s", d.String()) +} + +func TestSQLiteConfigZeroInterval(t *testing.T) { + cfg := &SQLiteConfig{DBResource: "app:db"} + d, err := cfg.StatusDuration() + require.NoError(t, err) + assert.Equal(t, int64(0), int64(d)) +} diff --git a/api/service/cdc/config_test.go b/api/service/cdc/config_test.go index b0454bfef..a266ce839 100644 --- a/api/service/cdc/config_test.go +++ b/api/service/cdc/config_test.go @@ -122,6 +122,42 @@ func TestConfigSnapshotFetchSizeRejectsNegative(t *testing.T) { require.ErrorIs(t, c.Validate(), ErrInvalidSnapshotFetchSize) } +func TestConfigTransactionLimitsUseFiniteDefaults(t *testing.T) { + c := validConfig() + assert.Equal(t, DefaultPostgresMaxTransactionChanges, c.EffectiveMaxTransactionChanges()) + assert.Equal(t, int64(DefaultPostgresMaxTransactionBytes), c.EffectiveMaxTransactionBytes()) + assert.Equal(t, DefaultPostgresMaxInflightChanges, c.EffectiveMaxInflightChanges()) + assert.Equal(t, int64(DefaultPostgresMaxInflightBytes), c.EffectiveMaxInflightBytes()) + + c.MaxTransactionChanges = 123 + c.MaxTransactionBytes = 456 + c.MaxInflightChanges = 789 + c.MaxInflightBytes = 101112 + assert.Equal(t, 123, c.EffectiveMaxTransactionChanges()) + assert.Equal(t, int64(456), c.EffectiveMaxTransactionBytes()) + assert.Equal(t, 789, c.EffectiveMaxInflightChanges()) + assert.Equal(t, int64(101112), c.EffectiveMaxInflightBytes()) + require.NoError(t, c.Validate()) +} + +func TestConfigTransactionLimitsRejectNegative(t *testing.T) { + c := validConfig() + c.MaxTransactionChanges = -1 + require.ErrorIs(t, c.Validate(), ErrInvalidMaxTransactionChanges) + + c = validConfig() + c.MaxTransactionBytes = -1 + require.ErrorIs(t, c.Validate(), ErrInvalidMaxTransactionBytes) + + c = validConfig() + c.MaxInflightChanges = -1 + require.ErrorIs(t, c.Validate(), ErrInvalidMaxInflightChanges) + + c = validConfig() + c.MaxInflightBytes = -1 + require.ErrorIs(t, c.Validate(), ErrInvalidMaxInflightBytes) +} + func TestConfigFailoverRequiresPersistentSlot(t *testing.T) { c := validConfig() c.Failover = true diff --git a/api/service/cdc/context.go b/api/service/cdc/context.go index cc0f98df0..259c2f632 100644 --- a/api/service/cdc/context.go +++ b/api/service/cdc/context.go @@ -2,17 +2,95 @@ package cdc -import "context" +import ( + "context" + + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/registry" +) + +// SourceState is the driver-neutral lifecycle state exposed by a CDC source. +// Driver-specific health details remain in SourceInfo.Error and the legacy +// compatibility fields below. +type SourceState string + +const ( + SourceStateUnknown SourceState = "unknown" + SourceStateStarting SourceState = "starting" + SourceStateRunning SourceState = "running" + SourceStateFaulted SourceState = "faulted" + SourceStateStopped SourceState = "stopped" +) + +// Capabilities describes guarantees provided by a source. The common API does +// not infer PostgreSQL or SQLite semantics from the source kind. +type Capabilities struct { + Snapshot bool `json:"snapshot,omitempty"` + Durable bool `json:"durable,omitempty"` + Replayable bool `json:"replayable,omitempty"` + CapturesExternalWrites bool `json:"captures_external_writes,omitempty"` + BeforeImages bool `json:"before_images,omitempty"` + Coalesced bool `json:"coalesced,omitempty"` +} + +// Stream is the driver-neutral event stream. Err reports the terminal cause +// after Changes is closed; a normal caller-initiated Close has a nil error. +// Keeping the terminal state on the stream avoids synthesizing an +// error-valued change row and gives every registry-backed source the same +// failure contract. +type Stream interface { + Changes() <-chan Change + Close() + Err() error +} + +// ErrStream is retained as a deprecated compatibility alias. Stream now +// requires Err directly; ChangeStream below remains the legacy stream shape +// used by pre-registry SourceStreamer implementations. +type ErrStream = Stream + +// Source is the common source contract implemented by every CDC driver. +// Subscribe receives a context so a source can bind snapshot work to the +// caller's lifetime. A source that supports startup snapshot handoff may +// accept subscriptions while it is idle or starting; sources that cannot +// establish that handoff return ErrSourceNotReady until they are running. +type Source interface { + Info() SourceInfo + Subscribe(context.Context, StreamOptions) (Stream, error) +} + +// Registry is the read-only system-level CDC registry exposed to services and +// runtimes. Registry IDs, rather than driver aliases such as PostgreSQL slots, +// are the only global identity. +type Registry interface { + List() []SourceInfo + Get(registry.ID) (Source, bool) +} type SourceInfo struct { - Name string `json:"name"` - Slot string `json:"slot"` - Publication string `json:"publication,omitempty"` - Tables []string `json:"tables,omitempty"` - Streaming bool `json:"streaming,omitempty"` - Failover bool `json:"failover,omitempty"` - Temporary bool `json:"temporary,omitempty"` - Snapshot bool `json:"snapshot,omitempty"` + // The fields in this struct are retained for wire compatibility with + // existing Lua and API consumers. New code must use ID, Kind, State, + // Capabilities and Generation; driver-specific metadata should not be + // added to the common contract. + ID registry.ID `json:"id,omitempty"` + Engine string `json:"engine,omitempty"` + Epoch string `json:"epoch,omitempty"` + Error string `json:"error,omitempty"` + Generation string `json:"generation,omitempty"` + Name string `json:"name"` + Slot string `json:"slot"` + Publication string `json:"publication,omitempty"` + Kind registry.Kind `json:"kind,omitempty"` + State SourceState `json:"state,omitempty"` + File string `json:"file,omitempty"` + DBResource string `json:"db_resource,omitempty"` + Tables []string `json:"tables,omitempty"` + Capabilities Capabilities `json:"capabilities,omitempty"` + Streaming bool `json:"streaming,omitempty"` + Failover bool `json:"failover,omitempty"` + Temporary bool `json:"temporary,omitempty"` + Snapshot bool `json:"snapshot,omitempty"` + Faulted bool `json:"faulted,omitempty"` } type SourceInspector interface { @@ -46,3 +124,32 @@ func GetSourceStreamer(ctx context.Context) SourceStreamer { v, _ := ctx.Value(sourceStreamerKey{}).(SourceStreamer) return v } + +var registryKey = &ctxapi.Key{Name: "cdc.registry"} + +// WithRegistry attaches the driver-neutral CDC registry to the application +// context. Like the network and resource APIs, this is a write-once boot +// dependency and is safe to read after the application context is sealed. +func WithRegistry(ctx context.Context, registry Registry) context.Context { + if registry == nil { + return ctx + } + ac := ctxapi.AppFromContext(ctx) + if ac == nil { + return ctx + } + if ac.Get(registryKey) == nil { + ac.With(registryKey, registry) + } + return ctx +} + +// GetRegistry retrieves the system CDC registry from the application context. +func GetRegistry(ctx context.Context) Registry { + ac := ctxapi.AppFromContext(ctx) + if ac == nil { + return nil + } + registry, _ := ac.Get(registryKey).(Registry) + return registry +} diff --git a/api/service/cdc/context_test.go b/api/service/cdc/context_test.go index 933ae8da8..4f9b5d0a0 100644 --- a/api/service/cdc/context_test.go +++ b/api/service/cdc/context_test.go @@ -7,16 +7,21 @@ import ( "testing" "github.com/stretchr/testify/assert" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/registry" ) type stubInspector struct{} type stubStreamer struct{} +type stubRegistry struct{} func (stubInspector) List() []SourceInfo { return nil } func (stubInspector) Get(string) (SourceInfo, bool) { return SourceInfo{}, false } func (stubStreamer) Stream(context.Context, string, StreamOptions) (ChangeStream, SourceInfo, error) { return nil, SourceInfo{}, nil } +func (stubRegistry) List() []SourceInfo { return nil } +func (stubRegistry) Get(registry.ID) (Source, bool) { return nil, false } func TestWithSourceInspectorRoundTrip(t *testing.T) { ctx := WithSourceInspector(context.Background(), stubInspector{}) @@ -47,3 +52,22 @@ func TestWithSourceStreamerNilDoesNotAttach(t *testing.T) { func TestGetSourceStreamerEmptyCtx(t *testing.T) { assert.Nil(t, GetSourceStreamer(context.Background())) } + +func TestWithRegistryRoundTripOnApplicationContext(t *testing.T) { + ctx := ctxapi.NewRootContext() + reg := stubRegistry{} + ctx = WithRegistry(ctx, reg) + assert.Equal(t, reg, GetRegistry(ctx)) +} + +func TestWithRegistryDoesNotAttachToPlainContext(t *testing.T) { + reg := stubRegistry{} + ctx := WithRegistry(context.Background(), reg) + assert.Nil(t, GetRegistry(ctx)) +} + +func TestWithRegistryNilDoesNotAttach(t *testing.T) { + ctx := ctxapi.NewRootContext() + ctx = WithRegistry(ctx, nil) + assert.Nil(t, GetRegistry(ctx)) +} diff --git a/api/service/cdc/errors.go b/api/service/cdc/errors.go index ffdeae95b..4c36ecfb4 100644 --- a/api/service/cdc/errors.go +++ b/api/service/cdc/errors.go @@ -5,14 +5,23 @@ package cdc import apierror "github.com/wippyai/runtime/api/error" var ( - ErrHostRequired = apierror.New(apierror.Invalid, "host is required").WithRetryable(apierror.False) - ErrInvalidPort = apierror.New(apierror.Invalid, "port must be greater than 0").WithRetryable(apierror.False) - ErrDatabaseRequired = apierror.New(apierror.Invalid, "database is required").WithRetryable(apierror.False) - ErrUsernameRequired = apierror.New(apierror.Invalid, "username is required").WithRetryable(apierror.False) - ErrPasswordRequired = apierror.New(apierror.Invalid, "password is required").WithRetryable(apierror.False) - ErrSlotNameRequired = apierror.New(apierror.Invalid, "slot_name is required").WithRetryable(apierror.False) - ErrPublicationRequired = apierror.New(apierror.Invalid, "publication or tables is required").WithRetryable(apierror.False) - ErrInvalidInterval = apierror.New(apierror.Invalid, "interval must be a non-negative duration (e.g. 10s)").WithRetryable(apierror.False) - ErrFailoverTemporary = apierror.New(apierror.Invalid, "failover cannot be set on a temporary slot").WithRetryable(apierror.False) - ErrInvalidSnapshotFetchSize = apierror.New(apierror.Invalid, "snapshot_fetch_size must be non-negative").WithRetryable(apierror.False) + ErrHostRequired = apierror.New(apierror.Invalid, "host is required").WithRetryable(apierror.False) + ErrInvalidPort = apierror.New(apierror.Invalid, "port must be greater than 0").WithRetryable(apierror.False) + ErrDatabaseRequired = apierror.New(apierror.Invalid, "database is required").WithRetryable(apierror.False) + ErrUsernameRequired = apierror.New(apierror.Invalid, "username is required").WithRetryable(apierror.False) + ErrPasswordRequired = apierror.New(apierror.Invalid, "password is required").WithRetryable(apierror.False) + ErrSlotNameRequired = apierror.New(apierror.Invalid, "slot_name is required").WithRetryable(apierror.False) + ErrPublicationRequired = apierror.New(apierror.Invalid, "publication or tables is required").WithRetryable(apierror.False) + ErrInvalidInterval = apierror.New(apierror.Invalid, "interval must be a non-negative duration (e.g. 10s)").WithRetryable(apierror.False) + ErrFailoverTemporary = apierror.New(apierror.Invalid, "failover cannot be set on a temporary slot").WithRetryable(apierror.False) + ErrInvalidSnapshotFetchSize = apierror.New(apierror.Invalid, "snapshot_fetch_size must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxTransactionChanges = apierror.New(apierror.Invalid, "max_transaction_changes must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxTransactionBytes = apierror.New(apierror.Invalid, "max_transaction_bytes must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxInflightChanges = apierror.New(apierror.Invalid, "max_inflight_changes must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxInflightBytes = apierror.New(apierror.Invalid, "max_inflight_bytes must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxBytes = apierror.New(apierror.Invalid, "stream max_bytes must be non-negative").WithRetryable(apierror.False) + ErrDBResourceRequired = apierror.New(apierror.Invalid, "db_resource is required").WithRetryable(apierror.False) + ErrSourceNotFound = apierror.New(apierror.NotFound, "cdc source not found").WithRetryable(apierror.False) + ErrUnsupported = apierror.New(apierror.Invalid, "cdc operation is not supported by this source").WithRetryable(apierror.False) + ErrSourceNotReady = apierror.New(apierror.Unavailable, "cdc source is not ready").WithRetryable(apierror.True) ) diff --git a/api/service/cdc/size.go b/api/service/cdc/size.go new file mode 100644 index 000000000..a66398ff4 --- /dev/null +++ b/api/service/cdc/size.go @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import "reflect" + +const ( + // DefaultMaxStreamBytes bounds a subscriber's retained event backlog when + // MaxBytes is omitted. It is deliberately finite for every driver. + DefaultMaxStreamBytes int64 = 64 << 20 + // DefaultMaxStreamItems is the historical Lua CDC stream capacity. It is + // also used by direct Go callers so process admission is always bounded. + DefaultMaxStreamItems = 64 + changeStructuralBytes = 128 + valueStructuralBytes = 24 + maxEstimateDepth = 256 + maxEstimateNodes = 1 << 20 +) + +// ValidateStreamOptions validates common stream resource limits. Buffer keeps +// its historical clamping behavior; MaxBytes is the only option with a +// rejected negative value. +func (o StreamOptions) Validate() error { + if o.MaxBytes < 0 { + return ErrInvalidMaxBytes + } + return nil +} + +// EffectiveMaxBytes returns the finite subscriber backlog limit selected by +// the options. Zero means the safe common default. +func (o StreamOptions) EffectiveMaxBytes() int64 { + if o.MaxBytes > 0 { + return o.MaxBytes + } + return DefaultMaxStreamBytes +} + +// EffectiveMaxStreamItems returns the finite item limit selected by options. +func (o StreamOptions) EffectiveMaxStreamItems() int { + if o.Buffer > 0 { + return o.Buffer + } + return DefaultMaxStreamItems +} + +// EstimateChangeBytes returns a conservative logical retained-size estimate +// for a Change and all nested values in its before/after images. It counts +// strings and byte blobs by length, includes container structure, saturates +// at MaxInt64, and terminates on cyclic pointers/maps/slices. +// +// The estimate is intentionally driver-neutral: both SQLite and PostgreSQL +// use this exact function for their sole subscriber backlog. +func EstimateChangeBytes(change Change) int64 { + e := sizeEstimator{seen: make(map[sizeVisit]struct{})} + return e.add(reflect.ValueOf(change), changeStructuralBytes) +} + +type sizeVisit struct { + typ reflect.Type + kind reflect.Kind + ptr uintptr + len int + cap int +} + +type sizeEstimator struct { + seen map[sizeVisit]struct{} + nodes int +} + +func (e *sizeEstimator) add(value reflect.Value, total int64) int64 { + return e.addDepth(value, total, 0) +} + +func (e *sizeEstimator) addDepth(value reflect.Value, total int64, depth int) int64 { + if total >= maxInt64Value { + return maxInt64Value + } + if !value.IsValid() { + return total + } + if depth > maxEstimateDepth || e.nodes >= maxEstimateNodes { + return maxInt64Value + } + e.nodes++ + + switch value.Kind() { + case reflect.Interface: + if value.IsNil() { + return total + } + return e.addDepth(value.Elem(), total, depth+1) + case reflect.Pointer: + if value.IsNil() { + return total + } + if e.visited(value, 0, 0) { + return total + } + return e.addDepth(value.Elem(), satAdd(total, valueStructuralBytes), depth+1) + case reflect.Map: + if value.IsNil() { + return total + } + if e.visited(value, 0, 0) { + return total + } + total = satAdd(total, satMul(valueStructuralBytes, int64(value.Len()))) + iter := value.MapRange() + for iter.Next() { + total = e.addDepth(iter.Key(), total, depth+1) + total = e.addDepth(iter.Value(), total, depth+1) + } + return total + case reflect.Slice: + if value.IsNil() { + return total + } + if value.Type().Elem().Kind() == reflect.Uint8 { + return satAdd(total, int64(value.Len())) + } + if e.visited(value, value.Len(), value.Cap()) { + return total + } + total = satAdd(total, satMul(valueStructuralBytes, int64(value.Len()))) + for i := 0; i < value.Len(); i++ { + total = e.addDepth(value.Index(i), total, depth+1) + } + return total + case reflect.Array: + total = satAdd(total, satMul(valueStructuralBytes, int64(value.Len()))) + for i := 0; i < value.Len(); i++ { + total = e.addDepth(value.Index(i), total, depth+1) + } + return total + case reflect.Struct: + total = satAdd(total, typeSize(value.Type())) + for i := 0; i < value.NumField(); i++ { + total = e.addDepth(value.Field(i), total, depth+1) + } + return total + case reflect.String: + return satAdd(total, int64(value.Len())) + case reflect.Chan, reflect.Func, reflect.UnsafePointer: + return satAdd(total, typeSize(value.Type())) + default: + return satAdd(total, typeSize(value.Type())) + } +} + +func (e *sizeEstimator) visited(value reflect.Value, length, capacity int) bool { + ptr := value.Pointer() + if ptr == 0 { + return false + } + key := sizeVisit{typ: value.Type(), kind: value.Kind(), ptr: ptr, len: length, cap: capacity} + if _, exists := e.seen[key]; exists { + return true + } + e.seen[key] = struct{}{} + return false +} + +const maxInt64Value = int64(^uint64(0) >> 1) + +func typeSize(typ reflect.Type) int64 { + size := typ.Size() + if uint64(size) > uint64(maxInt64Value) { + return maxInt64Value + } + return int64(size) +} + +func satAdd(a, b int64) int64 { + if a >= maxInt64Value || b >= maxInt64Value-a { + return maxInt64Value + } + return a + b +} + +func satMul(a, b int64) int64 { + if a <= 0 || b <= 0 { + return 0 + } + if a > maxInt64Value/b { + return maxInt64Value + } + return a * b +} diff --git a/api/service/cdc/size_test.go b/api/service/cdc/size_test.go new file mode 100644 index 000000000..19be1e7cb --- /dev/null +++ b/api/service/cdc/size_test.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStreamOptionsMaxBytesDefaultsAndValidation(t *testing.T) { + var options StreamOptions + assert.Equal(t, DefaultMaxStreamBytes, options.EffectiveMaxBytes()) + assert.NoError(t, options.Validate()) + + options.MaxBytes = 1024 + assert.Equal(t, int64(1024), options.EffectiveMaxBytes()) + assert.NoError(t, options.Validate()) + + options.MaxBytes = -1 + assert.ErrorIs(t, options.Validate(), ErrInvalidMaxBytes) +} + +func TestEstimateChangeBytesCountsNestedBlobs(t *testing.T) { + change := Change{ + Source: "source", + Before: map[string]any{ + "name": "alice", + "blob": []byte{1, 2, 3, 4}, + "nested": map[string]any{"values": []any{"nested-value", []byte{5, 6}}}, + }, + } + base := EstimateChangeBytes(Change{}) + got := EstimateChangeBytes(change) + assert.Greater(t, got, base) + assert.GreaterOrEqual(t, got-base, int64(len("alice")+4+len("nested-value")+2)) +} + +func TestEstimateChangeBytesTerminatesCyclicValues(t *testing.T) { + cyclic := map[string]any{} + cyclic["self"] = cyclic + change := Change{After: cyclic} + + assert.NotPanics(t, func() { + assert.Positive(t, EstimateChangeBytes(change)) + }) +} + +func TestEstimateChangeBytesSaturates(t *testing.T) { + assert.Equal(t, maxInt64Value, satAdd(maxInt64Value-1, 2)) + assert.Equal(t, maxInt64Value, satMul(maxInt64Value, 2)) + assert.LessOrEqual(t, EstimateChangeBytes(Change{}), maxInt64Value) +} + +func TestEstimateChangeBytesBoundsDeepAndWideValues(t *testing.T) { + deep := map[string]any{} + current := deep + for i := 0; i < maxEstimateDepth+2; i++ { + next := map[string]any{} + current["next"] = next + current = next + } + assert.Equal(t, maxInt64Value, EstimateChangeBytes(Change{After: deep})) + + wide := make([]any, maxEstimateNodes+1) + for i := range wide { + wide[i] = "x" + } + assert.Equal(t, maxInt64Value, EstimateChangeBytes(Change{After: map[string]any{"wide": wide}})) +} diff --git a/api/service/sql/config.go b/api/service/sql/config.go index 40ed40511..6a7195197 100644 --- a/api/service/sql/config.go +++ b/api/service/sql/config.go @@ -34,8 +34,25 @@ const ( // DefaultMaxLifetime is the default maximum lifetime of a connection DefaultMaxLifetime = 1 * time.Hour + + // DefaultMaxMutationChanges bounds the in-memory candidate row count held + // by the SQLite observer for one transaction. + DefaultMaxMutationChanges = 100000 + // DefaultMaxMutationBytes is the conservative retained logical-byte bound + // for one SQLite transaction. SQLite delivers a complete native row to the + // pre-update hook, so one row can transiently materialize before the bound + // rejects the candidate. + DefaultMaxMutationBytes = 64 * 1024 * 1024 ) +// EngineConfig is the contract every engine configuration satisfies, letting the +// generic pool lifecycle validate and read lifecycle settings without knowing the +// concrete engine type. +type EngineConfig interface { + Validate() error + LifecycleConfig() supervisor.LifecycleConfig +} + type ( // PoolConfig defines settings for a database connection pool PoolConfig struct { @@ -58,10 +75,14 @@ type ( // SQLiteConfig defines SQLite-specific configuration SQLiteConfig struct { - Options map[string]string `json:"options"` - File string `json:"file"` - Lifecycle supervisor.LifecycleConfig `json:"lifecycle"` - Pool PoolConfig `json:"pool"` + Options map[string]string `json:"options"` + File string `json:"file"` + Lifecycle supervisor.LifecycleConfig `json:"lifecycle"` + Pool PoolConfig `json:"pool"` + MaxMutationChanges int `json:"max_mutation_changes,omitempty"` + // MaxMutationBytes is a conservative retained logical-byte bound. A + // native SQLite row is materialized before the observer can reject it. + MaxMutationBytes int `json:"max_mutation_bytes,omitempty"` } ) @@ -104,6 +125,17 @@ func (c *SQLiteConfig) InitDefaults() { // Initialize lifecycle defaults c.Lifecycle.InitDefaults() + if c.MaxMutationChanges == 0 { + c.MaxMutationChanges = DefaultMaxMutationChanges + } + if c.MaxMutationBytes == 0 { + c.MaxMutationBytes = DefaultMaxMutationBytes + } +} + +// LifecycleConfig returns the supervisor lifecycle settings for the database. +func (c *DBConfig) LifecycleConfig() supervisor.LifecycleConfig { + return c.Lifecycle } // Validate checks if the DBConfig has all required fields set to valid values @@ -143,6 +175,11 @@ func (c *DBConfig) Validate() error { return nil } +// LifecycleConfig returns the supervisor lifecycle settings for the database. +func (c *SQLiteConfig) LifecycleConfig() supervisor.LifecycleConfig { + return c.Lifecycle +} + // Validate checks if the SQLiteConfig has all required fields set to valid values func (c *SQLiteConfig) Validate() error { if c.File == "" { @@ -152,6 +189,12 @@ func (c *SQLiteConfig) Validate() error { if c.Pool.MaxLifetime <= 0 { return ErrInvalidMaxLifetime } + if c.MaxMutationChanges < 0 { + return ErrInvalidMaxMutationChanges + } + if c.MaxMutationBytes < 0 { + return ErrInvalidMaxMutationBytes + } return nil } diff --git a/api/service/sql/config_test.go b/api/service/sql/config_test.go index ef47c55bb..3b50d58de 100644 --- a/api/service/sql/config_test.go +++ b/api/service/sql/config_test.go @@ -285,9 +285,25 @@ func TestSQLiteConfig_InitDefaults(t *testing.T) { assert.Equal(t, DefaultMaxOpen, config.Pool.MaxOpen) assert.Equal(t, DefaultMaxIdle, config.Pool.MaxIdle) assert.Equal(t, DefaultMaxLifetime, config.Pool.MaxLifetime) + assert.Equal(t, DefaultMaxMutationChanges, config.MaxMutationChanges) + assert.Equal(t, DefaultMaxMutationBytes, config.MaxMutationBytes) assert.NotNil(t, config.Options) } +func TestSQLiteConfig_RejectsNegativeMutationLimits(t *testing.T) { + base := SQLiteConfig{File: ":memory:", Pool: PoolConfig{MaxLifetime: time.Hour}} + base.MaxMutationChanges = -1 + assert.ErrorIs(t, base.Validate(), ErrInvalidMaxMutationChanges) + base.MaxMutationChanges = 0 + base.MaxMutationBytes = -1 + assert.ErrorIs(t, base.Validate(), ErrInvalidMaxMutationBytes) + + base.MaxMutationChanges = -1 + base.MaxMutationBytes = 0 + base.InitDefaults() + assert.Equal(t, -1, base.MaxMutationChanges, "negative limits must not be silently defaulted") +} + func TestPoolConfig_UnmarshalJSON_InvalidDuration(t *testing.T) { jsonData := `{"max_lifetime":"invalid"}` var config PoolConfig diff --git a/api/service/sql/errors.go b/api/service/sql/errors.go index b8792bdfb..ca3b5101f 100644 --- a/api/service/sql/errors.go +++ b/api/service/sql/errors.go @@ -5,13 +5,15 @@ package sql import apierror "github.com/wippyai/runtime/api/error" var ( - ErrHostRequired = apierror.New(apierror.Invalid, "host is required").WithRetryable(apierror.False) - ErrInvalidPort = apierror.New(apierror.Invalid, "port must be greater than 0").WithRetryable(apierror.False) - ErrDatabaseRequired = apierror.New(apierror.Invalid, "database is required").WithRetryable(apierror.False) - ErrUsernameRequired = apierror.New(apierror.Invalid, "username is required").WithRetryable(apierror.False) - ErrPasswordRequired = apierror.New(apierror.Invalid, "password is required").WithRetryable(apierror.False) - ErrInvalidMaxOpen = apierror.New(apierror.Invalid, "max open connections must be non-negative").WithRetryable(apierror.False) - ErrInvalidMaxIdle = apierror.New(apierror.Invalid, "max idle connections must be non-negative").WithRetryable(apierror.False) - ErrInvalidMaxLifetime = apierror.New(apierror.Invalid, "max lifetime must be greater than 0").WithRetryable(apierror.False) - ErrFileRequired = apierror.New(apierror.Invalid, "file path is required").WithRetryable(apierror.False) + ErrHostRequired = apierror.New(apierror.Invalid, "host is required").WithRetryable(apierror.False) + ErrInvalidPort = apierror.New(apierror.Invalid, "port must be greater than 0").WithRetryable(apierror.False) + ErrDatabaseRequired = apierror.New(apierror.Invalid, "database is required").WithRetryable(apierror.False) + ErrUsernameRequired = apierror.New(apierror.Invalid, "username is required").WithRetryable(apierror.False) + ErrPasswordRequired = apierror.New(apierror.Invalid, "password is required").WithRetryable(apierror.False) + ErrInvalidMaxOpen = apierror.New(apierror.Invalid, "max open connections must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxIdle = apierror.New(apierror.Invalid, "max idle connections must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxLifetime = apierror.New(apierror.Invalid, "max lifetime must be greater than 0").WithRetryable(apierror.False) + ErrFileRequired = apierror.New(apierror.Invalid, "file path is required").WithRetryable(apierror.False) + ErrInvalidMaxMutationChanges = apierror.New(apierror.Invalid, "max mutation changes must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxMutationBytes = apierror.New(apierror.Invalid, "max mutation bytes must be non-negative").WithRetryable(apierror.False) ) diff --git a/api/service/sql/observer.go b/api/service/sql/observer.go new file mode 100644 index 000000000..4d75ecb87 --- /dev/null +++ b/api/service/sql/observer.go @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sql + +import "context" + +// CommittedMutationSource is an optional capability exposed by a SQL resource. +// It reports mutations observed by the database engine after a transaction +// boundary. The interface deliberately contains no driver-specific handles or +// SQL connection types; consumers can use the same capability with different +// database engines. +// +// A source is owned by the SQL resource generation that created it. Closing the +// generation closes the source and all of its streams. +type CommittedMutationSource interface { + Subscribe(context.Context, MutationOptions) (MutationStream, error) + // Snapshot establishes the commit fence before starting its read view. The + // returned stream emits snapshot batches first and then live committed + // batches after the returned watermark, without exposing a database handle + // to the consumer. + Snapshot(context.Context, SnapshotOptions) (SnapshotStream, error) + Close() error +} + +// MutationOptions controls an observation stream. Filtering is intentionally +// expressed in the neutral mutation vocabulary so a consumer does not need to +// know which SQL driver produced the stream. +type MutationOptions struct { + Tables []string + Operations []string + MaxChanges int + // MaxBytes bounds retained logical mutation bytes. A native SQLite row is + // materialized before the observer can reject it, so one row may + // transiently exceed this conservative bound. + MaxBytes int +} + +// SnapshotOptions selects tables and the maximum number of rows in one +// snapshot batch. A non-positive BatchSize uses the engine default. +type SnapshotOptions struct { + Tables []string + BatchSize int + MaxChanges int + // MaxBytes bounds retained logical snapshot/live bytes. A native SQLite + // row is materialized before the observer can reject it, so one row may + // transiently exceed this conservative bound. + MaxBytes int +} + +// MutationStream delivers committed mutation batches in commit order. +type MutationStream interface { + Changes() <-chan MutationBatch + Err() error + Close() error +} + +// SnapshotStream is an atomic snapshot/live handoff. Snapshot batches are +// marked with MutationBatch.Snapshot; once they are exhausted, the stream +// carries live batches in commit order. Watermark identifies the fence at +// which the read view was established and is process-local unless an engine +// documents a durable position. +type SnapshotStream interface { + MutationStream + Watermark() string +} + +// MutationBatch is the atomic unit emitted by an observation source. A batch +// belongs to one database transaction; an empty batch is not emitted. +type MutationBatch struct { + Transaction string + Changes []Mutation + Snapshot bool +} + +// Mutation is a driver-neutral row mutation. Values retain the database/sql +// driver's native scalar representation. Column names are captured with the +// mutation so schema changes after capture cannot relabel an earlier row. +type Mutation struct { + Schema string + Table string + Op string + Columns []string + Before []any + After []any + // OldRowID is the row identifier before the change. It is zero for an + // insert; RowID is the identifier after the change and is zero for a + // delete. Drivers that cannot provide a stable row identifier must fail + // closed rather than emit an ambiguous mutation. + OldRowID int64 + RowID int64 +} diff --git a/boot/components/dispatchers/cdc_dispatcher.go b/boot/components/dispatchers/cdc_dispatcher.go index 79467bc93..c4c0971e4 100644 --- a/boot/components/dispatchers/cdc_dispatcher.go +++ b/boot/components/dispatchers/cdc_dispatcher.go @@ -7,13 +7,13 @@ import ( "github.com/wippyai/runtime/api/boot" dispatcherapi "github.com/wippyai/runtime/api/dispatcher" - "github.com/wippyai/runtime/service/cdc/postgres" + "github.com/wippyai/runtime/service/cdc" ) const CDCDefaultWorkers = 4 func CDC() boot.Component { - var d *postgres.Dispatcher + var d *cdc.Dispatcher return boot.New(boot.P{ Name: CDCDispatcherName, @@ -24,7 +24,7 @@ func CDC() boot.Component { return ctx, ErrDispatcherNotFound } - d = postgres.NewDispatcher(postgres.WithWorkers(CDCDefaultWorkers)) + d = cdc.NewDispatcher(cdc.WithWorkers(CDCDefaultWorkers)) d.RegisterAll(reg.Register) return ctx, nil }, diff --git a/boot/components/service/storage/cdc.go b/boot/components/service/storage/cdc.go index 77c206962..19e7f83f9 100644 --- a/boot/components/service/storage/cdc.go +++ b/boot/components/service/storage/cdc.go @@ -4,35 +4,53 @@ package storage import ( "context" + "errors" "github.com/wippyai/runtime/api/boot" "github.com/wippyai/runtime/api/event" logapi "github.com/wippyai/runtime/api/logs" "github.com/wippyai/runtime/api/payload" + resourceapi "github.com/wippyai/runtime/api/resource" cdcapi "github.com/wippyai/runtime/api/service/cdc" bootpkg "github.com/wippyai/runtime/boot" bootsystem "github.com/wippyai/runtime/boot/components/system" - cdc "github.com/wippyai/runtime/service/cdc/postgres" + cdcservice "github.com/wippyai/runtime/service/cdc" + pgcdc "github.com/wippyai/runtime/service/cdc/postgres" + sqlitecdc "github.com/wippyai/runtime/service/cdc/sqlite" ) func CDC() boot.Component { return boot.New(boot.P{ Name: CDCName, - DependsOn: []boot.Name{bootsystem.EnvironmentName}, + DependsOn: []boot.Name{bootsystem.EnvironmentName, bootsystem.ResourcesName, bootsystem.CDCRegistryName}, Load: func(ctx context.Context) (context.Context, error) { logger := logapi.GetLogger(ctx) dtt := payload.GetTranscoder(ctx) bus := event.GetBus(ctx) + resReg := resourceapi.GetRegistry(ctx) handlers := bootpkg.GetHandlerRegistry(ctx) + if cdcapi.GetRegistry(ctx) == nil { + return ctx, NewCDCManagerError(errors.New("cdc system registry not available")) + } - manager, err := cdc.NewManager(dtt, bus, logger.Named("cdc")) + cdcRegistry, ok := cdcapi.GetRegistry(ctx).(cdcservice.Registry) + if !ok { + return ctx, NewCDCManagerError(errors.New("cdc system registry has an unsupported implementation")) + } + manager, err := cdcservice.NewManager( + cdcRegistry, + dtt, + bus, + resReg, + logger.Named("cdc"), + cdcservice.WithDriver(pgcdc.NewDriver(), sqlitecdc.NewDriver()), + ) if err != nil { return ctx, NewCDCManagerError(err) } - handlers.RegisterListener("db.cdc.*", manager) - ctx = cdcapi.WithSourceInspector(ctx, manager) - ctx = cdcapi.WithSourceStreamer(ctx, manager) + handlers.RegisterListener("db.cdc.postgres", manager) + handlers.RegisterListener("db.cdc.sqlite", manager) return ctx, nil }, }) diff --git a/boot/components/service/storage/sql.go b/boot/components/service/storage/sql.go index 70e078c5c..3bcc3ec30 100644 --- a/boot/components/service/storage/sql.go +++ b/boot/components/service/storage/sql.go @@ -6,12 +6,14 @@ import ( "context" "github.com/wippyai/runtime/api/boot" + envapi "github.com/wippyai/runtime/api/env" "github.com/wippyai/runtime/api/event" logapi "github.com/wippyai/runtime/api/logs" "github.com/wippyai/runtime/api/payload" bootpkg "github.com/wippyai/runtime/boot" bootsystem "github.com/wippyai/runtime/boot/components/system" "github.com/wippyai/runtime/service/sql" + "github.com/wippyai/runtime/service/sql/engine/all" ) func SQL() boot.Component { @@ -22,12 +24,15 @@ func SQL() boot.Component { logger := logapi.GetLogger(ctx) dtt := payload.GetTranscoder(ctx) bus := event.GetBus(ctx) + envRegistry := envapi.GetRegistry(ctx) handlers := bootpkg.GetHandlerRegistry(ctx) manager, err := sql.NewManager( dtt, bus, logger.Named("sql"), + envRegistry, + sql.WithDriver(all.Drivers()...), ) if err != nil { return ctx, NewSQLManagerError(err) diff --git a/boot/components/system/all.go b/boot/components/system/all.go index ed389fcd0..b5410cfc0 100644 --- a/boot/components/system/all.go +++ b/boot/components/system/all.go @@ -21,6 +21,7 @@ func All() []boot.Component { Network(), SocketDispatcher(), Resources(), + CDC(), Factory(), ProcessManager(), Interceptor(), diff --git a/boot/components/system/cdc.go b/boot/components/system/cdc.go new file mode 100644 index 000000000..31b84e5c2 --- /dev/null +++ b/boot/components/system/cdc.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: MPL-2.0 + +package system + +import ( + "context" + + "github.com/wippyai/runtime/api/boot" + logapi "github.com/wippyai/runtime/api/logs" + cdcapi "github.com/wippyai/runtime/api/service/cdc" + cdcsystem "github.com/wippyai/runtime/system/cdc" +) + +// CDC creates the driver-neutral source registry. Concrete drivers are +// injected by the service layer after this component has published the +// registry into the application context. +func CDC() boot.Component { + return boot.New(boot.P{ + Name: CDCRegistryName, + Load: func(ctx context.Context) (context.Context, error) { + logger := logapi.GetLogger(ctx) + return cdcapi.WithRegistry(ctx, cdcsystem.NewRegistry(logger.Named("cdc"))), nil + }, + }) +} diff --git a/boot/components/system/constants.go b/boot/components/system/constants.go index 3723d2d73..bd3bacc3a 100644 --- a/boot/components/system/constants.go +++ b/boot/components/system/constants.go @@ -10,6 +10,7 @@ const ( EnvironmentName boot.Name = "env" NetworkName boot.Name = "network" ResourcesName boot.Name = "resources" + CDCRegistryName boot.Name = "cdc.registry" InterceptorName boot.Name = "interceptor" FrameResolversName boot.Name = "frame_resolvers" FunctionsName boot.Name = "functions" diff --git a/cluster/internode/codec.go b/cluster/internode/codec.go index 6c43ce319..85868ee54 100644 --- a/cluster/internode/codec.go +++ b/cluster/internode/codec.go @@ -21,8 +21,11 @@ type encodedPayload struct { } type encodedMessage struct { - Topic string - Payloads []encodedPayload + Topic string + Payloads []encodedPayload + PayloadBytes int64 + MaxBytes int64 + MaxItems int } type encodedPackage struct { @@ -101,8 +104,11 @@ func (c *MessageCodec) Encode(pkg *relay.Package) ([]byte, error) { for i, msg := range pkg.Messages { encMsg := &encodedMessage{ - Topic: msg.Topic, - Payloads: make([]encodedPayload, len(msg.Payloads)), + Topic: msg.Topic, + Payloads: make([]encodedPayload, len(msg.Payloads)), + PayloadBytes: msg.PayloadBytes, + MaxBytes: msg.MaxBytes, + MaxItems: msg.MaxItems, } for j, p := range msg.Payloads { @@ -159,6 +165,9 @@ func (c *MessageCodec) Decode(data []byte) (*relay.Package, error) { for i, encMsg := range encPkg.Messages { finalMsg := relay.AcquireMessage() finalMsg.Topic = encMsg.Topic + finalMsg.PayloadBytes = encMsg.PayloadBytes + finalMsg.MaxBytes = encMsg.MaxBytes + finalMsg.MaxItems = encMsg.MaxItems finalMsg.Payloads = make(payload.Payloads, len(encMsg.Payloads)) for j, encP := range encMsg.Payloads { diff --git a/cluster/internode/codec_test.go b/cluster/internode/codec_test.go index 7a6d1a0aa..92211920d 100644 --- a/cluster/internode/codec_test.go +++ b/cluster/internode/codec_test.go @@ -85,7 +85,10 @@ func TestMessageCodec_PackagePIDs_SourceTarget(t *testing.T) { Target: targetPID, Messages: []*relay.Message{ { - Topic: "test.topic", + Topic: "test.topic", + PayloadBytes: 4096, + MaxBytes: 8192, + MaxItems: 7, Payloads: []payload.Payload{ payload.NewString("test message"), }, @@ -140,6 +143,9 @@ func TestMessageCodec_PackagePIDs_SourceTarget(t *testing.T) { if decoded.Messages[0].Topic != "test.topic" { t.Errorf("Topic mismatch. Expected 'test.topic', got %q", decoded.Messages[0].Topic) } + if decoded.Messages[0].PayloadBytes != 4096 || decoded.Messages[0].MaxBytes != 8192 || decoded.Messages[0].MaxItems != 7 { + t.Errorf("retention metadata mismatch: bytes=%d max_bytes=%d max_items=%d", decoded.Messages[0].PayloadBytes, decoded.Messages[0].MaxBytes, decoded.Messages[0].MaxItems) + } } func TestMessageCodec_ConcurrentEncodeSharedMapPayload(t *testing.T) { diff --git a/runtime/lua/engine/cdc_process_regression_test.go b/runtime/lua/engine/cdc_process_regression_test.go new file mode 100644 index 000000000..f8df23cd6 --- /dev/null +++ b/runtime/lua/engine/cdc_process_regression_test.go @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "context" + "errors" + "sync/atomic" + "testing" + + lua "github.com/wippyai/go-lua" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/topology" +) + +func newCDCRegressionProcess(t *testing.T) *Process { + t.Helper() + proc := mustNewProcess(t, WithScript(`return 1`, "cdc_process_regression.lua")) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + proc.Close() + t.Fatalf("process init failed: %v", err) + } + return proc +} + +func cdcRegressionHandler(_ context.Context, _ *lua.LState, _ pid.PID, _ string, _ []payload.Payload) lua.LValue { + return lua.LTrue +} + +// A bounded relay message must wait for its exact subscription. In particular, +// a startup message cannot be consumed by the process inbox before the CDC +// subscription has finished registering. +func TestBoundedMessageDoesNotFallbackToInbox(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + inbox := NewChannel(1) + if err := proc.SubscribeExisting(topology.TopicInbox, inbox); err != nil { + t.Fatal(err) + } + const topic = "cdc.startup" + proc.enqueueMessage(queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewString("snapshot")}, + PayloadBytes: 8, + MaxItems: 4, + MaxBytes: 64, + }) + proc.flushMessageQueue(proc.subs) + + if got := inbox.Size(); got != 0 { + t.Fatalf("bounded startup message fell back to inbox, size=%d", got) + } + if got := len(proc.messageQueue); got != 1 { + t.Fatalf("bounded startup message was not retained, queue=%d", got) + } + + cdc := NewChannel(1) + if err := proc.SubscribeExisting(topic, cdc); err != nil { + t.Fatal(err) + } + proc.SetTopicHandler(topic, cdcRegressionHandler) + proc.flushMessageQueue(proc.subs) + if got := cdc.Size(); got != 1 { + t.Fatalf("exact subscription did not receive startup message, size=%d", got) + } + if got := len(proc.messageQueue); got != 0 { + t.Fatalf("startup message remained after exact subscription, queue=%d", got) + } +} + +func TestBoundedTerminalDoesNotCloseInbox(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + inbox := NewChannel(1) + if err := proc.SubscribeExisting(topology.TopicInbox, inbox); err != nil { + t.Fatal(err) + } + const topic = "cdc.startup-terminal" + proc.enqueueMessage(queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewError(errors.New("snapshot failed")), payload.NewTerminal()}, + MaxItems: 1, + MaxBytes: 64, + }) + proc.flushMessageQueue(proc.subs) + + if inbox.IsClosed() { + t.Fatal("bounded terminal closed the inbox fallback channel") + } + if got := len(proc.messageQueue); got != 1 { + t.Fatalf("bounded terminal was not retained for exact subscription, queue=%d", got) + } + + cdc := NewChannel(1) + if err := proc.SubscribeExisting(topic, cdc); err != nil { + t.Fatal(err) + } + proc.flushMessageQueue(proc.subs) + if !cdc.IsClosed() { + t.Fatal("exact subscription did not receive terminal close") + } + if inbox.IsClosed() { + t.Fatal("inbox was closed while delivering exact terminal") + } +} + +func TestOrdinaryClosePreservesQueuedMessage(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + const topic = "ordinary.close" + old := NewChannel(1) + if err := proc.SubscribeExisting(topic, old); err != nil { + t.Fatal(err) + } + proc.SetTopicHandler(topic, cdcRegressionHandler) + proc.enqueueMessage(queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewString("ordinary")}, + }) + if !proc.closeChannel(old) { + t.Fatal("closeChannel did not remove ordinary subscription") + } + if got := len(proc.messageQueue); got != 1 { + t.Fatalf("ordinary queued message was discarded on close, queue=%d", got) + } + if old.IsClosed() == false { + t.Fatal("closed ordinary channel remains open") + } + + current := NewChannel(1) + if err := proc.SubscribeExisting(topic, current); err != nil { + t.Fatal(err) + } + proc.SetTopicHandler(topic, cdcRegressionHandler) + proc.flushMessageQueue(proc.subs) + if got := current.Size(); got != 1 { + t.Fatalf("preserved ordinary message was not delivered, size=%d", got) + } +} + +func TestGoErrorWithoutTerminalConsumesBoundedCapacity(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + const topic = "cdc.error-data" + message := func() queuedMessage { + return queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewError(errors.New("row failed"))}, + MaxItems: 1, + } + } + proc.enqueueMessage(message()) + proc.enqueueMessage(message()) + + if got := len(proc.messageQueue); got != 2 { + t.Fatalf("GoError-only data bypassed bounded admission, queue=%d", got) + } + if got := proc.messageQueueItems[topic]; got != 1 { + t.Fatalf("GoError-only data was not charged as an item, items=%d", got) + } + if !isOverflowTerminal(proc.messageQueue[1].Payloads) { + t.Fatalf("second GoError-only message did not produce overflow terminal") + } +} + +func TestCleanupRegisteredAfterOverflowStillRuns(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + const topic = "cdc.late-cleanup" + ch := NewChannel(1) + if err := proc.SubscribeExisting(topic, ch); err != nil { + t.Fatal(err) + } + proc.enqueueMessage(queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewString("too large")}, + PayloadBytes: 2, + MaxBytes: 1, + }) + var calls atomic.Int32 + if !proc.SetSubscriptionCleanup(ch, func() { calls.Add(1) }) { + t.Fatal("SetSubscriptionCleanup failed") + } + if got := calls.Load(); got != 1 { + t.Fatalf("late cleanup callback count=%d, want 1", got) + } + proc.closeChannel(ch) + proc.drainSubscriptionChannels() + if got := calls.Load(); got != 1 { + t.Fatalf("cleanup callback ran more than once: %d", got) + } +} + +func TestProcessQueueBackingReferencesClearBeforeProcessReuse(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + oldPayloads := payload.Payloads{payload.NewString("large retained value")} + proc.enqueueMessage(queuedMessage{Topic: "unsubscribed", Payloads: oldPayloads}) + backing := proc.messageQueue + if len(backing) != 1 || backing[0].Payloads == nil { + t.Fatal("test message was not queued") + } + + proc.clearExecution() + if len(proc.messageQueue) != 1 { + t.Fatalf("completed execution lost observable queued messages: %d", len(proc.messageQueue)) + } + + nextCtx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(nextCtx, "", nil); err != nil { + t.Fatalf("process reinit failed: %v", err) + } + if len(proc.messageQueue) != 0 { + t.Fatalf("process reuse left queued messages: %d", len(proc.messageQueue)) + } + if backing[0].Payloads != nil { + t.Fatal("process reuse left payloads reachable through queue backing array") + } +} + +type cdcTestLease struct { + calls atomic.Int32 +} + +func (l *cdcTestLease) Release() { + l.calls.Add(1) +} + +func TestLeasedMessageUsesUpstreamBudgetAndReleasesOnClear(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + lease := &cdcTestLease{} + proc.enqueueMessage(queuedMessage{ + Topic: "cdc.leased", + Payloads: payload.Payloads{payload.NewString("leased")}, + MaxItems: 1, + MaxBytes: 32, + Lease: lease, + }) + if got := proc.messageQueueItems["cdc.leased"]; got != 0 { + t.Fatalf("leased message consumed a second local item budget: %d", got) + } + if got := proc.messageQueueBytes["cdc.leased"]; got != 0 { + t.Fatalf("leased message consumed a second local byte budget: %d", got) + } + proc.clearMessageQueue() + if got := lease.calls.Load(); got != 1 { + t.Fatalf("leased message release count=%d, want 1", got) + } +} diff --git a/runtime/lua/engine/message_queue_bytes_test.go b/runtime/lua/engine/message_queue_bytes_test.go new file mode 100644 index 000000000..9a7c765fb --- /dev/null +++ b/runtime/lua/engine/message_queue_bytes_test.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "bytes" + "context" + "sync/atomic" + "testing" + + lua "github.com/wippyai/go-lua" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" +) + +func TestMessageQueueByteLimitRejectsLargePayload(t *testing.T) { + proc := mustNewProcess(t, WithScript("return 1", "test.lua")) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatal(err) + } + defer proc.Close() + + const topic = "cdc.large" + const limit = int64(1 << 20) + ch := NewChannel(1) + if err := proc.SubscribeExisting(topic, ch); err != nil { + t.Fatal(err) + } + proc.SetTopicHandler(topic, func(context.Context, *lua.LState, pid.PID, string, []payload.Payload) lua.LValue { + return lua.LTrue + }) + var cleanup atomic.Bool + if !proc.SetSubscriptionCleanup(ch, func() { cleanup.Store(true) }) { + t.Fatal("subscription cleanup was not installed") + } + + blob := bytes.Repeat([]byte{'x'}, int(limit)+1) + proc.enqueueMessage(queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.New(blob)}, + PayloadBytes: int64(len(blob)), + MaxBytes: limit, + }) + + if got := len(proc.messageQueue); got != 1 { + t.Fatalf("expected one synthetic terminal, got %d queued messages", got) + } + if got := proc.messageQueue[0].Payloads[0].Format(); got != payload.GoError { + t.Fatalf("expected overflow error payload, got %q", got) + } + if got := proc.messageQueueBytes[topic]; got != 0 { + t.Fatalf("overflow retained %d bytes", got) + } + if !cleanup.Load() { + t.Fatal("overflow did not stop the existing producer") + } + + proc.flushMessageQueue(proc.subs) + if !ch.IsClosed() { + t.Fatal("overflow terminal did not close the subscription") + } +} + +func TestMessageQueueByteLimitBoundsSlowConsumer(t *testing.T) { + proc := mustNewProcess(t, WithScript("return 1", "test.lua")) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatal(err) + } + defer proc.Close() + + const topic = "cdc.slow" + const messageBytes = int64(100) + const limit = int64(128) + ch := NewChannel(0) + if err := proc.SubscribeExisting(topic, ch); err != nil { + t.Fatal(err) + } + proc.SetTopicHandler(topic, func(context.Context, *lua.LState, pid.PID, string, []payload.Payload) lua.LValue { + return lua.LTrue + }) + var cleanup atomic.Bool + if !proc.SetSubscriptionCleanup(ch, func() { cleanup.Store(true) }) { + t.Fatal("subscription cleanup was not installed") + } + + message := func() queuedMessage { + return queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewString("change")}, + PayloadBytes: messageBytes, + MaxBytes: limit, + } + } + + // With a rendezvous channel and no waiting consumer, the first value stays + // in Process.messageQueue; the next value exceeds the byte budget. + proc.enqueueMessage(message()) + proc.flushMessageQueue(proc.subs) + proc.enqueueMessage(message()) + if got := proc.messageQueueBytes[topic]; got != messageBytes { + t.Fatalf("expected %d retained bytes, got %d", messageBytes, got) + } + + // Further values are dropped after exactly one terminal is queued. + for i := 0; i < 100; i++ { + proc.enqueueMessage(message()) + } + if got := proc.messageQueueBytes[topic]; got > limit { + t.Fatalf("retained %d bytes above limit %d", got, limit) + } + if got := len(proc.messageQueue); got != 2 { + t.Fatalf("expected retained value plus terminal, got %d messages", got) + } + if !cleanup.Load() { + t.Fatal("overflow did not stop the existing producer") + } +} + +func TestMessageQueueItemLimitIsPerTopicAndTerminalOrdered(t *testing.T) { + proc := mustNewProcess(t, WithScript("return 1", "test.lua")) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatal(err) + } + defer proc.Close() + + const limit = 2 + for _, topic := range []string{"cdc.one", "cdc.two"} { + ch := NewChannel(0) + if err := proc.SubscribeExisting(topic, ch); err != nil { + t.Fatal(err) + } + for i := 0; i < limit+4; i++ { + proc.enqueueMessage(queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewString("change")}, + MaxItems: limit, + }) + } + } + + if got := len(proc.messageQueue); got != 2*(limit+1) { + t.Fatalf("expected two bounded queues with data plus terminal, got %d", got) + } + if got := len(proc.messageQueueOverflowed); got != 2 { + t.Fatalf("expected independent overflow tombstones, got %d", got) + } +} diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index 7a91b9a88..70490dbcf 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -79,17 +79,20 @@ func WithStateOptions(opts lua.Options) ProcessOption { // Combines VM + CVM + Runner into a single unit. // Module binders and state options are stored in Factory for sharing across processes. type Process struct { - ctx context.Context - linkDownError error - execErr error - result payload.Payload - channelQueue *TaskQueue - subs *subscribeContext - mainTask *Task - upgradeRequest *UpgradeRequest - proto *lua.FunctionProto - queue *TaskQueue - factory *Factory + ctx context.Context + linkDownError error + execErr error + result payload.Payload + // Message queue limits apply only to messages that opt into bounded + // retention through relay metadata. Ordinary process messages preserve + // their historical behavior. + messageQueueLimits map[string]int64 + messageQueueItemLimits map[string]int + mainTask *Task + upgradeRequest *UpgradeRequest + proto *lua.FunctionProto + queue *TaskQueue + factory *Factory // pendingOutdated holds the single coalesced OUTDATED event awaiting // delivery to an upgradable process's events channel. Nil when none pending. pendingOutdated *topology.OutdatedEvent @@ -97,36 +100,51 @@ type Process struct { channels map[*Channel]int state *lua.LState handlers map[string]TopicHandler - // stalledChans tracks channels that retained an undeliverable message in - // the current flush pass, keyed on the resolved *Channel. Once a channel - // stalls, every later mailbox message for it (including a terminal) is also - // retained so a terminal cannot overtake earlier retained data on the same - // channel. Lazily created on first stall, cleared at the start of each flush. - stalledChans map[*Channel]struct{} - exported map[string]*lua.LFunction - scriptName string - script string - outTasks []*Task - externalTasks []*Task - yieldBuf []*Task - messageQueue []queuedMessage - threads []*Task - yieldSeq uint64 + // stalledChans records channels that could not accept a message during the + // current flush. Later messages for the same channel, including terminals, + // remain queued so delivery order cannot be inverted. + stalledChans map[*Channel]struct{} + exported map[string]*lua.LFunction + messageQueueDiscarded map[string]struct{} + messageQueueOverflowed map[string]struct{} + channelQueue *TaskQueue + subs *subscribeContext + // messageQueueBytes accounts only messages that opt into a byte limit via + // relay metadata. Ordinary process messages keep their historical behavior. + messageQueueBytes map[string]int64 + messageQueueItems map[string]int + script string + scriptName string + messageQueue []queuedMessage + yieldBuf []*Task + externalTasks []*Task + outTasks []*Task + threads []*Task + yieldSeq uint64 // epoch is the monotonic incarnation counter. Incremented on every // Init / clearExecution / Close drain and on Abort. Producers stamp // every SubscriptionFrame with the epoch they were registered under; // deliverMessage compares atomically so frames from prior incarnations // are dropped without locking. - epoch atomic.Uint64 - trapLinks bool - upgradable bool + epoch atomic.Uint64 + flushingMessages bool + trapLinks bool + upgradable bool } // queuedMessage stores a message waiting to be delivered type queuedMessage struct { - Source pid.PID - Topic string - Payloads []payload.Payload + // Lease transfers the upstream EventQueue reservation into this mailbox. + // It is released only when this queued message is delivered, discarded, or + // the process execution is reset. A leased message is already bounded by + // the upstream queue and therefore does not consume a second local budget. + Lease relay.RetentionLease + Source pid.PID + Topic string + Payloads []payload.Payload + MaxItems int + PayloadBytes int64 + MaxBytes int64 } // GetProcess retrieves the Process from LState via Owner. @@ -196,16 +214,29 @@ func (p *Process) SetSubscriptionCleanup(ch *Channel, fn func()) bool { return false } p.subs.mu.Lock() - defer p.subs.mu.Unlock() topic, ok := p.subs.byChannel[ch] if !ok { + p.subs.mu.Unlock() return false } sub := p.subs.byTopic[topic] if sub == nil { + p.subs.mu.Unlock() return false } - sub.cleanup = fn + overflowed := false + if _, exists := p.messageQueueOverflowed[topic]; exists { + overflowed = true + } + p.subs.mu.Unlock() + sub.setCleanup(fn) + // If admission overflowed before the Lua subscription yield completed, + // stop the source as soon as its cleanup hook becomes available. Do this + // outside the subscription lock because cleanup may unsubscribe the same + // channel. + if overflowed { + sub.callCleanup() + } return true } @@ -253,6 +284,16 @@ func (p *Process) closeChannel(ch *Channel) bool { sub.gen.Add(1) sub.callCleanup() } + if p.topicHasBoundedMessages(topic) { + if p.flushingMessages { + if p.messageQueueDiscarded == nil { + p.messageQueueDiscarded = make(map[string]struct{}) + } + p.messageQueueDiscarded[topic] = struct{}{} + } else { + p.discardMessageTopic(topic) + } + } if !ch.IsClosed() { p.applyExternalChannelResult(ch.Close(nil)) } @@ -535,8 +576,10 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload p.channelQueue.Drain() } - // Clear message queue - p.messageQueue = p.messageQueue[:0] + // Clear message queue and all accounting, including payload references in + // the retained backing array. Processes are pooled, so truncating the slice + // alone would keep the previous execution's data alive. + p.clearMessageQueue() p.pendingOutdated = nil // Seal the frame - no more modifications allowed after this @@ -672,10 +715,17 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { // Add incoming messages to queue first (before any processing) for _, pkg := range messages { for _, msg := range pkg.Messages { - p.messageQueue = append(p.messageQueue, queuedMessage{ - Source: pkg.Source, - Topic: msg.Topic, - Payloads: msg.Payloads, + if msg == nil { + continue + } + p.enqueueMessage(queuedMessage{ + Source: pkg.Source, + Topic: msg.Topic, + Payloads: msg.Payloads, + MaxItems: msg.MaxItems, + PayloadBytes: msg.PayloadBytes, + MaxBytes: msg.MaxBytes, + Lease: msg.TakeRetentionLease(), }) } relay.ReleasePackage(pkg) @@ -1057,12 +1107,18 @@ func (p *Process) flushMessageQueue(subs *subscribeContext) { // Process queue, retaining undelivered messages in order. remaining := p.messageQueue[:0] + p.flushingMessages = true for _, qm := range p.messageQueue { if p.deliverMessage(subs, qm) { remaining = append(remaining, qm) // retain in queue + } else { + p.releaseQueuedMessage(qm) } } + p.flushingMessages = false p.messageQueue = remaining + clear(p.messageQueue[len(remaining):]) + p.finishDiscardedTopics() } // A coalesced OUTDATED event lives outside the queue in a single slot and is @@ -1072,6 +1128,264 @@ func (p *Process) flushMessageQueue(subs *subscribeContext) { } } +// clearMessageQueue releases queued payload references before truncating the +// reusable slice. This is required on both execution reset and process-pool +// reuse; otherwise a single large message remains reachable through the +// backing array until that slice grows past its old capacity. +func (p *Process) clearMessageQueue() { + for _, qm := range p.messageQueue { + p.releaseQueuedMessage(qm) + } + clear(p.messageQueue) + p.messageQueue = p.messageQueue[:0] + clear(p.messageQueueItems) + clear(p.messageQueueBytes) + clear(p.messageQueueItemLimits) + clear(p.messageQueueLimits) + clear(p.messageQueueOverflowed) + clear(p.messageQueueDiscarded) + clear(p.stalledChans) + p.flushingMessages = false +} + +// enqueueMessage is the single handoff from relay delivery into the process +// mailbox. Limits are opt-in: only messages carrying MaxItems/MaxBytes are +// bounded, so unrelated process topics retain their historical behavior. +func (p *Process) enqueueMessage(qm queuedMessage) { + if qm.MaxItems <= 0 { + qm.MaxItems = p.messageQueueItemLimits[qm.Topic] + } + if qm.MaxBytes <= 0 { + qm.MaxBytes = p.messageQueueLimits[qm.Topic] + } + if qm.MaxItems > 0 { + if p.messageQueueItemLimits == nil { + p.messageQueueItemLimits = make(map[string]int) + } + if previous := p.messageQueueItemLimits[qm.Topic]; previous > 0 && previous < qm.MaxItems { + qm.MaxItems = previous + } + p.messageQueueItemLimits[qm.Topic] = qm.MaxItems + } + if qm.MaxBytes > 0 { + if p.messageQueueLimits == nil { + p.messageQueueLimits = make(map[string]int64) + } + if previous := p.messageQueueLimits[qm.Topic]; previous > 0 && previous < qm.MaxBytes { + qm.MaxBytes = previous + } + p.messageQueueLimits[qm.Topic] = qm.MaxBytes + } + if _, discarded := p.messageQueueDiscarded[qm.Topic]; discarded && messageIsBounded(qm) { + releaseMessageLease(qm) + return + } + if !hasDataPayload(qm.Payloads) { + if _, overflowed := p.messageQueueOverflowed[qm.Topic]; overflowed { + releaseMessageLease(qm) + return + } + if isOverflowTerminal(qm.Payloads) { + if p.messageQueueOverflowed == nil { + p.messageQueueOverflowed = make(map[string]struct{}) + } + p.messageQueueOverflowed[qm.Topic] = struct{}{} + if p.subs != nil { + if sub, ok := p.subs.get(qm.Topic); ok { + sub.callCleanup() + } + } + } + // Terminals are always admissible and never consume backlog capacity. + p.messageQueue = append(p.messageQueue, qm) + return + } + + if _, overflowed := p.messageQueueOverflowed[qm.Topic]; overflowed { + releaseMessageLease(qm) + return + } + + if qm.Lease == nil && (qm.MaxItems > 0 || qm.MaxBytes > 0) { + // A bounded producer must provide a conservative size. If it does not, + // charge the whole budget rather than retaining an unaccounted value. + if qm.PayloadBytes <= 0 && hasDataPayload(qm.Payloads) { + if qm.MaxBytes > 0 { + qm.PayloadBytes = qm.MaxBytes + } + } + queuedItems := p.messageQueueItems[qm.Topic] + queuedBytes := p.messageQueueBytes[qm.Topic] + if (qm.MaxItems > 0 && queuedItems >= qm.MaxItems) || + (qm.MaxBytes > 0 && (qm.PayloadBytes > qm.MaxBytes || queuedBytes > qm.MaxBytes-qm.PayloadBytes)) { + p.overflowMessageQueue(qm) + return + } + if qm.MaxItems > 0 { + if p.messageQueueItems == nil { + p.messageQueueItems = make(map[string]int) + } + p.messageQueueItems[qm.Topic] = queuedItems + 1 + } + if qm.MaxBytes > 0 && qm.PayloadBytes > 0 { + if p.messageQueueBytes == nil { + p.messageQueueBytes = make(map[string]int64) + } + p.messageQueueBytes[qm.Topic] = queuedBytes + qm.PayloadBytes + } + } + p.messageQueue = append(p.messageQueue, qm) +} + +func (p *Process) overflowMessageQueue(qm queuedMessage) { + releaseMessageLease(qm) + if p.messageQueueOverflowed == nil { + p.messageQueueOverflowed = make(map[string]struct{}) + } + if _, exists := p.messageQueueOverflowed[qm.Topic]; exists { + return + } + p.messageQueueOverflowed[qm.Topic] = struct{}{} + + // Stop the producer through the subscription's existing ownership hook. + // The channel is closed by the terminal below on the process step goroutine. + if p.subs != nil { + if sub, ok := p.subs.get(qm.Topic); ok { + sub.callCleanup() + } + } + p.messageQueue = append(p.messageQueue, queuedMessage{ + Source: qm.Source, + Topic: qm.Topic, + Payloads: payload.Payloads{payload.NewError(process.ErrMessageQueueOverflow), payload.NewTerminal()}, + MaxItems: qm.MaxItems, + MaxBytes: qm.MaxBytes, + }) +} + +func (p *Process) releaseQueuedMessage(qm queuedMessage) { + releaseMessageLease(qm) + if qm.Lease != nil { + return + } + if qm.MaxItems > 0 && p.messageQueueItems != nil { + remaining := p.messageQueueItems[qm.Topic] - 1 + if remaining > 0 { + p.messageQueueItems[qm.Topic] = remaining + } else { + delete(p.messageQueueItems, qm.Topic) + } + } + if qm.MaxBytes > 0 && qm.PayloadBytes > 0 && p.messageQueueBytes != nil { + remaining := p.messageQueueBytes[qm.Topic] - qm.PayloadBytes + if remaining > 0 { + p.messageQueueBytes[qm.Topic] = remaining + } else { + delete(p.messageQueueBytes, qm.Topic) + } + } +} + +func releaseMessageLease(qm queuedMessage) { + if qm.Lease != nil { + qm.Lease.Release() + } +} + +func messageIsBounded(qm queuedMessage) bool { + return qm.Lease != nil || qm.MaxItems > 0 || qm.MaxBytes > 0 +} + +// topicHasBoundedMessages reports whether closing a subscription must leave a +// bounded-topic tombstone. The limit maps persist after delivery so a producer +// that is still racing with close is treated consistently; ordinary messages +// on the same topic remain eligible for their historical inbox behavior. +func (p *Process) topicHasBoundedMessages(topic string) bool { + if _, ok := p.messageQueueOverflowed[topic]; ok { + return true + } + if p.messageQueueItemLimits[topic] > 0 || p.messageQueueLimits[topic] > 0 { + return true + } + for _, qm := range p.messageQueue { + if qm.Topic == topic && messageIsBounded(qm) { + return true + } + } + return false +} + +func (p *Process) discardMessageTopic(topic string) { + queued := p.messageQueue + remaining := queued[:0] + for _, qm := range queued { + if qm.Topic == topic && messageIsBounded(qm) { + p.releaseQueuedMessage(qm) + continue + } + remaining = append(remaining, qm) + } + p.messageQueue = remaining + clear(queued[len(remaining):]) + delete(p.messageQueueItems, topic) + delete(p.messageQueueBytes, topic) + delete(p.messageQueueItemLimits, topic) + delete(p.messageQueueLimits, topic) + delete(p.messageQueueOverflowed, topic) + delete(p.messageQueueDiscarded, topic) +} + +func (p *Process) finishDiscardedTopics() { + for topic := range p.messageQueueDiscarded { + found := false + for _, qm := range p.messageQueue { + if qm.Topic == topic && messageIsBounded(qm) { + found = true + break + } + } + if !found { + delete(p.messageQueueItems, topic) + delete(p.messageQueueBytes, topic) + delete(p.messageQueueItemLimits, topic) + delete(p.messageQueueLimits, topic) + delete(p.messageQueueOverflowed, topic) + delete(p.messageQueueDiscarded, topic) + } + } +} + +func hasDataPayload(payloads payload.Payloads) bool { + hasTerminal := len(payloads) > 0 && payload.IsTerminal(payloads[len(payloads)-1]) + for _, pl := range payloads { + if pl == nil || payload.IsTerminal(pl) { + continue + } + // A Go error is data unless it is part of the terminal result shape. + // Treating every GoError as control would allow an unbounded stream of + // non-terminal errors to bypass the producer budget. + if hasTerminal && pl.Format() == payload.GoError { + continue + } + return true + } + return false +} + +func isOverflowTerminal(payloads payload.Payloads) bool { + if len(payloads) == 0 || !payload.IsTerminal(payloads[len(payloads)-1]) { + return false + } + for _, pl := range payloads { + if pl == nil || pl.Format() != payload.GoError { + continue + } + err, ok := pl.Data().(error) + return ok && errors.Is(err, process.ErrMessageQueueOverflow) + } + return false +} + // markStalled records that a channel retained a message in the current flush // pass, lazily creating the set. func (p *Process) markStalled(ch *Channel) { @@ -1100,6 +1414,9 @@ func (p *Process) isStalled(ch *Channel) bool { // stall; pure data retries normally and a full buffer preserves its order. func (p *Process) deliverMessage(subs *subscribeContext, qm queuedMessage) (keep bool) { topic := qm.Topic + if _, discarded := p.messageQueueDiscarded[topic]; discarded && messageIsBounded(qm) { + return false + } handlerTopic := topic frame, hasFrame := subscriptionFrameFromPayloads(qm.Payloads) @@ -1132,6 +1449,13 @@ func (p *Process) deliverMessage(subs *subscribeContext, qm queuedMessage) (keep if hasFrame { return false } + // Bounded producer messages must wait for their exact subscription. The + // inbox is a compatibility fallback for ordinary process messages, but + // routing a bounded startup message there can consume/close the wrong + // channel before the producer's subscription is registered. + if messageIsBounded(qm) { + return true + } // Fallback to inbox for non-@ topics if !strings.HasPrefix(topic, "@") { sub, exists = subs.get(topology.TopicInbox) @@ -1566,6 +1890,7 @@ func (p *Process) Close() { p.yieldBuf = p.yieldBuf[:0] p.externalTasks = p.externalTasks[:0] p.outTasks = p.outTasks[:0] + p.clearMessageQueue() // Clear all references p.ctx = nil @@ -1583,6 +1908,13 @@ func (p *Process) Close() { p.subs = nil p.handlers = nil p.messageQueue = nil + p.messageQueueItems = nil + p.messageQueueBytes = nil + p.messageQueueItemLimits = nil + p.messageQueueLimits = nil + p.messageQueueOverflowed = nil + p.messageQueueDiscarded = nil + p.flushingMessages = false p.stalledChans = nil p.trapLinks = false p.upgradable = false @@ -1699,6 +2031,9 @@ func (p *Process) clearExecution() { if p.channelQueue != nil { p.channelQueue.Drain() } + // Keep undelivered ordinary messages observable until the scheduler retires + // or reinitializes this execution. Init and Close clear the backing storage, + // including bounded retention leases, before a pooled process can be reused. // Clear yield buffer p.yieldBuf = p.yieldBuf[:0] diff --git a/runtime/lua/engine/subscribe.go b/runtime/lua/engine/subscribe.go index 4ea07b65d..b7419660a 100644 --- a/runtime/lua/engine/subscribe.go +++ b/runtime/lua/engine/subscribe.go @@ -147,24 +147,64 @@ func (m *subscribeContext) snapshotSubscriptions() []*subscription { // subscription links a topic to a channel. type subscription struct { - cleanup func() - channel *Channel - topic string - id uint64 - gen atomic.Uint64 - cleanupOnce sync.Once + channel *Channel + // Cleanup can be requested before the producer has finished registering + // its hook (for example when a bounded relay overflows during startup). + // Keep that request pending until the hook is installed instead of + // consuming a one-shot guard while cleanup is nil. + cleanup func() + topic string + id uint64 + gen atomic.Uint64 + cleanupMu sync.Mutex + cleanupRequested bool + cleanupDone bool } func (s *subscription) callCleanup() { if s == nil { return } - s.cleanupOnce.Do(func() { - if s.cleanup != nil { - s.cleanup() - s.cleanup = nil - } - }) + s.cleanupMu.Lock() + if s.cleanupDone { + s.cleanupMu.Unlock() + return + } + s.cleanupRequested = true + cleanup := s.cleanup + if cleanup != nil { + s.cleanup = nil + s.cleanupDone = true + } + s.cleanupMu.Unlock() + + if cleanup != nil { + cleanup() + } +} + +// setCleanup installs a producer cleanup hook. A cleanup request that arrived +// before registration is fulfilled exactly once after the hook is visible. +// The callback always runs outside cleanupMu so it may safely tear down the +// subscription or call back into the process. +func (s *subscription) setCleanup(cleanup func()) { + if s == nil || cleanup == nil { + return + } + s.cleanupMu.Lock() + if s.cleanupDone { + s.cleanupMu.Unlock() + return + } + s.cleanup = cleanup + if s.cleanupRequested { + s.cleanup = nil + s.cleanupDone = true + s.cleanupMu.Unlock() + cleanup() + return + } + s.cleanupMu.Unlock() } // SubscriptionFrame carries process-epoch, subscription-id, and generation diff --git a/runtime/lua/modules/cdc/module.go b/runtime/lua/modules/cdc/module.go index aef3ade82..69cf7f4c2 100644 --- a/runtime/lua/modules/cdc/module.go +++ b/runtime/lua/modules/cdc/module.go @@ -4,10 +4,13 @@ package cdc import ( "fmt" + "math" + "strings" "sync" "sync/atomic" lua "github.com/wippyai/go-lua" + "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/runtime" luaapi "github.com/wippyai/runtime/api/runtime/lua" cdcapi "github.com/wippyai/runtime/api/service/cdc" @@ -15,13 +18,26 @@ import ( "github.com/wippyai/runtime/runtime/lua/engine/value" ) -const cdcStreamTypeName = "cdc.Stream" +const ( + cdcStreamTypeName = "cdc.Stream" + + // Keep Lua-side option allocations bounded independently of the source + // implementation. The source may apply a smaller limit, but this upper + // bound prevents a malformed table or direct internal option from causing + // an unbounded map/slice/channel allocation in the Lua adapter. + defaultStreamBuffer = 64 + maxStreamItems = 65536 + // LNumber is float64. Values above this boundary are not all exactly + // representable; integer-valued Lua literals use LInteger below and retain + // the complete int64 range instead. + maxExactLuaNumber = int64(1<<53 - 1) +) var subscriptionCounter uint64 var Module = &luaapi.ModuleDef{ Name: "cdc", - Description: "Postgres CDC source streams", + Description: "Driver-neutral CDC source streams", Class: []string{luaapi.ClassStorage, luaapi.ClassNondeterministic}, Build: func() (*lua.LTable, []luaapi.YieldType) { value.RegisterTypeMethods(nil, cdcStreamTypeName, nil, streamMethods) @@ -67,16 +83,21 @@ func listSources(l *lua.LState) int { return 2 } - inspector := cdcapi.GetSourceInspector(ctx) - if inspector == nil { - l.Push(lua.LNil) - l.Push(lua.NewLuaError(l, "cdc source inspector not found"). - WithKind(lua.Internal). - WithRetryable(false)) - return 2 + var infos []cdcapi.SourceInfo + if registry := cdcapi.GetRegistry(ctx); registry != nil { + infos = registry.List() + } else { + inspector := cdcapi.GetSourceInspector(ctx) + if inspector == nil { + l.Push(lua.LNil) + l.Push(lua.NewLuaError(l, "cdc source inspector not found"). + WithKind(lua.Internal). + WithRetryable(false)) + return 2 + } + infos = inspector.List() } - infos := inspector.List() result := l.CreateTable(len(infos), 0) for i, info := range infos { result.RawSetInt(i+1, sourceInfoToTable(l, info)) @@ -105,8 +126,26 @@ func getSource(l *lua.LState) int { return 2 } - inspector := cdcapi.GetSourceInspector(ctx) - if inspector == nil { + var ( + info cdcapi.SourceInfo + ok bool + ) + if cdcRegistry := cdcapi.GetRegistry(ctx); cdcRegistry != nil { + id := registry.ParseID(name) + source, found := cdcRegistry.Get(id) + if found && source != nil { + info = source.Info() + if info.ID.NS == "" && info.ID.Name == "" { + info.ID = id + } + if info.Name == "" { + info.Name = info.ID.String() + } + ok = true + } + } else if inspector := cdcapi.GetSourceInspector(ctx); inspector != nil { + info, ok = inspector.Get(name) + } else { l.Push(lua.LNil) l.Push(lua.NewLuaError(l, "cdc source inspector not found"). WithKind(lua.Internal). @@ -114,7 +153,6 @@ func getSource(l *lua.LState) int { return 2 } - info, ok := inspector.Get(name) if !ok { l.Push(lua.LNil) l.Push(lua.LNil) @@ -151,7 +189,12 @@ func openStream(l *lua.LState) int { return 2 } - ch := engine.NewChannel(64) + // CDC backlog capacity is enforced by the source/relay/process mailbox. + // Keep the Lua channel rendezvous-only so buffered Lua values cannot form a + // second queue outside that shared budget. Normalize the historical default + // before the command crosses the Lua boundary. + opts.Buffer = streamBufferCapacity(opts.Buffer) + ch := engine.NewChannel(0) engine.PushChannel(l, ch) l.Pop(1) @@ -264,9 +307,50 @@ func (s *Stream) closeWithUnsubscribe(unsubscribe bool) { } func sourceInfoToTable(l *lua.LState, info cdcapi.SourceInfo) *lua.LTable { - t := l.CreateTable(0, 8) + t := l.CreateTable(0, 24) + if !isZeroRegistryID(info.ID) { + t.RawSetString("id", lua.LString(registryIDString(info.ID))) + } + if info.Kind != "" { + t.RawSetString("kind", lua.LString(info.Kind)) + } + state := string(info.State) + if state == "" { + state = string(cdcapi.SourceStateUnknown) + } + t.RawSetString("state", lua.LString(state)) + if info.Generation != "" { + t.RawSetString("generation", lua.LString(info.Generation)) + } + + capabilities := l.CreateTable(0, 6) + capabilities.RawSetString("snapshot", lua.LBool(info.Capabilities.Snapshot)) + capabilities.RawSetString("durable", lua.LBool(info.Capabilities.Durable)) + capabilities.RawSetString("replayable", lua.LBool(info.Capabilities.Replayable)) + capabilities.RawSetString("captures_external_writes", lua.LBool(info.Capabilities.CapturesExternalWrites)) + capabilities.RawSetString("before_images", lua.LBool(info.Capabilities.BeforeImages)) + capabilities.RawSetString("coalesced", lua.LBool(info.Capabilities.Coalesced)) + t.RawSetString("capabilities", capabilities) + + // Keep the legacy identity fields present with their historical defaults; + // newer callers should use id/kind/state above. t.RawSetString("name", lua.LString(info.Name)) t.RawSetString("slot", lua.LString(info.Slot)) + if info.Engine != "" { + t.RawSetString("engine", lua.LString(info.Engine)) + } + if info.File != "" { + t.RawSetString("file", lua.LString(info.File)) + } + if info.DBResource != "" { + t.RawSetString("db_resource", lua.LString(info.DBResource)) + } + if info.Epoch != "" { + t.RawSetString("epoch", lua.LString(info.Epoch)) + } + if info.Error != "" { + t.RawSetString("error", lua.LString(info.Error)) + } if info.Publication != "" { t.RawSetString("publication", lua.LString(info.Publication)) } @@ -281,6 +365,7 @@ func sourceInfoToTable(l *lua.LState, info cdcapi.SourceInfo) *lua.LTable { t.RawSetString("failover", lua.LBool(info.Failover)) t.RawSetString("temporary", lua.LBool(info.Temporary)) t.RawSetString("snapshot", lua.LBool(info.Snapshot)) + t.RawSetString("faulted", lua.LBool(info.Faulted)) return t } @@ -296,41 +381,185 @@ func streamOptionsFromLua(l *lua.LState, idx int) (cdcapi.StreamOptions, *lua.Er WithRetryable(false) } - opts.Tables = stringArrayField(table, "tables") - opts.Ops = stringArrayField(table, "ops") + if errMsg := validateOptionKeys(table); errMsg != "" { + return opts, invalidStreamOption(l, errMsg) + } + + var errMsg string + if opts.Tables, errMsg = stringArrayField(table, "tables"); errMsg != "" { + return opts, invalidStreamOption(l, errMsg) + } + if opts.Ops, errMsg = stringArrayField(table, "ops"); errMsg != "" { + return opts, invalidStreamOption(l, errMsg) + } if v := table.RawGetString("buffer"); v != lua.LNil { if v.Type() != lua.LTNumber && v.Type() != lua.LTInteger { - return opts, lua.NewLuaError(l, "buffer must be a number"). - WithKind(lua.Invalid). - WithRetryable(false) + return opts, invalidStreamOption(l, "buffer must be a number") } - n := int(lua.LVAsNumber(v)) - if n <= 0 || n > 65536 { - return opts, lua.NewLuaError(l, "buffer must be between 1 and 65536"). - WithKind(lua.Invalid). - WithRetryable(false) + number := lua.LVAsNumber(v) + if math.IsNaN(float64(number)) || math.IsInf(float64(number), 0) || + math.Trunc(float64(number)) != float64(number) || + number < 1 || number > lua.LNumber(maxStreamItems) { + return opts, invalidStreamOption(l, "buffer must be a positive integer") } + n := int(number) opts.Buffer = n } + if v := table.RawGetString("max_bytes"); v != lua.LNil { + if v.Type() != lua.LTNumber && v.Type() != lua.LTInteger { + return opts, invalidStreamOption(l, "max_bytes must be a number") + } + if v.Type() == lua.LTInteger { + number, ok := v.(lua.LInteger) + if !ok || number < 1 { + return opts, invalidStreamOption(l, "max_bytes must be a positive integer") + } + opts.MaxBytes = int64(number) + } else { + number := lua.LVAsNumber(v) + floatNumber := float64(number) + if math.IsNaN(floatNumber) || math.IsInf(floatNumber, 0) || + math.Trunc(floatNumber) != floatNumber || + number < 1 || number > lua.LNumber(maxExactLuaNumber) { + return opts, invalidStreamOption(l, "max_bytes must be an exact positive integer") + } + opts.MaxBytes = int64(number) + } + } + if v := table.RawGetString("snapshot"); v != lua.LNil { + if v.Type() != lua.LTBool { + return opts, invalidStreamOption(l, "snapshot must be a boolean") + } + opts.Snapshot = lua.LVAsBool(v) + } + if v := table.RawGetString("after"); v != lua.LNil { + if v.Type() != lua.LTString { + return opts, invalidStreamOption(l, "after must be a string") + } + after := string(v.(lua.LString)) + if strings.TrimSpace(after) == "" { + return opts, invalidStreamOption(l, "after must not be empty") + } + opts.After = after + } return opts, nil } -func stringArrayField(table *lua.LTable, field string) []string { +func streamBufferCapacity(buffer int) int { + if buffer <= 0 { + return defaultStreamBuffer + } + if buffer > maxStreamItems { + return maxStreamItems + } + return buffer +} + +func invalidStreamOption(l *lua.LState, message string) *lua.Error { + return lua.NewLuaError(l, message). + WithKind(lua.Invalid). + WithRetryable(false) +} + +func validateOptionKeys(table *lua.LTable) string { + var errMsg string + table.ForEach(func(key, _ lua.LValue) { + if errMsg != "" { + return + } + name, ok := key.(lua.LString) + if !ok { + errMsg = "stream options contains unknown or non-string field" + return + } + switch string(name) { + case "tables", "ops", "buffer", "max_bytes", "snapshot", "after": + default: + errMsg = "stream options contains unknown field: " + string(name) + } + }) + return errMsg +} + +func stringArrayField(table *lua.LTable, field string) ([]string, string) { v := table.RawGetString(field) if v == lua.LNil { - return nil + return nil, "" } t, ok := v.(*lua.LTable) if !ok { - return nil + return nil, field + " must be an array of strings" + } + count := t.Len() + if count > maxStreamItems { + return nil, field + " must contain at most 65536 entries" } - out := make([]string, 0, t.Len()) - t.ForEach(func(_, value lua.LValue) { - if value.Type() == lua.LTString { - out = append(out, value.String()) + values := make(map[int]string, count) + max := 0 + var errMsg string + t.ForEach(func(key, value lua.LValue) { + if errMsg != "" { + return + } + var position int + switch key.Type() { + case lua.LTInteger: + index, ok := key.(lua.LInteger) + if !ok || index <= 0 || index > lua.LInteger(maxStreamItems) { + errMsg = field + " must be an array of strings" + return + } + position = int(index) + case lua.LTNumber: + number := lua.LVAsNumber(key) + if math.IsNaN(float64(number)) || math.IsInf(float64(number), 0) || + math.Trunc(float64(number)) != float64(number) || number <= 0 || + number > lua.LNumber(maxStreamItems) { + errMsg = field + " must be an array of strings" + return + } + position = int(number) + if position <= 0 { + errMsg = field + " must be an array of strings" + return + } + default: + errMsg = field + " must be an array of strings" + return + } + if value.Type() != lua.LTString || strings.TrimSpace(value.String()) == "" { + errMsg = field + " must contain non-empty strings" + return + } + if _, exists := values[position]; !exists && len(values) >= maxStreamItems { + errMsg = field + " must contain at most 65536 entries" + return + } + values[position] = string(value.(lua.LString)) + if position > max { + max = position } }) - return out + if errMsg != "" { + return nil, errMsg + } + out := make([]string, max) + for i := 1; i <= max; i++ { + value, ok := values[i] + if !ok { + return nil, field + " must be a contiguous array of strings" + } + out[i-1] = value + } + return out, "" +} + +func isZeroRegistryID(id registry.ID) bool { + return id.NS == "" && id.Name == "" +} + +func registryIDString(id registry.ID) string { + return id.String() } func markSubscribed(stream *Stream, proc *engine.Process, topic string, cancelCleanup func()) error { diff --git a/runtime/lua/modules/cdc/module_test.go b/runtime/lua/modules/cdc/module_test.go index a645f8a7d..f1f4e6a02 100644 --- a/runtime/lua/modules/cdc/module_test.go +++ b/runtime/lua/modules/cdc/module_test.go @@ -4,12 +4,16 @@ package cdc import ( "context" + "errors" "testing" "github.com/stretchr/testify/require" lua "github.com/wippyai/go-lua" + "github.com/wippyai/go-lua/types/typ" + ctxapi "github.com/wippyai/runtime/api/context" "github.com/wippyai/runtime/api/payload" "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/registry" cdcapi "github.com/wippyai/runtime/api/service/cdc" ) @@ -43,6 +47,45 @@ func newStateWithInspector(t *testing.T, inspector cdcapi.SourceInspector) *lua. return l } +type fakeSource struct { + info cdcapi.SourceInfo +} + +func (f *fakeSource) Info() cdcapi.SourceInfo { return f.info } + +func (f *fakeSource) Subscribe(context.Context, cdcapi.StreamOptions) (cdcapi.Stream, error) { + return nil, nil +} + +type fakeRegistry struct { + source cdcapi.Source + all []cdcapi.SourceInfo +} + +func (f *fakeRegistry) List() []cdcapi.SourceInfo { return f.all } + +func (f *fakeRegistry) Get(id registry.ID) (cdcapi.Source, bool) { + if f.source == nil { + return nil, false + } + info := f.source.Info() + return f.source, !isZeroRegistryID(info.ID) && registryIDString(info.ID) == registryIDString(id) +} + +func newStateWithRegistry(t *testing.T, registry cdcapi.Registry) *lua.LState { + t.Helper() + l := lua.NewState() + t.Cleanup(l.Close) + + ctx := ctxapi.NewRootContext() + ctx = cdcapi.WithRegistry(ctx, registry) + l.SetContext(ctx) + + tbl, _ := Module.Build() + l.SetGlobal(Module.Name, tbl) + return l +} + func TestModuleLoad(t *testing.T) { l := lua.NewState() defer l.Close() @@ -60,8 +103,34 @@ func TestModuleLoad(t *testing.T) { func TestListSourcesReturnsAllInfos(t *testing.T) { l := newStateWithInspector(t, &fakeInspector{ all: []cdcapi.SourceInfo{ - {Name: "id-a", Slot: "slot_a", Publication: "pub_a", Streaming: true}, - {Name: "id-b", Slot: "slot_b", Tables: []string{"public.t"}, Failover: true}, + { + ID: registry.NewID("test", "id-a"), + Kind: "db.cdc.postgres", + State: cdcapi.SourceStateRunning, + Generation: "generation-a", + Capabilities: cdcapi.Capabilities{ + Snapshot: true, + Durable: true, + Replayable: true, + CapturesExternalWrites: true, + BeforeImages: true, + }, + Name: "id-a", + Slot: "slot_a", + Publication: "pub_a", + Streaming: true, + }, + { + ID: registry.NewID("test", "id-b"), + Kind: "db.cdc.sqlite", + State: cdcapi.SourceStateFaulted, + Capabilities: cdcapi.Capabilities{BeforeImages: true, Coalesced: true}, + Name: "id-b", + Slot: "slot_b", + Tables: []string{"public.t"}, + Failover: true, + Faulted: true, + }, }, }) @@ -69,6 +138,15 @@ func TestListSourcesReturnsAllInfos(t *testing.T) { local rows, err = cdc.list_sources() assert(err == nil, "unexpected error: " .. tostring(err)) assert(#rows == 2, "expected 2 rows, got " .. tostring(#rows)) + assert(rows[1].id == "test:id-a") + assert(rows[1].kind == "db.cdc.postgres") + assert(rows[1].state == "running") + assert(rows[1].generation == "generation-a") + assert(rows[1].capabilities.snapshot == true) + assert(rows[1].capabilities.durable == true) + assert(rows[1].capabilities.replayable == true) + assert(rows[1].capabilities.captures_external_writes == true) + assert(rows[1].capabilities.before_images == true) assert(rows[1].slot == "slot_a") assert(rows[1].publication == "pub_a") assert(rows[1].tables == nil, "row with no tables should omit the tables key") @@ -78,6 +156,27 @@ func TestListSourcesReturnsAllInfos(t *testing.T) { assert(rows[2].tables[1] == "public.t") assert(#rows[2].tables == 1) assert(rows[2].failover == true) + assert(rows[2].capabilities.coalesced == true) + `)) +} + +func TestListSourcesUsesDriverNeutralRegistry(t *testing.T) { + id := registry.NewID("test", "registry-source") + l := newStateWithRegistry(t, &fakeRegistry{ + all: []cdcapi.SourceInfo{{ + ID: id, + Kind: "db.cdc.sqlite", + State: cdcapi.SourceStateRunning, + Name: id.String(), + }}, + }) + + require.NoError(t, l.DoString(` + local rows, err = cdc.list_sources() + assert(err == nil, "unexpected error: " .. tostring(err)) + assert(#rows == 1) + assert(rows[1].id == "test:registry-source") + assert(rows[1].kind == "db.cdc.sqlite") `)) } @@ -99,6 +198,25 @@ func TestSourceByName(t *testing.T) { `)) } +func TestSourceByRegistryID(t *testing.T) { + id := registry.NewID("test", "source") + source := &fakeSource{info: cdcapi.SourceInfo{ + ID: id, + Kind: "db.cdc.sqlite", + State: cdcapi.SourceStateRunning, + Name: id.String(), + }} + l := newStateWithRegistry(t, &fakeRegistry{source: source}) + + require.NoError(t, l.DoString(` + local info, err = cdc.source("test:source") + assert(err == nil, "unexpected error: " .. tostring(err)) + assert(info ~= nil) + assert(info.id == "test:source") + assert(info.kind == "db.cdc.sqlite") + `)) +} + func TestSourceRequiresName(t *testing.T) { l := newStateWithInspector(t, &fakeInspector{}) @@ -117,6 +235,9 @@ func TestStreamOpenAndRelease(t *testing.T) { tables = {"public.accounts"}, ops = {"insert", "update"}, buffer = 4, + max_bytes = 4096, + snapshot = true, + after = "cursor-1", }) assert(err == nil, "unexpected error: " .. tostring(err)) assert(stream ~= nil) @@ -146,30 +267,131 @@ func TestStreamRejectsInvalidBuffer(t *testing.T) { `)) } +func TestStreamRejectsMalformedOptions(t *testing.T) { + cases := map[string]string{ + "tables type": `{ tables = "accounts" }`, + "tables element": `{ tables = { 1 } }`, + "ops element": `{ ops = { "insert", 2 } }`, + "fractional buffer": `{ buffer = 1.5 }`, + "zero buffer": `{ buffer = 0 }`, + "oversized buffer": `{ buffer = 65537 }`, + "max bytes type": `{ max_bytes = "4096" }`, + "fractional max bytes": `{ max_bytes = 1.5 }`, + "zero max bytes": `{ max_bytes = 0 }`, + "negative max bytes": `{ max_bytes = -1 }`, + "infinite max bytes": `{ max_bytes = math.huge }`, + "snapshot type": `{ snapshot = "true" }`, + "after type": `{ after = 42 }`, + "empty after": `{ after = "" }`, + "whitespace after": `{ after = " \t\n" }`, + "unknown field": `{ unsupported = true }`, + "numeric field": `{ [1] = "unsupported" }`, + } + for name, options := range cases { + t.Run(name, func(t *testing.T) { + l := newStateWithInspector(t, &fakeInspector{}) + script := ` + local stream, err = cdc.stream("source", ` + options + `) + assert(stream == nil) + assert(err ~= nil) + ` + require.NoError(t, l.DoString(script)) + }) + } +} + +func TestStreamMaxBytesPreservesIntegerAndRejectsInexactFloat(t *testing.T) { + l := lua.NewState() + defer l.Close() + + integerOptions := l.CreateTable(0, 1) + integerOptions.RawSetString("max_bytes", lua.LInteger(1<<63-1)) + l.Push(integerOptions) + options, luaErr := streamOptionsFromLua(l, 1) + require.Nil(t, luaErr) + require.Equal(t, int64(1<<63-1), options.MaxBytes) + l.SetTop(0) + + floatOptions := l.CreateTable(0, 1) + floatOptions.RawSetString("max_bytes", lua.LNumber(1<<53)) + l.Push(floatOptions) + _, luaErr = streamOptionsFromLua(l, 1) + require.NotNil(t, luaErr) +} + +func TestStringArrayFieldRejectsOutOfRangeIndex(t *testing.T) { + l := lua.NewState() + defer l.Close() + + values := l.CreateTable(0, 1) + values.RawSetInt(maxStreamItems+1, lua.LString("out-of-range")) + options := l.CreateTable(0, 1) + options.RawSetString("tables", values) + _, errMsg := stringArrayField(options, "tables") + require.NotEmpty(t, errMsg) +} + +func TestStreamBufferCapacityIsBounded(t *testing.T) { + for _, test := range []struct { + name string + input int + wanted int + }{ + {name: "default", input: 0, wanted: defaultStreamBuffer}, + {name: "negative uses default", input: -1, wanted: defaultStreamBuffer}, + {name: "configured", input: 128, wanted: 128}, + {name: "maximum", input: maxStreamItems, wanted: maxStreamItems}, + {name: "internal overflow is capped", input: maxStreamItems + 1, wanted: maxStreamItems}, + } { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, test.wanted, streamBufferCapacity(test.input)) + }) + } +} + +func TestModuleTypesUseIntegerBufferAndTypedChannel(t *testing.T) { + manifest := ModuleTypes() + streamOptions, ok := manifest.LookupType("StreamOptions") + require.True(t, ok) + optionsRecord, ok := streamOptions.(*typ.Record) + require.True(t, ok) + require.Equal(t, typ.Integer, optionsRecord.GetField("buffer").Type) + require.Equal(t, typ.Integer, optionsRecord.GetField("max_bytes").Type) + require.NotEqual(t, typ.Any, cdcChannelType) +} + func TestChangeHandlerUsesCanonicalLuaKeys(t *testing.T) { l := lua.NewState() defer l.Close() got := cdcChangeHandler(context.Background(), l, pid.Zero(), "cdc@1", []payload.Payload{ payload.New(cdcapi.Change{ - Source: "test:cdc", - Op: "insert", - Schema: "public", - Table: "accounts", - Relation: "public.accounts", - LSN: "0/16B6C50", - CommitLSN: "0/16B6C98", - XID: 42, - After: map[string]any{"email": "a@w.ai"}, + SourceID: registry.NewID("test", "cdc"), + Source: "test:cdc", + Op: "insert", + Schema: "public", + Table: "accounts", + Relation: "public.accounts", + LSN: "0/16B6C50", + CommitLSN: "0/16B6C98", + Cursor: "cursor-1", + Generation: "generation-1", + Transaction: "transaction-1", + XID: 42, + After: map[string]any{"email": "a@w.ai"}, }), }) tbl, ok := got.(*lua.LTable) require.True(t, ok) require.Equal(t, "test:cdc", tbl.RawGetString("source").String()) + require.Equal(t, "test:cdc", tbl.RawGetString("source_id").String()) require.Equal(t, "insert", tbl.RawGetString("op").String()) require.Equal(t, "public.accounts", tbl.RawGetString("relation").String()) require.Equal(t, "0/16B6C98", tbl.RawGetString("commit_lsn").String()) + require.Equal(t, "cursor-1", tbl.RawGetString("cursor").String()) + require.Equal(t, "generation-1", tbl.RawGetString("generation").String()) + require.Equal(t, "transaction-1", tbl.RawGetString("transaction").String()) require.Equal(t, lua.LNil, tbl.RawGetString("commit_lsn,omitempty")) require.Equal(t, lua.LNil, tbl.RawGetString("after,omitempty")) @@ -178,6 +400,47 @@ func TestChangeHandlerUsesCanonicalLuaKeys(t *testing.T) { require.Equal(t, "a@w.ai", after.RawGetString("email").String()) } +func TestChangeHandlerConvertsTypedStreamError(t *testing.T) { + l := lua.NewState() + defer l.Close() + + got := cdcChangeHandler(context.Background(), l, pid.Zero(), "cdc@1", []payload.Payload{ + payload.NewError(errors.New("capture gap")), + }) + streamErr, ok := lua.AsError(got) + require.True(t, ok) + require.Contains(t, streamErr.Error(), "capture gap") +} + +func TestModuleTypesMatchRuntimeFields(t *testing.T) { + manifest := ModuleTypes() + + assertRecordFields := func(name string, want []string) { + t.Helper() + value, ok := manifest.LookupType(name) + require.True(t, ok, "%s type is not defined", name) + record, ok := value.(*typ.Record) + require.True(t, ok, "%s is %T, want record", name, value) + for _, field := range want { + require.NotNil(t, record.GetField(field), "%s.%s is missing", name, field) + } + } + + assertRecordFields("Capabilities", []string{ + "snapshot", "durable", "replayable", "captures_external_writes", "before_images", "coalesced", + }) + assertRecordFields("SourceInfo", []string{ + "id", "kind", "state", "generation", "capabilities", "name", "slot", "publication", + "engine", "file", "db_resource", "epoch", "error", "tables", "streaming", "failover", + "temporary", "snapshot", "faulted", + }) + assertRecordFields("StreamOptions", []string{"tables", "ops", "buffer", "max_bytes", "snapshot", "after"}) + assertRecordFields("Change", []string{ + "source_id", "source", "op", "schema", "table", "relation", "lsn", "commit_lsn", "cursor", + "generation", "transaction", "error", "xid", "before", "after", + }) +} + func TestListSourcesFailsWithoutInspector(t *testing.T) { l := lua.NewState() defer l.Close() diff --git a/runtime/lua/modules/cdc/types.go b/runtime/lua/modules/cdc/types.go index bffd78314..fed2a3729 100644 --- a/runtime/lua/modules/cdc/types.go +++ b/runtime/lua/modules/cdc/types.go @@ -5,34 +5,95 @@ package cdc import ( "github.com/wippyai/go-lua/types/io" "github.com/wippyai/go-lua/types/typ" + "github.com/wippyai/runtime/runtime/lua/engine" ) +var cdcChannelType typ.Type + +var sourceCapabilitiesType = typ.NewRecord(). + Field("snapshot", typ.Boolean). + Field("durable", typ.Boolean). + Field("replayable", typ.Boolean). + Field("captures_external_writes", typ.Boolean). + Field("before_images", typ.Boolean). + Field("coalesced", typ.Boolean). + Build() + var sourceInfoType = typ.NewRecord(). + OptField("id", typ.String). + OptField("kind", typ.String). + Field("state", typ.String). + OptField("generation", typ.String). + Field("capabilities", sourceCapabilitiesType). Field("name", typ.String). Field("slot", typ.String). OptField("publication", typ.String). + OptField("engine", typ.String). + OptField("file", typ.String). + OptField("db_resource", typ.String). + OptField("epoch", typ.String). + OptField("error", typ.String). OptField("tables", typ.NewArray(typ.String)). Field("streaming", typ.Boolean). Field("failover", typ.Boolean). Field("temporary", typ.Boolean). Field("snapshot", typ.Boolean). + Field("faulted", typ.Boolean). Build() var streamOptionsType = typ.NewRecord(). OptField("tables", typ.NewArray(typ.String)). OptField("ops", typ.NewArray(typ.String)). - OptField("buffer", typ.Number). + OptField("buffer", typ.Integer). + OptField("max_bytes", typ.Integer). + OptField("snapshot", typ.Boolean). + OptField("after", typ.String). + Build() + +var changeType = typ.NewRecord(). + OptField("source_id", typ.String). + Field("source", typ.String). + Field("op", typ.String). + Field("schema", typ.String). + Field("table", typ.String). + Field("relation", typ.String). + Field("lsn", typ.String). + OptField("commit_lsn", typ.String). + OptField("cursor", typ.String). + OptField("generation", typ.String). + OptField("transaction", typ.String). + OptField("error", typ.String). + OptField("xid", typ.Integer). + OptField("before", typ.NewMap(typ.String, typ.Any)). + OptField("after", typ.NewMap(typ.String, typ.Any)). Build() -var cdcStreamType = typ.NewInterface("cdc.Stream", []typ.Method{ - {Name: "channel", Type: typ.Func().Param("self", typ.Self).Returns(typ.Any).Build()}, - {Name: "receive", Type: typ.Func().Param("self", typ.Self).Returns(typ.Any).Build()}, - {Name: "close", Type: typ.Func().Param("self", typ.Self).Returns(typ.Boolean, typ.NewOptional(typ.LuaError)).Build()}, - {Name: "release", Type: typ.Func().Param("self", typ.Self).Returns(typ.Boolean, typ.NewOptional(typ.LuaError)).Build()}, -}) +var cdcStreamType *typ.Interface + +func init() { + cdcChannelType = typ.Any + if manifest := engine.ChannelModuleTypes(); manifest != nil { + if t, ok := manifest.LookupType("Channel"); ok { + if gen, ok := t.(*typ.Generic); ok { + cdcChannelType = typ.Instantiate(gen, changeType) + } + } + } + + cdcStreamType = typ.NewInterface("cdc.Stream", []typ.Method{ + {Name: "channel", Type: typ.Func().Param("self", typ.Self).Returns(cdcChannelType).Build()}, + {Name: "receive", Type: typ.Func().Param("self", typ.Self).Returns(cdcChannelType).Build()}, + {Name: "close", Type: typ.Func().Param("self", typ.Self).Returns(typ.Boolean, typ.NewOptional(typ.LuaError)).Build()}, + {Name: "release", Type: typ.Func().Param("self", typ.Self).Returns(typ.Boolean, typ.NewOptional(typ.LuaError)).Build()}, + }) +} func ModuleTypes() *io.Manifest { m := io.NewManifest("cdc") + m.DefineType("Capabilities", sourceCapabilitiesType) + m.DefineType("SourceInfo", sourceInfoType) + m.DefineType("StreamOptions", streamOptionsType) + m.DefineType("Change", changeType) moduleType := typ.NewInterface("cdc", []typ.Method{ {Name: "list_sources", Type: typ.Func().Returns(typ.NewArray(sourceInfoType), typ.NewOptional(typ.LuaError)).Build()}, diff --git a/runtime/lua/modules/cdc/yields.go b/runtime/lua/modules/cdc/yields.go index 476d3bd84..d272e3c5b 100644 --- a/runtime/lua/modules/cdc/yields.go +++ b/runtime/lua/modules/cdc/yields.go @@ -122,6 +122,15 @@ func cdcChangeHandler(_ context.Context, l *lua.LState, _ pid.PID, _ string, pay if len(payloads) == 0 { return lua.LNil } + if payloads[0].Format() == payload.GoError { + streamErr, ok := payloads[0].Data().(error) + if !ok { + return lua.NewLuaError(l, fmt.Sprintf("cdc stream error payload has invalid type %T", payloads[0].Data())). + WithKind(lua.Internal). + WithRetryable(false) + } + return lua.WrapErrorWithLua(l, streamErr, "cdc stream") + } change, ok := payloads[0].Data().(cdcapi.Change) if !ok { if ptr, ptrOK := payloads[0].Data().(*cdcapi.Change); ptrOK && ptr != nil { @@ -145,7 +154,10 @@ func cdcChangeHandler(_ context.Context, l *lua.LState, _ pid.PID, _ string, pay } func changeToLua(l *lua.LState, change cdcapi.Change) (lua.LValue, error) { - tbl := l.CreateTable(0, 10) + tbl := l.CreateTable(0, 18) + if !isZeroRegistryID(change.SourceID) { + tbl.RawSetString("source_id", lua.LString(registryIDString(change.SourceID))) + } tbl.RawSetString("source", lua.LString(change.Source)) tbl.RawSetString("op", lua.LString(change.Op)) tbl.RawSetString("schema", lua.LString(change.Schema)) @@ -155,6 +167,18 @@ func changeToLua(l *lua.LState, change cdcapi.Change) (lua.LValue, error) { if change.CommitLSN != "" { tbl.RawSetString("commit_lsn", lua.LString(change.CommitLSN)) } + if change.Cursor != "" { + tbl.RawSetString("cursor", lua.LString(change.Cursor)) + } + if change.Generation != "" { + tbl.RawSetString("generation", lua.LString(change.Generation)) + } + if change.Transaction != "" { + tbl.RawSetString("transaction", lua.LString(change.Transaction)) + } + if change.Error != "" { + tbl.RawSetString("error", lua.LString(change.Error)) + } if change.XID != 0 { tbl.RawSetString("xid", lua.LInteger(change.XID)) } diff --git a/service/cdc/dispatcher.go b/service/cdc/dispatcher.go new file mode 100644 index 000000000..8b4f548e5 --- /dev/null +++ b/service/cdc/dispatcher.go @@ -0,0 +1,626 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package cdc provides the command dispatcher shared by all CDC drivers. +package cdc + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/wippyai/runtime/api/dispatcher" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + cdcapi "github.com/wippyai/runtime/api/service/cdc" + "go.uber.org/zap" +) + +const defaultWorkers = 4 + +var ( + // ErrDispatcherNotStarted is returned to a command submitted before Start. + ErrDispatcherNotStarted = errors.New("cdc dispatcher is not started") + // ErrDispatcherStopping is returned to a command submitted during Stop. + ErrDispatcherStopping = errors.New("cdc dispatcher is stopping") + // ErrDispatcherStarted is returned when Start is called for an active run. + ErrDispatcherStarted = errors.New("cdc dispatcher is already started") + // ErrUnknownCommand identifies a command that was not registered by this dispatcher. + ErrUnknownCommand = errors.New("unknown cdc dispatcher command") + // ErrNoSourceStreamer indicates that CDC sources were not installed in the context. + ErrNoSourceStreamer = errors.New("cdc source streamer not available") + // ErrNilSource indicates a corrupt registry entry that claims to exist but + // does not provide a source implementation. + ErrNilSource = errors.New("cdc registry returned a nil source") + // ErrNoRelayNode indicates that the process has no relay transport. + ErrNoRelayNode = errors.New("cdc relay node not available") + // ErrRelayNotCancellable indicates that a relay node cannot bind delivery + // to the subscription lifecycle. Detaching an unowned Send goroutine would + // leak it, so CDC refuses that delivery path instead. + ErrRelayNotCancellable = errors.New("cdc relay node does not support cancellable delivery") +) + +type dispatcherState uint8 + +const ( + stateNew dispatcherState = iota + stateRunning + stateStopping + stateStopped +) + +// Dispatcher routes CDC subscriptions from the process dispatcher to the +// configured source manager. The dispatcher owns subscription relays; a +// driver owns the source and its stream implementation. +type Dispatcher struct { + ctx context.Context + log *zap.Logger + cancel context.CancelFunc + jobs chan dispatchJob + sessions map[uint64]*relaySession + stopDone chan struct{} + // admissionsDone is a per-run barrier. Stop cancels workers first, then + // waits for handles that already passed the state check before workers + // drain the queue. This closes the Handle/Stop admission race without + // holding mu across a potentially blocking queue send. + admissionsDone chan struct{} + workersWG sync.WaitGroup + relaysWG sync.WaitGroup + admissions int + workers int + nextID uint64 + mu sync.Mutex + state dispatcherState +} + +type dispatchJob struct { + ctx context.Context + cmd dispatcher.Command + receiver dispatcher.ResultReceiver + tag uint64 +} + +// DispatcherOption configures a Dispatcher. +type DispatcherOption func(*Dispatcher) + +// WithWorkers sets the number of command workers. Values less than one are +// ignored and leave the default (or previously configured) value unchanged. +func WithWorkers(n int) DispatcherOption { + return func(d *Dispatcher) { + if n > 0 { + d.workers = n + } + } +} + +// WithLogger sets the dispatcher logger. +func WithLogger(log *zap.Logger) DispatcherOption { + return func(d *Dispatcher) { + if log != nil { + d.log = log + } + } +} + +// NewDispatcher creates a CDC dispatcher. +func NewDispatcher(opts ...DispatcherOption) *Dispatcher { + d := &Dispatcher{ + workers: defaultWorkers, + state: stateNew, + sessions: make(map[uint64]*relaySession), + log: zap.NewNop(), + } + for _, opt := range opts { + if opt != nil { + opt(d) + } + } + return d +} + +// Start starts the command workers. A dispatcher can be started again after a +// completed Stop, but cannot be started concurrently with an active run. +func (d *Dispatcher) Start(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + + d.mu.Lock() + if d.state == stateRunning { + d.mu.Unlock() + return ErrDispatcherStarted + } + if d.state == stateStopping { + d.mu.Unlock() + return ErrDispatcherStopping + } + + d.ctx, d.cancel = context.WithCancel(ctx) + d.jobs = make(chan dispatchJob, d.workers*2) + d.sessions = make(map[uint64]*relaySession) + d.stopDone = make(chan struct{}) + d.admissionsDone = make(chan struct{}) + d.admissions = 0 + d.state = stateRunning + for i := 0; i < d.workers; i++ { + d.workersWG.Add(1) + } + runCtx := d.ctx + stopDone := d.stopDone + admissionsDone := d.admissionsDone + d.mu.Unlock() + + for i := 0; i < d.workers; i++ { + go d.worker(runCtx, admissionsDone) + } + go d.monitorContext(runCtx, stopDone) + return nil +} + +// Stop stops workers and all active relays. It is safe to call concurrently +// with Handle and can be called more than once. If ctx expires, cleanup +// continues in the background and a later Stop observes its final result. +func (d *Dispatcher) Stop(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + + d.mu.Lock() + switch d.state { + case stateNew, stateStopped: + d.mu.Unlock() + return nil + case stateStopping: + done := d.stopDone + d.mu.Unlock() + return waitForStop(ctx, done) + case stateRunning: + d.state = stateStopping + done := d.stopDone + cancel := d.cancel + d.closeAdmissionsLocked() + sessions := make([]*relaySession, 0, len(d.sessions)) + for _, session := range d.sessions { + sessions = append(sessions, session) + } + d.mu.Unlock() + + if cancel != nil { + cancel() + } + for _, session := range sessions { + session.stop() + } + go d.finishStop(done) + return waitForStop(ctx, done) + default: + d.mu.Unlock() + return nil + } +} + +// monitorContext turns cancellation of the context supplied to Start into a +// normal dispatcher stop. The stopDone identity prevents a stale monitor from +// stopping a later run after the dispatcher has been restarted. +func (d *Dispatcher) monitorContext(runCtx context.Context, stopDone chan struct{}) { + <-runCtx.Done() + + d.mu.Lock() + valid := d.state == stateRunning && d.stopDone == stopDone + d.mu.Unlock() + if valid { + _ = d.Stop(context.Background()) + } +} + +func (d *Dispatcher) finishStop(done chan struct{}) { + d.workersWG.Wait() + d.relaysWG.Wait() + + d.mu.Lock() + if d.state == stateStopping && d.stopDone == done { + d.state = stateStopped + d.ctx = nil + d.cancel = nil + d.jobs = nil + close(done) + } + d.mu.Unlock() +} + +func waitForStop(ctx context.Context, done <-chan struct{}) error { + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (d *Dispatcher) worker(ctx context.Context, admissionsDone <-chan struct{}) { + defer d.workersWG.Done() + + for { + select { + case job := <-d.jobs: + if ctx.Err() != nil { + complete(job.receiver, job.tag, nil, ErrDispatcherStopping) + continue + } + d.execute(ctx, job) + case <-ctx.Done(): + // A Handle may have passed the running-state check but not yet + // enqueued its job. Wait for those admissions before draining; + // otherwise a send racing cancellation could land in a queue with + // no worker left to complete it. + <-admissionsDone + d.drainJobs() + return + } + } +} + +func (d *Dispatcher) closeAdmissionsLocked() { + if d.admissions == 0 && d.admissionsDone != nil { + select { + case <-d.admissionsDone: + default: + close(d.admissionsDone) + } + } +} + +func (d *Dispatcher) endAdmission(done chan struct{}) { + d.mu.Lock() + d.admissions-- + if d.admissions == 0 && d.state != stateRunning { + select { + case <-done: + default: + close(done) + } + } + d.mu.Unlock() +} + +// drainJobs completes commands accepted before cancellation. Jobs are never +// dropped silently, which prevents a process yield from remaining pending +// when the dispatcher is stopped. +func (d *Dispatcher) drainJobs() { + for { + select { + case job := <-d.jobs: + complete(job.receiver, job.tag, nil, ErrDispatcherStopping) + default: + return + } + } +} + +func (d *Dispatcher) execute(dispatchCtx context.Context, job dispatchJob) { + switch cmd := job.cmd.(type) { + case cdcapi.SubscribeCmd: + d.executeSubscribe(dispatchCtx, job.ctx, cmd, job.tag, job.receiver) + case *cdcapi.SubscribeCmd: + if cmd == nil { + complete(job.receiver, job.tag, nil, fmt.Errorf("%w: nil subscribe command", ErrUnknownCommand)) + return + } + d.executeSubscribe(dispatchCtx, job.ctx, *cmd, job.tag, job.receiver) + default: + complete(job.receiver, job.tag, nil, fmt.Errorf("%w: %T", ErrUnknownCommand, job.cmd)) + } +} + +func (d *Dispatcher) executeSubscribe(dispatchCtx, requestCtx context.Context, cmd cdcapi.SubscribeCmd, tag uint64, receiver dispatcher.ResultReceiver) { + if err := cmd.Options.Validate(); err != nil { + complete(receiver, tag, nil, err) + return + } + // Keep source, relay, and process admission on one finite item limit. + // Lua normalizes this before yielding; direct Go callers get the same + // bounded default here. + cmd.Options.Buffer = cmd.Options.EffectiveMaxStreamItems() + ctx, cancelContext := linkedContext(dispatchCtx, requestCtx) + node := relay.GetNode(ctx) + if node == nil { + cancelContext() + complete(receiver, tag, nil, ErrNoRelayNode) + return + } + + stream, err := d.openStream(ctx, cmd) + if err != nil { + cancelContext() + if dispatchCtx != nil && dispatchCtx.Err() != nil { + err = ErrDispatcherStopping + } + complete(receiver, tag, nil, err) + return + } + if stream == nil { + cancelContext() + complete(receiver, tag, nil, errors.New("cdc source returned a nil stream")) + return + } + + loopCtx, cancelLoop := context.WithCancel(ctx) + session := &relaySession{ + maxBytes: cmd.Options.EffectiveMaxBytes(), + maxItems: cmd.Options.EffectiveMaxStreamItems(), + cancel: func() { + cancelLoop() + cancelContext() + }, + close: stream.Close, + } + if !d.addSession(session) { + session.stop() + complete(receiver, tag, nil, ErrDispatcherStopping) + return + } + + changes := stream.Changes() + go d.relay(loopCtx, session, changes, stream, node, cmd.PID, cmd.Topic, cmd.Source) + + complete(receiver, tag, cdcapi.Subscription{ + Source: cmd.Source, + Topic: cmd.Topic, + Stop: session.stop, + }, nil) +} + +// openStream resolves the canonical system registry first. The legacy +// SourceStreamer path is retained temporarily for callers that predate the +// driver-neutral registry; boot uses the registry path for every driver. +func (d *Dispatcher) openStream(ctx context.Context, cmd cdcapi.SubscribeCmd) (changeStream, error) { + if reg := cdcapi.GetRegistry(ctx); reg != nil { + source, ok := reg.Get(registry.ParseID(cmd.Source)) + if ok { + if source == nil { + return nil, fmt.Errorf("%w: %s", ErrNilSource, cmd.Source) + } + return source.Subscribe(ctx, cmd.Options) + } + + // A registry miss can still be served by a pre-registry caller's + // streamer. A present registry entry, including a corrupt nil entry, + // remains authoritative so legacy aliases cannot shadow canonical IDs. + if streamer := cdcapi.GetSourceStreamer(ctx); streamer != nil { + stream, _, err := streamer.Stream(ctx, cmd.Source, cmd.Options) + return stream, err + } + return nil, fmt.Errorf("%w: %s", cdcapi.ErrSourceNotFound, cmd.Source) + } + + streamer := cdcapi.GetSourceStreamer(ctx) + if streamer == nil { + return nil, ErrNoSourceStreamer + } + stream, _, err := streamer.Stream(ctx, cmd.Source, cmd.Options) + return stream, err +} + +// linkedContext preserves the request context's values (including relay +// routing) while making dispatcher shutdown a second cancellation parent. The +// returned cleanup must be held by the relay session until the stream ends. +func linkedContext(dispatchCtx, requestCtx context.Context) (context.Context, context.CancelFunc) { + if requestCtx == nil { + requestCtx = context.Background() + } + if dispatchCtx == nil { + return context.WithCancel(requestCtx) + } + ctx, cancel := context.WithCancel(requestCtx) + stopPropagation := context.AfterFunc(dispatchCtx, cancel) + return ctx, func() { + stopPropagation() + cancel() + } +} + +func (d *Dispatcher) addSession(session *relaySession) bool { + d.mu.Lock() + defer d.mu.Unlock() + + if d.state != stateRunning { + return false + } + d.nextID++ + session.id = d.nextID + d.sessions[session.id] = session + d.relaysWG.Add(1) + return true +} + +func (d *Dispatcher) relay(ctx context.Context, session *relaySession, changes <-chan cdcapi.Change, stream changeStream, node relay.Node, target pid.PID, topic, source string) { + defer func() { + // Closing a naturally exhausted stream is still the dispatcher's + // ownership responsibility. stop is idempotent and suppresses any + // duplicate terminal caused by the close. + session.stop() + d.relayDone(session.id) + d.relaysWG.Done() + }() + + for { + select { + case change, ok := <-changes: + if !ok { + if err := streamError(stream); err != nil { + d.sendTerminal(ctx, node, target, topic, session.maxItems, session.maxBytes, err) + } else { + d.sendTerminal(ctx, node, target, topic, session.maxItems, session.maxBytes, nil) + } + return + } + + pkg := relay.NewPackage(pid.Zero(), target, topic, payload.New(change)) + pkg.Messages[0].PayloadBytes = cdcapi.EstimateChangeBytes(change) + pkg.Messages[0].MaxBytes = session.maxBytes + pkg.Messages[0].MaxItems = session.maxItems + if err := sendRelay(ctx, node, pkg); err != nil { + d.log.Debug("failed to relay cdc change", + zap.String("source", source), + zap.Error(err)) + // A failed relay cannot make progress. Close the source stream + // and cancel this relay so it cannot retain a worker or source. + session.stop() + return + } + case <-ctx.Done(): + return + } + } +} + +func (d *Dispatcher) relayDone(id uint64) { + d.mu.Lock() + delete(d.sessions, id) + d.mu.Unlock() +} + +func (d *Dispatcher) sendTerminal(ctx context.Context, node relay.Node, target pid.PID, topic string, maxItems int, maxBytes int64, err error) { + var terminal payload.Payloads + if err != nil { + terminal = append(terminal, payload.NewError(err)) + } + terminal = append(terminal, payload.NewTerminal()) + pkg := relay.NewPackage(pid.Zero(), target, topic, terminal...) + pkg.Messages[0].MaxItems = maxItems + pkg.Messages[0].MaxBytes = maxBytes + if sendErr := sendRelay(ctx, node, pkg); sendErr != nil { + d.log.Debug("failed to send cdc terminal", + zap.String("topic", topic), + zap.Error(sendErr)) + } +} + +// sendRelay uses the relay-owned cancellation contract. Starting an +// untracked goroutine around Receiver.Send would make Stop return while that +// goroutine retained the stream, node, and process context indefinitely; a +// node without the capability therefore fails the delivery explicitly. +func sendRelay(ctx context.Context, node relay.Node, pkg *relay.Package) error { + if pkg == nil { + return ErrNoRelayNode + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + relay.ReleasePackage(pkg) + return err + } + if node == nil { + relay.ReleasePackage(pkg) + return ErrNoRelayNode + } + sender, ok := node.(relay.ContextSender) + if !ok { + relay.ReleasePackage(pkg) + return ErrRelayNotCancellable + } + if err := sender.SendContext(ctx, pkg); err != nil { + // ContextSender ownership is transactional: a nil result transfers + // ownership to the destination queue; an error leaves it with the + // caller. CDC is the caller at this boundary, so release exactly once. + relay.ReleasePackage(pkg) + return err + } + return nil +} + +// streamError is an optional extension implemented by streams that can +// report a typed terminal error after their change channel closes. Keeping it +// optional preserves compatibility with the original stream interface while +// allowing all drivers to expose terminal failures consistently. +func streamError(stream changeStream) error { + if s, ok := stream.(interface{ Err() error }); ok { + return s.Err() + } + return nil +} + +type changeStream interface { + Changes() <-chan cdcapi.Change + Close() +} + +func complete(receiver dispatcher.ResultReceiver, tag uint64, data any, err error) { + if receiver != nil { + receiver.CompleteYield(tag, data, err) + } +} + +type relaySession struct { + cancel context.CancelFunc + close func() + maxBytes int64 + maxItems int + id uint64 + once sync.Once +} + +func (s *relaySession) stop() { + s.once.Do(func() { + if s.cancel != nil { + s.cancel() + } + if s.close != nil { + s.close() + } + }) +} + +// Handle queues a command for execution by the dispatcher worker pool. +func (d *Dispatcher) Handle(ctx context.Context, cmd dispatcher.Command, tag uint64, receiver dispatcher.ResultReceiver) error { + if ctx == nil { + ctx = context.Background() + } + + d.mu.Lock() + if d.state != stateRunning { + err := ErrDispatcherNotStarted + if d.state == stateStopping { + err = ErrDispatcherStopping + } + d.mu.Unlock() + complete(receiver, tag, nil, err) + return nil + } + jobs := d.jobs + runCtx := d.ctx + d.admissions++ + admissionsDone := d.admissionsDone + d.mu.Unlock() + + job := dispatchJob{ctx: ctx, cmd: cmd, tag: tag, receiver: receiver} + enqueued := false + select { + case jobs <- job: + enqueued = true + case <-runCtx.Done(): + // The admission barrier keeps workers from draining until this + // in-flight send has resolved, even if both cases are ready. + case <-ctx.Done(): + // Complete below after releasing the admission barrier. + } + d.endAdmission(admissionsDone) + if !enqueued { + err := ErrDispatcherStopping + if runCtx.Err() == nil && ctx.Err() != nil { + err = ctx.Err() + } + complete(receiver, tag, nil, err) + } + return nil +} + +// RegisterAll registers all CDC command handlers with the process dispatcher. +func (d *Dispatcher) RegisterAll(register func(id dispatcher.CommandID, h dispatcher.Handler)) { + if register != nil { + register(cdcapi.Subscribe, dispatcher.HandlerFunc(d.Handle)) + } +} diff --git a/service/cdc/dispatcher_test.go b/service/cdc/dispatcher_test.go new file mode 100644 index 000000000..234dbc6fa --- /dev/null +++ b/service/cdc/dispatcher_test.go @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + cdcapi "github.com/wippyai/runtime/api/service/cdc" + cdcsystem "github.com/wippyai/runtime/system/cdc" +) + +type dispatcherTestStream struct { + changes chan cdcapi.Change + err error + closed atomic.Bool + once sync.Once +} + +func (s *dispatcherTestStream) Changes() <-chan cdcapi.Change { return s.changes } + +func (s *dispatcherTestStream) Close() { + s.once.Do(func() { + s.closed.Store(true) + }) +} + +func (s *dispatcherTestStream) Err() error { return s.err } + +type dispatcherTestSource struct { + stream *dispatcherTestStream + info cdcapi.SourceInfo + subscribeN atomic.Int32 +} + +func (s *dispatcherTestSource) Info() cdcapi.SourceInfo { return s.info } + +func (s *dispatcherTestSource) Subscribe(context.Context, cdcapi.StreamOptions) (cdcapi.Stream, error) { + s.subscribeN.Add(1) + return s.stream, nil +} + +type dispatcherLegacyStreamer struct { + stream *dispatcherTestStream + calls atomic.Int32 +} + +func (s *dispatcherLegacyStreamer) Stream(context.Context, string, cdcapi.StreamOptions) (cdcapi.ChangeStream, cdcapi.SourceInfo, error) { + s.calls.Add(1) + return s.stream, cdcapi.SourceInfo{Name: "legacy"}, nil +} + +type blockingSubscribeSource struct { + started chan struct{} + once sync.Once +} + +func (s *blockingSubscribeSource) Info() cdcapi.SourceInfo { + return cdcapi.SourceInfo{ID: registry.NewID("test", "blocking")} +} + +func (s *blockingSubscribeSource) Subscribe(ctx context.Context, _ cdcapi.StreamOptions) (cdcapi.Stream, error) { + s.once.Do(func() { close(s.started) }) + <-ctx.Done() + return nil, ctx.Err() +} + +type nilSourceRegistry struct{} + +func (nilSourceRegistry) List() []cdcapi.SourceInfo { return nil } + +func (nilSourceRegistry) Get(registry.ID) (cdcapi.Source, bool) { return nil, true } + +type dispatcherTestNode struct { + packages chan *relay.Package + send func(*relay.Package) error + sendContext func(context.Context, *relay.Package) error +} + +func (n *dispatcherTestNode) ID() pid.NodeID { return "cdc-dispatcher-test" } + +func (n *dispatcherTestNode) Send(pkg *relay.Package) error { + if n.send != nil { + return n.send(pkg) + } + n.packages <- pkg + return nil +} + +func (n *dispatcherTestNode) SendContext(ctx context.Context, pkg *relay.Package) error { + if n.sendContext != nil { + return n.sendContext(ctx, pkg) + } + return n.Send(pkg) +} + +func (n *dispatcherTestNode) RegisterHost(pid.HostID, relay.Receiver) error { return nil } +func (n *dispatcherTestNode) UnregisterHost(pid.HostID) {} +func (n *dispatcherTestNode) GetHost(pid.HostID) (relay.Receiver, bool) { return nil, false } +func (n *dispatcherTestNode) Attach(pid.PID, chan *relay.Package) (context.CancelFunc, error) { + return func() {}, nil +} +func (n *dispatcherTestNode) Detach(pid.PID) {} + +type dispatcherTestReceiver struct { + data any + err error + done chan struct{} + once sync.Once +} + +func (r *dispatcherTestReceiver) CompleteYield(_ uint64, data any, err error) { + r.data = data + r.err = err + r.once.Do(func() { close(r.done) }) +} + +func dispatcherTestContext(t *testing.T, source cdcapi.Source, id registry.ID, node relay.Node) context.Context { + t.Helper() + reg := cdcsystem.NewRegistry(nil) + require.NoError(t, reg.Register(id, source, cdcapi.SQLite)) + return dispatcherTestContextWithRegistry(reg, node) +} + +func dispatcherTestContextWithRegistry(reg cdcapi.Registry, node relay.Node) context.Context { + root := ctxapi.NewRootContext() + root = cdcapi.WithRegistry(root, reg) + root = relay.WithNode(root, node) + ctx, _ := ctxapi.OpenFrameContext(root) + return ctx +} + +func dispatcherTestCommand(id registry.ID) cdcapi.SubscribeCmd { + target := pid.PID{Host: "test", UniqID: "cdc"} + return cdcapi.SubscribeCmd{ + PID: target.Precomputed(), + Source: id.String(), + Topic: "cdc@test", + } +} + +func waitResult(t *testing.T, receiver *dispatcherTestReceiver) { + t.Helper() + select { + case <-receiver.done: + case <-time.After(time.Second): + t.Fatal("timed out waiting for dispatcher result") + } +} + +func TestDispatcherUsesSystemRegistryAndRelaysChanges(t *testing.T) { + stream := &dispatcherTestStream{changes: make(chan cdcapi.Change, 1)} + id := registry.NewID("test", "source") + node := &dispatcherTestNode{packages: make(chan *relay.Package, 2)} + ctx := dispatcherTestContext(t, &dispatcherTestSource{stream: stream}, id, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + defer func() { require.NoError(t, d.Stop(context.Background())) }() + + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + + sub, ok := receiver.data.(cdcapi.Subscription) + require.True(t, ok) + require.NotNil(t, sub.Stop) + + stream.changes <- cdcapi.Change{SourceID: id, Source: id.String(), Op: "insert"} + select { + case pkg := <-node.packages: + require.Len(t, pkg.Messages, 1) + require.Len(t, pkg.Messages[0].Payloads, 1) + got, ok := pkg.Messages[0].Payloads[0].Data().(cdcapi.Change) + require.True(t, ok) + assert.Equal(t, "insert", got.Op) + case <-time.After(time.Second): + t.Fatal("timed out waiting for relayed change") + } + + sub.Stop() + assert.Eventually(t, stream.closed.Load, time.Second, 10*time.Millisecond) +} + +func TestDispatcherRegistryPrecedesLegacyStreamer(t *testing.T) { + id := registry.NewID("test", "canonical") + canonical := &dispatcherTestSource{stream: &dispatcherTestStream{changes: make(chan cdcapi.Change)}} + legacy := &dispatcherLegacyStreamer{stream: &dispatcherTestStream{changes: make(chan cdcapi.Change)}} + node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} + ctx := dispatcherTestContext(t, canonical, id, node) + ctx = cdcapi.WithSourceStreamer(ctx, legacy) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + defer func() { require.NoError(t, d.Stop(context.Background())) }() + + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + assert.Equal(t, int32(1), canonical.subscribeN.Load()) + assert.Zero(t, legacy.calls.Load()) + + sub, ok := receiver.data.(cdcapi.Subscription) + require.True(t, ok) + sub.Stop() +} + +func TestDispatcherRegistryMissFallsBackToLegacyStreamer(t *testing.T) { + id := registry.NewID("legacy", "source") + legacyStream := &dispatcherTestStream{changes: make(chan cdcapi.Change)} + legacy := &dispatcherLegacyStreamer{stream: legacyStream} + node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} + reg := cdcsystem.NewRegistry(nil) + ctx := dispatcherTestContextWithRegistry(reg, node) + ctx = cdcapi.WithSourceStreamer(ctx, legacy) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + defer func() { require.NoError(t, d.Stop(context.Background())) }() + + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + cmd := dispatcherTestCommand(id) + require.NoError(t, d.Handle(ctx, cmd, 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + assert.Equal(t, int32(1), legacy.calls.Load()) + + sub, ok := receiver.data.(cdcapi.Subscription) + require.True(t, ok) + sub.Stop() +} + +func TestDispatcherRelaysTypedStreamErrorBeforeTerminal(t *testing.T) { + streamErr := errors.New("capture gap") + stream := &dispatcherTestStream{changes: make(chan cdcapi.Change), err: streamErr} + id := registry.NewID("test", "source") + node := &dispatcherTestNode{packages: make(chan *relay.Package, 2)} + ctx := dispatcherTestContext(t, &dispatcherTestSource{stream: stream}, id, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + defer func() { require.NoError(t, d.Stop(context.Background())) }() + + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + + close(stream.changes) + select { + case pkg := <-node.packages: + require.Len(t, pkg.Messages, 1) + payloads := pkg.Messages[0].Payloads + require.Len(t, payloads, 2) + gotErr, ok := payloads[0].Data().(error) + require.True(t, ok) + assert.ErrorIs(t, gotErr, streamErr) + assert.True(t, payload.IsTerminal(payloads[1])) + case <-time.After(time.Second): + t.Fatal("timed out waiting for typed terminal") + } +} + +func TestDispatcherStopsActiveRelayAndWaitsForCleanup(t *testing.T) { + stream := &dispatcherTestStream{changes: make(chan cdcapi.Change)} + id := registry.NewID("test", "source") + node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} + ctx := dispatcherTestContext(t, &dispatcherTestSource{stream: stream}, id, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + + stopCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, d.Stop(stopCtx)) + assert.True(t, stream.closed.Load()) + + postStop := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 2, postStop)) + waitResult(t, postStop) + assert.ErrorIs(t, postStop.err, ErrDispatcherNotStarted) +} + +func TestDispatcherCancelsRelayAfterNodeFailure(t *testing.T) { + relayErr := errors.New("node unavailable") + var sends atomic.Int32 + node := &dispatcherTestNode{send: func(*relay.Package) error { + sends.Add(1) + return relayErr + }} + stream := &dispatcherTestStream{changes: make(chan cdcapi.Change, 1)} + id := registry.NewID("test", "source") + ctx := dispatcherTestContext(t, &dispatcherTestSource{stream: stream}, id, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + defer func() { require.NoError(t, d.Stop(context.Background())) }() + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + + stream.changes <- cdcapi.Change{Source: id.String(), Op: "insert"} + assert.Eventually(t, stream.closed.Load, time.Second, 10*time.Millisecond) + assert.Equal(t, int32(1), sends.Load()) +} + +func TestDispatcherRejectsNilRegistrySource(t *testing.T) { + id := registry.NewID("test", "nil") + node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} + ctx := dispatcherTestContextWithRegistry(nilSourceRegistry{}, node) + legacy := &dispatcherLegacyStreamer{stream: &dispatcherTestStream{changes: make(chan cdcapi.Change)}} + ctx = cdcapi.WithSourceStreamer(ctx, legacy) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + defer func() { require.NoError(t, d.Stop(context.Background())) }() + + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + assert.ErrorIs(t, receiver.err, ErrNilSource) + assert.Zero(t, legacy.calls.Load()) +} + +func TestDispatcherStopCancelsBlockedRelayDelivery(t *testing.T) { + started := make(chan struct{}) + var startOnce sync.Once + node := &dispatcherTestNode{ + sendContext: func(ctx context.Context, _ *relay.Package) error { + startOnce.Do(func() { close(started) }) + <-ctx.Done() + return ctx.Err() + }, + } + stream := &dispatcherTestStream{changes: make(chan cdcapi.Change, 1)} + id := registry.NewID("test", "blocked-relay") + ctx := dispatcherTestContext(t, &dispatcherTestSource{stream: stream}, id, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + + stream.changes <- cdcapi.Change{Source: id.String(), Op: "insert"} + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("timed out waiting for blocked relay delivery") + } + + stopCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, d.Stop(stopCtx)) + assert.True(t, stream.closed.Load()) +} + +func TestDispatcherStopCancelsBlockingSubscribe(t *testing.T) { + source := &blockingSubscribeSource{started: make(chan struct{})} + id := registry.NewID("test", "blocking") + reg := cdcsystem.NewRegistry(nil) + require.NoError(t, reg.Register(id, source, cdcapi.SQLite)) + node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} + ctx := dispatcherTestContextWithRegistry(reg, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + select { + case <-source.started: + case <-time.After(time.Second): + t.Fatal("timed out waiting for blocking subscription") + } + + stopCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, d.Stop(stopCtx)) + waitResult(t, receiver) + assert.ErrorIs(t, receiver.err, ErrDispatcherStopping) +} + +func TestDispatcherHandleAndStopAreSafeConcurrently(t *testing.T) { + root := ctxapi.NewRootContext() + ctx, _ := ctxapi.OpenFrameContext(root) + d := NewDispatcher(WithWorkers(2)) + require.NoError(t, d.Start(ctx)) + require.ErrorIs(t, d.Start(ctx), ErrDispatcherStarted) + + const commands = 32 + receivers := make([]*dispatcherTestReceiver, commands) + var wg sync.WaitGroup + for i := 0; i < commands; i++ { + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + receivers[i] = receiver + wg.Add(1) + go func(tag uint64, receiver *dispatcherTestReceiver) { + defer wg.Done() + _ = d.Handle(ctx, dispatcherTestCommand(registry.NewID("test", "source")), tag, receiver) + }(uint64(i), receiver) + } + + stopDone := make(chan error, 1) + go func() { stopDone <- d.Stop(context.Background()) }() + wg.Wait() + require.NoError(t, <-stopDone) + for _, receiver := range receivers { + waitResult(t, receiver) + } +} + +func TestDispatcherAdmissionStopCompletesEveryJob(t *testing.T) { + for round := 0; round < 100; round++ { + source := &blockingSubscribeSource{started: make(chan struct{})} + id := registry.NewID("test", "admission") + reg := cdcsystem.NewRegistry(nil) + require.NoError(t, reg.Register(id, source, cdcapi.SQLite)) + node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} + ctx := dispatcherTestContextWithRegistry(reg, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + + const jobs = 16 + receivers := make([]*dispatcherTestReceiver, jobs) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := range receivers { + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + receivers[i] = receiver + wg.Add(1) + go func(tag uint64, receiver *dispatcherTestReceiver) { + defer wg.Done() + <-start + _ = d.Handle(ctx, dispatcherTestCommand(id), tag, receiver) + }(uint64(i), receiver) + } + + close(start) + stopDone := make(chan error, 1) + go func() { stopDone <- d.Stop(context.Background()) }() + wg.Wait() + require.NoError(t, <-stopDone) + for _, receiver := range receivers { + waitResult(t, receiver) + } + } +} diff --git a/service/cdc/manager.go b/service/cdc/manager.go new file mode 100644 index 000000000..9c822f9fc --- /dev/null +++ b/service/cdc/manager.go @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package cdc contains the driver router and lifecycle owner for CDC sources. +// Database-specific packages implement Driver; this package never imports a +// concrete database implementation. +package cdc + +import ( + "context" + "errors" + "fmt" + "reflect" + "sort" + "sync" + + "github.com/wippyai/runtime/api/event" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/resource" + api "github.com/wippyai/runtime/api/service/cdc" + "github.com/wippyai/runtime/api/supervisor" + "go.uber.org/zap" +) + +var ( + ErrRegistryRequired = errors.New("cdc manager: registry is required") + ErrEventBusRequired = errors.New("cdc manager: event bus is required") + ErrDriverRequired = errors.New("cdc manager: driver is required") + ErrUnsupportedKind = errors.New("cdc manager: unsupported source kind") + ErrSourceExists = errors.New("cdc manager: source already exists") + ErrSourceNotFound = errors.New("cdc manager: source not found") + ErrSourceKindChange = errors.New("cdc manager: source kind cannot change") + ErrSourceKindMismatch = errors.New("cdc manager: source kind does not match entry") + ErrExclusiveOwned = errors.New("cdc manager: exclusive resource is already owned") +) + +// Dependencies are the shared collaborators available to concrete drivers. +// The manager owns lifecycle event emission; drivers only construct sources. +type Dependencies struct { + Transcoder payload.Transcoder + Resources resource.Registry + Logger *zap.Logger +} + +// ManagedSource is the internal source returned by a driver. Source +// construction and lifecycle remain separate from the system registry, while +// one value is used by both the public source API and supervisor. +type ManagedSource interface { + api.Source + supervisor.Service +} + +// Registry is the mutable capability the manager needs from the system CDC +// registry. Keeping the concrete system implementation behind this interface +// lets boot and tests inject the canonical registry without coupling the +// service package to a particular registry implementation. +type Registry interface { + api.Registry + Register(registry.ID, api.Source, registry.Kind) error + Unregister(registry.ID) (api.Source, bool) +} + +// Disposable is an optional destructive-delete hook. It is invoked only for +// manager.Delete, while the source remains as a non-subscribable tombstone; +// registry removal commits only after disposal succeeds. Ordinary +// Stop/replacement never calls it, which lets drivers retain durable resources +// such as PostgreSQL replication slots across restart and update while still +// cleaning them up on dynamic uninstall. +type Disposable interface { + Dispose(context.Context) error +} + +// ExclusiveResource identifies a resource that cannot be held by two source +// generations at once (for example, a persistent PostgreSQL replication slot). +// Sources with different keys can be started before the registry swap. Sources +// with the same non-empty key use the slot's stop-start-restart handoff. +type ExclusiveResource interface { + ExclusiveResourceKey() string +} + +// Driver constructs a source for one registry kind. Drivers are injected when +// the manager is built; package initialization must not mutate global routing. +type Driver interface { + Kind() registry.Kind + // Create validates the entry and returns an idle source. It must not start + // network, filesystem, or other durable resources; Start owns activation. + // Once a source is returned, the manager owns cleanup on every later + // registration/update failure. A driver that allocates while constructing + // must clean those allocations before returning an error because no source + // exists for the manager to reclaim. + Create(context.Context, registry.Entry, Dependencies) (ManagedSource, error) +} + +// Option configures a Manager. +type Option func(*Manager) + +// WithDriver injects one or more concrete source drivers. A later driver for +// the same kind intentionally replaces an earlier test or extension driver. +func WithDriver(drivers ...Driver) Option { + return func(m *Manager) { + m.mu.Lock() + defer m.mu.Unlock() + for _, driver := range drivers { + if driver == nil { + continue + } + m.drivers[driver.Kind()] = driver + } + } +} + +// Manager routes registry entries to injected drivers and owns their +// supervisor/system-registry lifecycle. +type Manager struct { + registry Registry + bus event.Bus + deps Dependencies + log *zap.Logger + drivers map[registry.Kind]Driver + leases map[string]resourceLease + ops map[registry.ID]*sourceOperation + // mu protects the injected driver map. Drivers are normally immutable after + // construction; keeping this lock makes test and extension replacement + // safe without holding it across driver calls. + mu sync.RWMutex + // leaseMu is deliberately separate from source operation locks. Lease + // release is synchronous at the resource cleanup commit point and never + // waits behind another source's network or filesystem work. + leaseMu sync.Mutex + leaseSeq uint64 + opsMu sync.Mutex +} + +type resourceLease struct { + id registry.ID + token uint64 +} + +type sourceOperation struct { + mu sync.Mutex + refs int +} + +func NewManager( + reg Registry, + dtt payload.Transcoder, + bus event.Bus, + resources resource.Registry, + log *zap.Logger, + opts ...Option, +) (*Manager, error) { + if reg == nil { + return nil, ErrRegistryRequired + } + if bus == nil { + return nil, ErrEventBusRequired + } + if log == nil { + log = zap.NewNop() + } + m := &Manager{ + registry: reg, + bus: bus, + deps: Dependencies{ + Transcoder: dtt, + Resources: resources, + Logger: log, + }, + log: log, + drivers: make(map[registry.Kind]Driver), + leases: make(map[string]resourceLease), + ops: make(map[registry.ID]*sourceOperation), + } + for _, opt := range opts { + if opt != nil { + opt(m) + } + } + return m, nil +} + +func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { + id := canonicalID(entry.ID) + release := m.lockSource(id) + defer release() + + driver, ok := m.driver(entry.Kind) + if !ok { + return fmt.Errorf("%w: %s", ErrUnsupportedKind, entry.Kind) + } + if _, exists := m.registry.Get(id); exists { + return fmt.Errorf("%w: %s", ErrSourceExists, id.String()) + } + + source, err := driver.Create(ctx, entry, m.deps) + if err != nil { + return err + } + if isNilSource(source) { + return ErrDriverRequired + } + key := exclusiveResourceKey(source) + leaseToken, err := m.reserveLease(key, id) + if err != nil { + _ = stopUnstartedSource(ctx, source) + return err + } + slot := newSourceSlot(id, entry.Kind, source, m.log.With(zap.String("id", id.String()))) + slot.setRetiredCleanupHook(func(retiredKey string, retiredToken uint64) { + m.releaseLease(id, retiredKey, retiredToken) + }) + if err := m.registry.Register(id, slot, entry.Kind); err != nil { + m.releaseLease(id, key, leaseToken) + _ = stopUnstartedSource(ctx, source) + return err + } + m.registerSupervisor(ctx, id, slot) + m.log.Info("added cdc source", zap.String("id", id.String()), zap.String("kind", entry.Kind)) + return nil +} + +func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { + id := canonicalID(entry.ID) + release := m.lockSource(id) + defer release() + + existing, exists := m.registry.Get(id) + if exists { + existingKind, knownKind := sourceKind(existing) + if !knownKind { + return errors.New("cdc manager: registered source has no kind") + } + if existingKind != entry.Kind { + return fmt.Errorf("%w: %s -> %s", ErrSourceKindChange, existingKind, entry.Kind) + } + } + driver, ok := m.driver(entry.Kind) + if !ok { + return fmt.Errorf("%w: %s", ErrUnsupportedKind, entry.Kind) + } + if !exists { + return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) + } + // Build the replacement before changing visibility. A malformed entry or + // failed dependency acquisition leaves the old source untouched. + replacement, err := driver.Create(ctx, entry, m.deps) + if err != nil { + return err + } + if isNilSource(replacement) { + return ErrDriverRequired + } + managedSlot, ok := existing.(*sourceSlot) + if !ok { + _ = stopUnstartedSource(ctx, replacement) + return errors.New("cdc manager: source is not managed by a stable slot") + } + oldKey := exclusiveResourceKey(managedSlot.currentSource()) + newKey := exclusiveResourceKey(replacement) + oldToken := m.leaseToken(oldKey, id) + reservedNew := oldKey != newKey || (newKey != "" && oldToken == 0) + newToken := oldToken + if reservedNew { + if managedSlot.hasRetiredKey(newKey) { + _ = stopUnstartedSource(ctx, replacement) + return ErrSourceBusy + } + newToken, err = m.reserveLease(newKey, id) + if err != nil { + _ = stopUnstartedSource(ctx, replacement) + return err + } + } + oldLifecycle := normalizeLifecycleConfig(managedSlot.LifecycleConfig()) + oldLease := leaseRef{key: oldKey, token: oldToken} + newLease := leaseRef{key: newKey, token: newToken, owned: reservedNew} + if replaceErr := managedSlot.Replace(ctx, replacement, oldLease, newLease); replaceErr != nil { + // A failed handoff never publishes the candidate. If its cleanup also + // failed, the stable slot retains it as retired work and therefore owns + // the candidate lease until Stop/Delete retries that cleanup. + retainedCandidate := managedSlot.hasRetiredSource(replacement) + if reservedNew && !retainedCandidate { + m.releaseLease(id, newKey, newToken) + } + return replaceErr + } + if oldKey != newKey && !managedSlot.hasRetiredKey(oldKey) { + m.releaseLease(id, oldKey, oldToken) + } + m.reconfigureSupervisorIfChanged(ctx, id, managedSlot, oldLifecycle) + m.log.Info("updated cdc source", zap.String("id", id.String()), zap.String("kind", entry.Kind)) + return nil +} + +func (m *Manager) reconfigureSupervisorIfChanged(ctx context.Context, id registry.ID, source *sourceSlot, old supervisor.LifecycleConfig) { + newLifecycle := normalizeLifecycleConfig(source.LifecycleConfig()) + if reflect.DeepEqual(old, newLifecycle) { + return + } + // ServiceUpdate is emitted by the supervisor for status changes and is not + // a reconfiguration primitive. Keep the canonical remove/register pair; + // the supervisor transaction retains both operations for a same-ID + // replacement and rebuilds the controller through its normal sequencer. + m.unregisterSupervisor(ctx, id) + m.registerSupervisorWithConfig(ctx, id, source, newLifecycle) +} + +func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { + id := canonicalID(entry.ID) + release := m.lockSource(id) + defer release() + + source, ok := m.registry.Get(id) + if !ok { + return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) + } + if entry.Kind != "" { + kind, known := sourceKind(source) + if !known || kind != entry.Kind { + return fmt.Errorf("%w: %s", ErrSourceKindMismatch, entry.Kind) + } + } + var err error + if disposable, ok := source.(Disposable); ok { + err = disposable.Dispose(ctx) + } else { + err = stopSource(ctx, source) + } + if err != nil { + m.log.Warn("cdc source failed to stop during delete", + zap.String("id", id.String()), zap.Error(err)) + return err + } + if _, ok := m.registry.Unregister(id); !ok { + return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) + } + if slot, ok := source.(*sourceSlot); ok { + for _, key := range slot.resourceKeys() { + m.releaseLease(id, key, m.leaseToken(key, id)) + } + } + m.unregisterSupervisor(ctx, id) + m.log.Info("removed cdc source", zap.String("id", id.String())) + return nil +} + +func (m *Manager) List() []api.SourceInfo { + return m.registry.List() +} + +func (m *Manager) Get(id registry.ID) (api.Source, bool) { + return m.registry.Get(id) +} + +func (m *Manager) driver(kind registry.Kind) (Driver, bool) { + m.mu.RLock() + driver, ok := m.drivers[kind] + m.mu.RUnlock() + return driver, ok +} + +func (m *Manager) lockSource(id registry.ID) func() { + m.opsMu.Lock() + op := m.ops[id] + if op == nil { + op = &sourceOperation{} + m.ops[id] = op + } + op.refs++ + m.opsMu.Unlock() + + op.mu.Lock() + return func() { + m.opsMu.Lock() + op.refs-- + if op.refs == 0 { + delete(m.ops, id) + } + op.mu.Unlock() + m.opsMu.Unlock() + } +} + +func (m *Manager) registerSupervisor(ctx context.Context, id registry.ID, source ManagedSource) { + cfg := supervisor.LifecycleConfig{} + if configured, ok := source.(interface { + LifecycleConfig() supervisor.LifecycleConfig + }); ok { + cfg = configured.LifecycleConfig() + } + m.registerSupervisorWithConfig(ctx, id, source, normalizeLifecycleConfig(cfg)) +} + +func (m *Manager) registerSupervisorWithConfig(ctx context.Context, id registry.ID, source ManagedSource, cfg supervisor.LifecycleConfig) { + m.bus.Send(ctx, event.Event{ + System: supervisor.System, + Kind: supervisor.ServiceRegister, + Path: id.String(), + Data: &supervisor.Entry{ + Service: source, + Config: cfg, + }, + }) +} + +func (m *Manager) unregisterSupervisor(ctx context.Context, id registry.ID) { + m.bus.Send(ctx, event.Event{ + System: supervisor.System, + Kind: supervisor.ServiceRemove, + Path: id.String(), + }) +} + +func (m *Manager) reserveLease(key string, id registry.ID) (uint64, error) { + m.leaseMu.Lock() + defer m.leaseMu.Unlock() + return m.reserveLeaseLocked(key, id) +} + +func (m *Manager) reserveLeaseLocked(key string, id registry.ID) (uint64, error) { + if key == "" { + return 0, nil + } + if owner, ok := m.leases[key]; ok { + if owner.id != id { + return 0, fmt.Errorf("%w: %s (owner %s)", ErrExclusiveOwned, key, owner.id) + } + return owner.token, nil + } + m.leaseSeq++ + owner := resourceLease{id: id, token: m.leaseSeq} + m.leases[key] = owner + return owner.token, nil +} + +func (m *Manager) releaseLease(id registry.ID, key string, token uint64) { + m.leaseMu.Lock() + m.releaseLeaseLocked(id, key, token) + m.leaseMu.Unlock() +} + +func (m *Manager) releaseLeaseLocked(id registry.ID, key string, token uint64) { + if key == "" { + return + } + if owner, ok := m.leases[key]; ok && owner.id == id && owner.token == token { + delete(m.leases, key) + } +} + +func (m *Manager) leaseTokenLocked(key string, id registry.ID) uint64 { + if key == "" { + return 0 + } + if owner, ok := m.leases[key]; ok && owner.id == id { + return owner.token + } + return 0 +} + +func (m *Manager) leaseToken(key string, id registry.ID) uint64 { + m.leaseMu.Lock() + defer m.leaseMu.Unlock() + return m.leaseTokenLocked(key, id) +} + +func stopSource(ctx context.Context, source api.Source) error { + if isNilSource(source) { + return nil + } + if managed, ok := source.(supervisor.Service); ok { + return managed.Stop(ctx) + } + return nil +} + +func isNilSource(source api.Source) bool { + if source == nil { + return true + } + v := reflect.ValueOf(source) + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return v.IsNil() + default: + return false + } +} + +func canonicalID(id registry.ID) registry.ID { + return registry.ParseID(id.String()) +} + +func sourceKind(source api.Source) (registry.Kind, bool) { + if isNilSource(source) { + return "", false + } + if slot, ok := source.(*sourceSlot); ok { + return slot.kind, slot.kind != "" + } + info := source.Info() + return info.Kind, info.Kind != "" +} + +func normalizeLifecycleConfig(cfg supervisor.LifecycleConfig) supervisor.LifecycleConfig { + cfg.InitDefaults() + dependencies := cfg.RequiredServices() + sort.Strings(dependencies) + cfg.Requires = dependencies + cfg.DependsOn = nil + return cfg +} + +var ( + _ registry.EntryListener = (*Manager)(nil) + _ api.Registry = (*Manager)(nil) +) diff --git a/service/cdc/manager_test.go b/service/cdc/manager_test.go new file mode 100644 index 000000000..e6566962d --- /dev/null +++ b/service/cdc/manager_test.go @@ -0,0 +1,1264 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/event" + "github.com/wippyai/runtime/api/registry" + api "github.com/wippyai/runtime/api/service/cdc" + "github.com/wippyai/runtime/api/supervisor" + cdcsystem "github.com/wippyai/runtime/system/cdc" + "github.com/wippyai/runtime/system/eventbus" +) + +type testStream struct { + changes chan api.Change +} + +func (s *testStream) Changes() <-chan api.Change { return s.changes } +func (s *testStream) Close() { close(s.changes) } +func (s *testStream) Err() error { return nil } + +type managedTestSource struct { + startErr error + stopErr error + stream *testStream + lifecycle *supervisor.LifecycleConfig + onStart func() + exclusive string + info api.SourceInfo + startCount atomic.Int32 + stopCount atomic.Int32 + active atomic.Int32 + maxActive atomic.Int32 +} + +type disposableTestSource struct { + disposeErr error + *managedTestSource + disposeCount atomic.Int32 + failedOnce atomic.Bool +} + +type blockingStopSource struct { + *managedTestSource + stopEntered chan struct{} + releaseStop chan struct{} + stopOnce sync.Once +} + +type startupSnapshotSource struct { + *managedTestSource + snapshot api.Change +} + +func newBlockingStopSource(source *managedTestSource) *blockingStopSource { + return &blockingStopSource{ + managedTestSource: source, + stopEntered: make(chan struct{}), + releaseStop: make(chan struct{}), + } +} + +func (s *startupSnapshotSource) Start(ctx context.Context) (<-chan any, error) { + status, err := s.managedTestSource.Start(ctx) + if err != nil { + return status, err + } + s.stream.changes <- s.snapshot + return status, nil +} + +func (s *blockingStopSource) Stop(ctx context.Context) error { + s.stopOnce.Do(func() { close(s.stopEntered) }) + select { + case <-s.releaseStop: + case <-ctx.Done(): + return ctx.Err() + } + return s.managedTestSource.Stop(ctx) +} + +func (s *disposableTestSource) Dispose(ctx context.Context) error { + s.disposeCount.Add(1) + if s.disposeErr != nil && s.failedOnce.CompareAndSwap(false, true) { + return s.disposeErr + } + return s.Stop(ctx) +} + +func (s *managedTestSource) Info() api.SourceInfo { return s.info } + +func (s *managedTestSource) Subscribe(context.Context, api.StreamOptions) (api.Stream, error) { + if s.stream != nil { + return s.stream, nil + } + return &testStream{changes: make(chan api.Change)}, nil +} + +func (s *managedTestSource) Start(context.Context) (<-chan any, error) { + s.startCount.Add(1) + if s.onStart != nil { + s.onStart() + } + if s.startErr == nil { + active := s.active.Add(1) + for { + max := s.maxActive.Load() + if active <= max || s.maxActive.CompareAndSwap(max, active) { + break + } + } + } + return nil, s.startErr +} + +func (s *managedTestSource) Stop(context.Context) error { + s.stopCount.Add(1) + if s.active.Load() > 0 { + s.active.Add(-1) + } + return s.stopErr +} + +func (s *managedTestSource) LifecycleConfig() supervisor.LifecycleConfig { + if s.lifecycle != nil { + return *s.lifecycle + } + return supervisor.LifecycleConfig{AutoStart: true} +} + +func (s *managedTestSource) ExclusiveResourceKey() string { return s.exclusive } + +type testDriver struct { + create func(registry.Entry) (ManagedSource, error) + kind registry.Kind +} + +func (d testDriver) Kind() registry.Kind { return d.kind } + +func (d testDriver) Create(_ context.Context, entry registry.Entry, _ Dependencies) (ManagedSource, error) { + return d.create(entry) +} + +type recordingBus struct { + events []event.Event + mu sync.Mutex +} + +func (b *recordingBus) Subscribe(context.Context, event.System, chan<- event.Event) (event.SubscriberID, error) { + return "", nil +} + +func (b *recordingBus) SubscribeP(context.Context, event.System, event.Kind, chan<- event.Event) (event.SubscriberID, error) { + return "", nil +} + +func (*recordingBus) Unsubscribe(context.Context, event.SubscriberID) {} + +func (b *recordingBus) Send(_ context.Context, e event.Event) { + b.mu.Lock() + b.events = append(b.events, e) + b.mu.Unlock() +} + +func (b *recordingBus) snapshot() []event.Event { + b.mu.Lock() + defer b.mu.Unlock() + return append([]event.Event(nil), b.events...) +} + +func newManagerTest(t *testing.T, drivers ...Driver) (*Manager, *eventbus.Bus) { + t.Helper() + bus := eventbus.NewBus() + m, err := NewManager(cdcsystem.NewRegistry(nil), nil, bus, nil, nil, WithDriver(drivers...)) + require.NoError(t, err) + t.Cleanup(bus.Stop) + return m, bus +} + +func TestNewManagerRequiresRegistryAndBus(t *testing.T) { + bus := eventbus.NewBus() + t.Cleanup(bus.Stop) + _, err := NewManager(nil, nil, bus, nil, nil) + require.ErrorIs(t, err, ErrRegistryRequired) + _, err = NewManager(cdcsystem.NewRegistry(nil), nil, nil, nil, nil) + require.ErrorIs(t, err, ErrEventBusRequired) +} + +func TestManagerRoutesCanonicalIDsAndOwnsLifecycle(t *testing.T) { + var created []*managedTestSource + driver := testDriver{ + kind: "db.cdc.test", + create: func(entry registry.Entry) (ManagedSource, error) { + source := &managedTestSource{info: api.SourceInfo{Name: "driver-name", Generation: entry.ID.String()}} + created = append(created, source) + return source, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.ID{NS: "app", Name: "events"} + entry := registry.Entry{ID: id, Kind: driver.kind} + + require.NoError(t, m.Add(context.Background(), entry)) + got, ok := m.Get(registry.ParseID("app:events")) + require.True(t, ok) + slot, ok := got.(*sourceSlot) + require.True(t, ok) + slot.mu.RLock() + require.Same(t, created[0], slot.current) + slot.mu.RUnlock() + require.Equal(t, "app:events", m.List()[0].ID.String()) + require.Equal(t, driver.kind, m.List()[0].Kind) + require.ErrorIs(t, m.Add(context.Background(), entry), ErrSourceExists) + + require.NoError(t, m.Delete(context.Background(), registry.Entry{ID: registry.ParseID("app:events")})) + require.EqualValues(t, 1, created[0].stopCount.Load()) + _, ok = m.Get(id) + require.False(t, ok) +} + +func TestManagerPreStartSubscriptionsReceiveIndependentStartupSnapshots(t *testing.T) { + kind := registry.Kind("db.cdc.test") + aID := registry.NewID("app", "db-a") + bID := registry.NewID("app", "db-b") + aSource := &startupSnapshotSource{ + managedTestSource: &managedTestSource{ + info: api.SourceInfo{Snapshot: true}, + stream: &testStream{changes: make(chan api.Change, 1)}, + }, + snapshot: api.Change{Op: "snapshot", Table: "a"}, + } + bSource := &startupSnapshotSource{ + managedTestSource: &managedTestSource{ + info: api.SourceInfo{Snapshot: true}, + stream: &testStream{changes: make(chan api.Change, 1)}, + }, + snapshot: api.Change{Op: "snapshot", Table: "b"}, + } + driver := testDriver{ + kind: kind, + create: func(entry registry.Entry) (ManagedSource, error) { + switch entry.ID { + case aID: + return aSource, nil + case bID: + return bSource, nil + default: + return nil, errors.New("unexpected source id") + } + }, + } + m, _ := newManagerTest(t, driver) + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: aID, Kind: kind})) + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: bID, Kind: kind})) + + aStream, err := mustSlot(t, m, aID).Subscribe(context.Background(), api.StreamOptions{}) + require.NoError(t, err) + bStream, err := mustSlot(t, m, bID).Subscribe(context.Background(), api.StreamOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + aStream.Close() + bStream.Close() + }) + + _, err = mustSlot(t, m, aID).Start(context.Background()) + require.NoError(t, err) + _, err = mustSlot(t, m, bID).Start(context.Background()) + require.NoError(t, err) + + select { + case change := <-aStream.Changes(): + require.Equal(t, "a", change.Table) + require.Equal(t, aID, change.SourceID) + require.Equal(t, aID.String(), change.Source) + case <-time.After(time.Second): + t.Fatal("source A startup snapshot was lost") + } + select { + case change := <-bStream.Changes(): + require.Equal(t, "b", change.Table) + require.Equal(t, bID, change.SourceID) + require.Equal(t, bID.String(), change.Source) + case <-time.After(time.Second): + t.Fatal("source B startup snapshot was lost") + } + require.NoError(t, m.Delete(context.Background(), registry.Entry{ID: aID, Kind: kind})) + require.NoError(t, m.Delete(context.Background(), registry.Entry{ID: bID, Kind: kind})) +} + +func TestManagerDeleteInvokesDisposeOnlyAfterUnregister(t *testing.T) { + source := &disposableTestSource{managedTestSource: &managedTestSource{info: api.SourceInfo{Name: "source"}}} + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { return source, nil }, + } + bus := &recordingBus{} + m, err := NewManager(cdcsystem.NewRegistry(nil), nil, bus, nil, nil, WithDriver(driver)) + require.NoError(t, err) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Delete(context.Background(), entry)) + + require.EqualValues(t, 1, source.disposeCount.Load()) + require.EqualValues(t, 1, source.stopCount.Load()) + _, ok := m.Get(id) + require.False(t, ok) + events := bus.snapshot() + require.Len(t, events, 2) + require.Equal(t, supervisor.ServiceRegister, events[0].Kind) + require.Equal(t, supervisor.ServiceRemove, events[1].Kind) +} + +func TestManagerDeleteRetainsTombstoneForDisposeRetry(t *testing.T) { + source := &disposableTestSource{ + managedTestSource: &managedTestSource{info: api.SourceInfo{Name: "source"}}, + disposeErr: errors.New("cleanup failed"), + } + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { return source, nil }, + } + bus := &recordingBus{} + m, err := NewManager(cdcsystem.NewRegistry(nil), nil, bus, nil, nil, WithDriver(driver)) + require.NoError(t, err) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + + require.EqualError(t, m.Delete(context.Background(), entry), "cleanup failed") + _, ok := m.Get(id) + require.True(t, ok, "failed disposal must leave a retryable tombstone") + require.Len(t, bus.snapshot(), 1, "supervisor removal waits for successful disposal") + + require.NoError(t, m.Delete(context.Background(), entry)) + _, ok = m.Get(id) + require.False(t, ok) + require.EqualValues(t, 2, source.disposeCount.Load()) + require.EqualValues(t, 1, source.stopCount.Load()) + events := bus.snapshot() + require.Len(t, events, 2) + require.Equal(t, supervisor.ServiceRemove, events[1].Kind) +} + +func TestManagerUpdateBuildFailureLeavesOldSource(t *testing.T) { + old := &managedTestSource{info: api.SourceInfo{Name: "old"}} + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + return nil, errors.New("candidate rejected") + }, + } + m, _ := newManagerTest(t, driver) + // Seed the system registry through the same manager with a temporary + // successful driver, then switch the injected driver to the failing one. + seed := testDriver{kind: driver.kind, create: func(registry.Entry) (ManagedSource, error) { return old, nil }} + m.drivers[driver.kind] = seed + id := registry.NewID("app", "events") + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: id, Kind: driver.kind})) + m.drivers[driver.kind] = driver + + err := m.Update(context.Background(), registry.Entry{ID: id, Kind: driver.kind}) + require.EqualError(t, err, "candidate rejected") + got, ok := m.Get(id) + require.True(t, ok) + slot, ok := got.(*sourceSlot) + require.True(t, ok) + slot.mu.RLock() + require.Same(t, old, slot.current) + slot.mu.RUnlock() + require.EqualValues(t, 0, old.stopCount.Load()) +} + +func TestManagerUpdateRejectsEntryKindChange(t *testing.T) { + oldKind := registry.Kind("db.cdc.old") + newKind := registry.Kind("db.cdc.new") + var replacementCreates atomic.Int32 + old := &managedTestSource{info: api.SourceInfo{Engine: "old"}} + oldDriver := testDriver{ + kind: oldKind, + create: func(registry.Entry) (ManagedSource, error) { return old, nil }, + } + newDriver := testDriver{ + kind: newKind, + create: func(registry.Entry) (ManagedSource, error) { + replacementCreates.Add(1) + return &managedTestSource{info: api.SourceInfo{Engine: "new"}}, nil + }, + } + m, _ := newManagerTest(t, oldDriver) + m.drivers[newKind] = newDriver + id := registry.NewID("app", "events") + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: id, Kind: oldKind})) + + err := m.Update(context.Background(), registry.Entry{ID: id, Kind: newKind}) + require.ErrorIs(t, err, ErrSourceKindChange) + require.EqualValues(t, 0, replacementCreates.Load(), "a kind-changing update must not construct another driver") + slot := mustSlot(t, m, id) + require.Same(t, old, slot.currentSource()) +} + +func TestManagerUpdateAtomicallyReplacesAndStopsOld(t *testing.T) { + var next int + var created []*managedTestSource + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + source := &managedTestSource{info: api.SourceInfo{Name: string(rune('a' + next))}} + created = append(created, source) + return source, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Update(context.Background(), entry)) + require.Len(t, created, 2) + got, ok := m.Get(id) + require.True(t, ok) + slot, ok := got.(*sourceSlot) + require.True(t, ok) + slot.mu.RLock() + require.Same(t, created[1], slot.current) + slot.mu.RUnlock() + require.EqualValues(t, 1, created[0].stopCount.Load()) +} + +func TestManagerUpdateDoesNotPublishAfterOldStopFails(t *testing.T) { + old := &managedTestSource{info: api.SourceInfo{Name: "old"}, stopErr: errors.New("old cleanup failed")} + newSource := &managedTestSource{info: api.SourceInfo{Name: "new"}} + next := 0 + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return newSource, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.EqualError(t, m.Update(context.Background(), entry), "old cleanup failed") + require.Same(t, old, mustSlot(t, m, id).currentSource()) + require.EqualValues(t, 1, old.stopCount.Load()) + require.EqualValues(t, 1, newSource.stopCount.Load()) +} + +func TestManagerUpdateKeepsStableSupervisorRegistration(t *testing.T) { + var next int + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + return &managedTestSource{info: api.SourceInfo{Name: string(rune('a' + next))}}, nil + }, + } + bus := &recordingBus{} + m, err := NewManager(cdcsystem.NewRegistry(nil), nil, bus, nil, nil, WithDriver(driver)) + require.NoError(t, err) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Update(context.Background(), entry)) + + events := bus.snapshot() + require.Len(t, events, 1) + require.Equal(t, supervisor.ServiceRegister, events[0].Kind) + require.Equal(t, id.String(), events[0].Path) + registered := events[0].Data.(*supervisor.Entry) + current, ok := m.Get(id) + require.True(t, ok) + require.Same(t, current, registered.Service) +} + +func TestManagerUpdateReRegistersSupervisorWhenLifecycleChanges(t *testing.T) { + oldLifecycle := supervisor.LifecycleConfig{ + AutoStart: true, + Requires: []string{"service-a"}, + } + newLifecycle := supervisor.LifecycleConfig{ + AutoStart: false, + DependsOn: []string{"service-b"}, + StartTimeout: 20 * time.Second, + } + old := &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + lifecycle: &oldLifecycle, + } + candidate := &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + lifecycle: &newLifecycle, + } + next := 0 + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + bus := &recordingBus{} + m, err := NewManager(cdcsystem.NewRegistry(nil), nil, bus, nil, nil, WithDriver(driver)) + require.NoError(t, err) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Update(context.Background(), entry)) + + events := bus.snapshot() + require.Len(t, events, 3) + require.Equal(t, supervisor.ServiceRegister, events[0].Kind) + require.Equal(t, supervisor.ServiceRemove, events[1].Kind) + require.Equal(t, supervisor.ServiceRegister, events[2].Kind) + require.Equal(t, id.String(), events[1].Path) + require.Equal(t, id.String(), events[2].Path) + registered := events[2].Data.(*supervisor.Entry) + require.Same(t, mustSlot(t, m, id), registered.Service) + require.False(t, registered.Config.AutoStart) + require.Equal(t, []string{"service-b"}, registered.Config.Requires) + require.Empty(t, registered.Config.DependsOn) +} + +func TestManagerExclusiveResourceLeaseAcrossIDs(t *testing.T) { + kind := registry.Kind("db.cdc.test") + driver := testDriver{ + kind: kind, + create: func(entry registry.Entry) (ManagedSource, error) { + return &managedTestSource{info: api.SourceInfo{Name: entry.ID.String()}, exclusive: "cluster/slot"}, nil + }, + } + m, _ := newManagerTest(t, driver) + first := registry.NewID("app", "first") + second := registry.NewID("app", "second") + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: first, Kind: kind})) + require.ErrorIs(t, m.Add(context.Background(), registry.Entry{ID: second, Kind: kind}), ErrExclusiveOwned) + require.NoError(t, m.Delete(context.Background(), registry.Entry{ID: first, Kind: kind})) + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: second, Kind: kind})) +} + +func TestManagerConcurrentSourcesDoNotSerializeAndRemovalPreservesOthers(t *testing.T) { + kind := registry.Kind("db.cdc.test") + aID := registry.NewID("app", "db-a") + bID := registry.NewID("app", "db-b") + aOld := newBlockingStopSource(&managedTestSource{ + info: api.SourceInfo{Name: "db-a-old"}, + exclusive: "slot-a", + }) + aNew := &managedTestSource{info: api.SourceInfo{Name: "db-a-new"}, exclusive: "slot-a-new"} + bOld := &managedTestSource{info: api.SourceInfo{Name: "db-b-old"}, exclusive: "slot-b"} + bNew := &managedTestSource{info: api.SourceInfo{Name: "db-b-new"}, exclusive: "slot-b"} + var aCreates atomic.Int32 + var bCreates atomic.Int32 + driver := testDriver{ + kind: kind, + create: func(entry registry.Entry) (ManagedSource, error) { + switch entry.ID { + case aID: + if aCreates.Add(1) == 1 { + return aOld, nil + } + return aNew, nil + case bID: + if bCreates.Add(1) == 1 { + return bOld, nil + } + return bNew, nil + default: + return nil, errors.New("unexpected source id") + } + }, + } + bus := &recordingBus{} + m, err := NewManager(cdcsystem.NewRegistry(nil), nil, bus, nil, nil, WithDriver(driver)) + require.NoError(t, err) + entryA := registry.Entry{ID: aID, Kind: kind} + entryB := registry.Entry{ID: bID, Kind: kind} + require.NoError(t, m.Add(context.Background(), entryA)) + require.NoError(t, m.Add(context.Background(), entryB)) + _, err = mustSlot(t, m, aID).Start(context.Background()) + require.NoError(t, err) + _, err = mustSlot(t, m, bID).Start(context.Background()) + require.NoError(t, err) + + aUpdateDone := make(chan error, 1) + go func() { aUpdateDone <- m.Update(context.Background(), entryA) }() + select { + case <-aOld.stopEntered: + case <-time.After(time.Second): + t.Fatal("source A update did not reach its blocking Stop") + } + + bUpdateDone := make(chan error, 1) + go func() { bUpdateDone <- m.Update(context.Background(), entryB) }() + select { + case err := <-bUpdateDone: + require.NoError(t, err, "source B update must not wait for source A") + case <-time.After(time.Second): + t.Fatal("source B update was serialized behind source A") + } + require.Same(t, bNew, mustSlot(t, m, bID).currentSource()) + require.Same(t, aOld, mustSlot(t, m, aID).currentSource()) + + close(aOld.releaseStop) + select { + case err := <-aUpdateDone: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("source A update did not finish after Stop release") + } + require.Same(t, aNew, mustSlot(t, m, aID).currentSource()) + + require.NoError(t, m.Delete(context.Background(), entryA)) + _, aExists := m.Get(aID) + require.False(t, aExists) + _, bExists := m.Get(bID) + require.True(t, bExists, "removing source A must preserve source B") + m.leaseMu.Lock() + _, aLease := m.leases["slot-a-new"] + bLease, bLeaseHeld := m.leases["slot-b"] + m.leaseMu.Unlock() + require.False(t, aLease) + require.True(t, bLeaseHeld) + require.Equal(t, bID, bLease.id) + events := bus.snapshot() + for _, event := range events { + if event.Kind == supervisor.ServiceRemove { + require.Equal(t, aID.String(), event.Path, "source B supervisor registration must remain") + } + } + require.NoError(t, m.Delete(context.Background(), entryB)) +} + +func TestManagerUpdateRejectsExclusiveResourceOwnedByAnotherID(t *testing.T) { + kind := registry.Kind("db.cdc.test") + first := registry.NewID("app", "first") + second := registry.NewID("app", "second") + old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-old"} + other := &managedTestSource{info: api.SourceInfo{Name: "other"}, exclusive: "slot-new"} + candidate := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + exclusive: "slot-new", + }} + next := map[string]int{} + driver := testDriver{ + kind: kind, + create: func(entry registry.Entry) (ManagedSource, error) { + next[entry.ID.String()]++ + if entry.ID == first { + if next[entry.ID.String()] == 1 { + return old, nil + } + return candidate, nil + } + return other, nil + }, + } + m, _ := newManagerTest(t, driver) + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: first, Kind: kind})) + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: second, Kind: kind})) + + require.ErrorIs(t, m.Update(context.Background(), registry.Entry{ID: first, Kind: kind}), ErrExclusiveOwned) + require.Same(t, old, mustSlot(t, m, first).currentSource()) + require.EqualValues(t, 1, candidate.stopCount.Load(), "a lease conflict must stop the uncommitted candidate") + require.EqualValues(t, 0, candidate.disposeCount.Load(), "an unstarted candidate must never destructively dispose a shared resource") +} + +func TestManagerDeleteChecksEntryKind(t *testing.T) { + kind := registry.Kind("db.cdc.test") + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { return &managedTestSource{}, nil }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: id, Kind: kind})) + require.ErrorIs(t, m.Delete(context.Background(), registry.Entry{ID: id, Kind: "db.cdc.other"}), ErrSourceKindMismatch) + _, ok := m.Get(id) + require.True(t, ok) +} + +func TestManagerUpdateDisposesDifferentExclusiveResource(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + exclusive: "slot-old", + }} + candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-new"} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Update(context.Background(), entry)) + require.EqualValues(t, 1, old.disposeCount.Load()) + require.EqualValues(t, 2, old.stopCount.Load(), "replacement stops before destructive disposal and Dispose remains idempotent") + require.EqualValues(t, 1, candidate.startCount.Load()) + require.Same(t, candidate, mustSlot(t, m, id).currentSource()) + require.NotContains(t, m.leases, "slot-old") +} + +func TestManagerUpdateSameExclusiveResourceNeverDisposesOld(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + exclusive: "slot-shared", + }} + candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-shared"} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Update(context.Background(), entry)) + require.EqualValues(t, 0, old.disposeCount.Load()) + require.EqualValues(t, 0, old.stopCount.Load(), "an idle same-key source is retained without destructive cleanup") +} + +func TestManagerUpdateRetriesFailedRetiredDisposalBeforeRestart(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + exclusive: "slot-old", + }, disposeErr: errors.New("retired cleanup failed")} + candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-new"} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.EqualError(t, m.Update(context.Background(), entry), "retired cleanup failed") + slot := mustSlot(t, m, id) + require.Same(t, old, slot.currentSource(), "the candidate must not become visible before old disposal succeeds") + require.Equal(t, slotFaulted, slot.state) + require.EqualValues(t, 1, candidate.stopCount.Load()) + require.Contains(t, m.leases, "slot-old") + + require.NoError(t, slot.Stop(context.Background())) + require.EqualValues(t, 2, old.disposeCount.Load(), "shutdown Stop must retry the failed destructive cleanup") + require.EqualValues(t, 1, candidate.stopCount.Load()) + require.Eventually(t, func() bool { + m.leaseMu.Lock() + defer m.leaseMu.Unlock() + _, ok := m.leases["slot-old"] + return !ok + }, time.Second, time.Millisecond) + require.Equal(t, slotStopped, slot.state) +} + +func TestManagerDeleteRetriesRetiredDisposalBeforeUnregister(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + exclusive: "slot-old", + }, disposeErr: errors.New("retired cleanup failed")} + candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-new"} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.EqualError(t, m.Update(context.Background(), entry), "retired cleanup failed") + require.NoError(t, m.Delete(context.Background(), entry)) + _, ok := m.Get(id) + require.False(t, ok, "delete must unregister only after current and retired cleanup") + require.EqualValues(t, 2, old.disposeCount.Load()) +} + +func TestManagerRetiredLeaseTokenCannotReleaseReplacement(t *testing.T) { + m, _ := newManagerTest(t) + id := registry.NewID("app", "events") + m.leaseMu.Lock() + m.leaseSeq = 1 + m.leases["slot"] = resourceLease{id: id, token: 1} + m.releaseLeaseLocked(id, "slot", 1) + newToken, err := m.reserveLeaseLocked("slot", id) + require.NoError(t, err) + require.NotEqual(t, uint64(1), newToken) + m.releaseLeaseLocked(id, "slot", 1) + owner, ok := m.leases["slot"] + m.leaseMu.Unlock() + require.True(t, ok) + require.Equal(t, newToken, owner.token) +} + +func TestManagerUpdateFailedStartRetainsRunningGeneration(t *testing.T) { + var next int + old := &managedTestSource{info: api.SourceInfo{Name: "old"}} + failed := &managedTestSource{info: api.SourceInfo{Name: "failed"}, startErr: errors.New("candidate start failed")} + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return failed, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + source, ok := m.Get(id) + require.True(t, ok) + slot := source.(*sourceSlot) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + + require.EqualError(t, m.Update(context.Background(), entry), "candidate start failed") + slot.mu.RLock() + require.Same(t, old, slot.current) + require.Equal(t, slotRunning, slot.state) + slot.mu.RUnlock() + require.EqualValues(t, 1, old.stopCount.Load(), "the old running generation must be stopped before a candidate can start") + require.Equal(t, "2", slot.Info().Generation) +} + +func TestManagerUpdateSameExclusiveKeyStopsAndRestores(t *testing.T) { + old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-1"} + candidate := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + exclusive: "slot-1", + startErr: errors.New("candidate start failed"), + }} + next := 0 + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + slot := mustSlot(t, m, id) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + + require.EqualError(t, m.Update(context.Background(), entry), "candidate start failed") + slot.mu.RLock() + require.Same(t, old, slot.current) + require.Equal(t, slotRunning, slot.state) + slot.mu.RUnlock() + require.EqualValues(t, 1, old.stopCount.Load()) + require.EqualValues(t, 2, old.startCount.Load(), "old generation must be restored after its initial start") + require.EqualValues(t, 1, old.maxActive.Load(), "exclusive generations must never overlap") + require.EqualValues(t, 1, candidate.stopCount.Load(), "failed same-key candidate must be stopped") + require.EqualValues(t, 0, candidate.disposeCount.Load(), "failed same-key candidate must not dispose the old resource") + require.Equal(t, "2", slot.Info().Generation, "restoring old ownership creates a new stream generation") +} + +func TestManagerUpdateDifferentResourceCleansSpeculativeCandidateDestructively(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-old", stopErr: errors.New("old stop failed")} + candidate := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + exclusive: "slot-new", + }} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + _, err := mustSlot(t, m, id).Start(context.Background()) + require.NoError(t, err) + + require.EqualError(t, m.Update(context.Background(), entry), "old stop failed") + require.Same(t, old, mustSlot(t, m, id).currentSource()) + require.EqualValues(t, 1, candidate.startCount.Load()) + require.EqualValues(t, 1, candidate.disposeCount.Load(), "a started different-key candidate may be destructively cleaned") + require.EqualValues(t, 1, old.stopCount.Load()) +} + +func TestManagerUpdateRetainsCandidateLeaseWhenCleanupFailsAfterOldStopFailure(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + exclusive: "slot-old", + stopErr: errors.New("old stop failed"), + } + candidate := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + exclusive: "slot-new", + }, disposeErr: errors.New("candidate dispose failed")} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + slot := mustSlot(t, m, id) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + + err = m.Update(context.Background(), entry) + require.ErrorContains(t, err, "old stop failed") + require.ErrorContains(t, err, "candidate dispose failed") + require.Same(t, old, slot.currentSource()) + require.True(t, slot.hasRetiredSource(candidate), "failed candidate cleanup must remain retryable") + require.EqualValues(t, 1, candidate.disposeCount.Load()) + m.leaseMu.Lock() + _, oldLeaseHeld := m.leases["slot-old"] + _, candidateLeaseHeld := m.leases["slot-new"] + m.leaseMu.Unlock() + require.True(t, oldLeaseHeld) + require.True(t, candidateLeaseHeld, "candidate lease must survive failed cleanup") + + old.stopErr = nil + require.NoError(t, slot.Stop(context.Background())) + require.EqualValues(t, 2, candidate.disposeCount.Load()) + require.False(t, slot.hasRetiredSource(candidate)) + m.leaseMu.Lock() + _, candidateLeaseHeld = m.leases["slot-new"] + m.leaseMu.Unlock() + require.False(t, candidateLeaseHeld, "successful retry must release candidate lease") + require.NoError(t, m.Delete(context.Background(), entry)) +} + +func TestManagerUpdateRetainsBothLeasesWhenOldAndCandidateDisposeFail(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + exclusive: "slot-old", + }, disposeErr: errors.New("old dispose failed")} + candidate := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + exclusive: "slot-new", + }, disposeErr: errors.New("candidate dispose failed")} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + slot := mustSlot(t, m, id) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + + err = m.Update(context.Background(), entry) + require.ErrorContains(t, err, "old dispose failed") + require.ErrorContains(t, err, "candidate dispose failed") + require.Same(t, old, slot.currentSource()) + require.True(t, slot.hasRetiredSource(old)) + require.True(t, slot.hasRetiredSource(candidate)) + m.leaseMu.Lock() + _, oldLeaseHeld := m.leases["slot-old"] + _, candidateLeaseHeld := m.leases["slot-new"] + m.leaseMu.Unlock() + require.True(t, oldLeaseHeld) + require.True(t, candidateLeaseHeld) + + require.NoError(t, slot.Stop(context.Background())) + require.False(t, slot.hasRetiredSource(old)) + require.False(t, slot.hasRetiredSource(candidate)) + m.leaseMu.Lock() + _, oldLeaseHeld = m.leases["slot-old"] + _, candidateLeaseHeld = m.leases["slot-new"] + m.leaseMu.Unlock() + require.False(t, oldLeaseHeld) + require.False(t, candidateLeaseHeld) + require.NoError(t, m.Delete(context.Background(), entry)) +} + +func TestManagerUpdateRetainsCandidateOnLatePrecommitAbort(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-old"} + candidate := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + exclusive: "slot-new", + }, disposeErr: errors.New("candidate dispose failed")} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + slot := mustSlot(t, m, id) + candidate.onStart = func() { + slot.mu.Lock() + slot.disposing = true + slot.mu.Unlock() + } + + err := m.Update(context.Background(), entry) + require.ErrorContains(t, err, ErrSourceBusy.Error()) + require.ErrorContains(t, err, "candidate dispose failed") + require.Same(t, old, slot.currentSource()) + require.True(t, slot.hasRetiredSource(candidate)) + m.leaseMu.Lock() + _, candidateLeaseHeld := m.leases["slot-new"] + m.leaseMu.Unlock() + require.True(t, candidateLeaseHeld) + + require.NoError(t, slot.Stop(context.Background())) + require.False(t, slot.hasRetiredSource(candidate)) + m.leaseMu.Lock() + _, candidateLeaseHeld = m.leases["slot-new"] + m.leaseMu.Unlock() + require.False(t, candidateLeaseHeld) + require.NoError(t, m.Delete(context.Background(), entry)) +} + +func TestManagerUpdateSameExclusiveKeyStopFailureFaultsSlot(t *testing.T) { + old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-1", stopErr: errors.New("old stop failed")} + candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-1"} + next := 0 + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + slot := mustSlot(t, m, id) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + + require.EqualError(t, m.Update(context.Background(), entry), "old stop failed") + require.Same(t, old, slot.currentSource()) + slot.mu.RLock() + require.Equal(t, slotFaulted, slot.state) + slot.mu.RUnlock() + require.EqualValues(t, 0, candidate.startCount.Load(), "candidate must not start while old ownership is uncertain") +} + +func TestSourceSlotStampsCanonicalIdentityAndGeneration(t *testing.T) { + id := registry.NewID("app", "events") + upstream := &testStream{changes: make(chan api.Change, 1)} + source := &managedTestSource{stream: upstream} + slot := newSourceSlot(id, "db.cdc.test", source) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + + stream, err := slot.Subscribe(context.Background(), api.StreamOptions{}) + require.NoError(t, err) + upstream.changes <- api.Change{Source: "driver-alias", Generation: "driver-generation"} + + select { + case change := <-stream.Changes(): + require.Equal(t, id, change.SourceID) + require.Equal(t, id.String(), change.Source) + require.Equal(t, "1", change.Generation) + case <-time.After(time.Second): + t.Fatal("timed out waiting for stamped change") + } + stream.Close() + require.NoError(t, slot.Stop(context.Background())) +} + +func TestSourceSlotRejectsSubscribeDuringReplacement(t *testing.T) { + id := registry.NewID("app", "events") + source := &managedTestSource{stream: &testStream{changes: make(chan api.Change, 1)}} + slot := newSourceSlot(id, "db.cdc.test", source) + slot.mu.Lock() + slot.state = slotRunning + slot.replacing = true + slot.mu.Unlock() + + stream, err := slot.Subscribe(context.Background(), api.StreamOptions{}) + require.Nil(t, stream) + require.ErrorIs(t, err, api.ErrSourceNotReady) +} + +func TestSourceSlotInfoNormalizesLegacyAliases(t *testing.T) { + id := registry.NewID("app", "events") + source := &managedTestSource{info: api.SourceInfo{ + Name: "driver-alias", + Engine: "driver-engine", + Epoch: "driver-epoch", + Streaming: true, + Faulted: true, + DBResource: "resource:db", + }} + slot := newSourceSlot(id, "db.cdc.test", source) + + info := slot.Info() + require.Equal(t, id, info.ID) + require.Equal(t, id.String(), info.Name) + require.Equal(t, "1", info.Generation) + require.Equal(t, "1", info.Epoch) + require.Equal(t, api.SourceStateUnknown, info.State) + require.False(t, info.Streaming) + require.False(t, info.Faulted) + require.Equal(t, "driver-engine", info.Engine) + require.Equal(t, "resource:db", info.DBResource) + + slot.mu.Lock() + slot.state = slotRunning + slot.mu.Unlock() + info = slot.Info() + require.Equal(t, api.SourceStateRunning, info.State) + require.True(t, info.Streaming) + require.False(t, info.Faulted) + + slot.mu.Lock() + slot.state = slotFaulted + slot.mu.Unlock() + info = slot.Info() + require.Equal(t, api.SourceStateFaulted, info.State) + require.False(t, info.Streaming) + require.True(t, info.Faulted) +} + +func TestSourceSlotInfoHandlesNilSource(t *testing.T) { + id := registry.NewID("app", "events") + info := newSourceSlot(id, "db.cdc.test", nil).Info() + require.Equal(t, id, info.ID) + require.Equal(t, id.String(), info.Name) + require.Equal(t, "1", info.Generation) + require.Equal(t, "1", info.Epoch) + require.Equal(t, api.SourceStateUnknown, info.State) +} + +func TestSourceSlotRestartAdvancesGeneration(t *testing.T) { + id := registry.NewID("app", "events") + source := &managedTestSource{stream: &testStream{changes: make(chan api.Change, 1)}} + slot := newSourceSlot(id, "db.cdc.test", source) + + _, err := slot.Start(context.Background()) + require.NoError(t, err) + require.Equal(t, "1", slot.Info().Generation) + require.NoError(t, slot.Stop(context.Background())) + + _, err = slot.Start(context.Background()) + require.NoError(t, err) + require.Equal(t, "2", slot.Info().Generation) + stream, err := slot.Subscribe(context.Background(), api.StreamOptions{}) + require.NoError(t, err) + source.stream.changes <- api.Change{Op: "insert"} + select { + case change := <-stream.Changes(): + require.Equal(t, "2", change.Generation) + case <-time.After(time.Second): + t.Fatal("timed out waiting for restarted stream") + } + stream.Close() + require.NoError(t, slot.Stop(context.Background())) +} + +func mustSlot(t *testing.T, m *Manager, id registry.ID) *sourceSlot { + t.Helper() + source, ok := m.Get(id) + require.True(t, ok) + slot, ok := source.(*sourceSlot) + require.True(t, ok) + return slot +} + +func TestManagerRejectsUnsupportedAndMissingSources(t *testing.T) { + m, _ := newManagerTest(t) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: "db.cdc.unknown"} + require.ErrorIs(t, m.Add(context.Background(), entry), ErrUnsupportedKind) + require.ErrorIs(t, m.Update(context.Background(), entry), ErrUnsupportedKind) + require.ErrorIs(t, m.Delete(context.Background(), entry), ErrSourceNotFound) +} + +var _ event.Bus = (*eventbus.Bus)(nil) diff --git a/service/cdc/postgres/bench_test.go b/service/cdc/postgres/bench_test.go index e2acfe792..7375cbf97 100644 --- a/service/cdc/postgres/bench_test.go +++ b/service/cdc/postgres/bench_test.go @@ -24,6 +24,12 @@ func BenchmarkDecoderInsert(b *testing.B) { if _, err := d.apply(msg, 0x20); err != nil { b.Fatal(err) } + if _, err := d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x30}, 0); err != nil { + b.Fatal(err) + } + if _, err := d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0); err != nil { + b.Fatal(err) + } } } @@ -40,6 +46,12 @@ func BenchmarkDecoderUpdate(b *testing.B) { if _, err := d.apply(msg, 0x30); err != nil { b.Fatal(err) } + if _, err := d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x40}, 0); err != nil { + b.Fatal(err) + } + if _, err := d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0); err != nil { + b.Fatal(err) + } } } diff --git a/service/cdc/postgres/checkpoint.go b/service/cdc/postgres/checkpoint.go index aee2225e7..37e64f545 100644 --- a/service/cdc/postgres/checkpoint.go +++ b/service/cdc/postgres/checkpoint.go @@ -37,7 +37,9 @@ func (m *MemoryCheckpointer) Load(_ context.Context, slot string) (pglogrepl.LSN func (m *MemoryCheckpointer) Save(_ context.Context, slot string, lsn pglogrepl.LSN) error { m.mu.Lock() defer m.mu.Unlock() - m.pos[slot] = lsn + if current, ok := m.pos[slot]; !ok || lsn > current { + m.pos[slot] = lsn + } return nil } @@ -84,7 +86,9 @@ func (c *DBCheckpointer) Load(ctx context.Context, slot string) (pglogrepl.LSN, func (c *DBCheckpointer) Save(ctx context.Context, slot string, lsn pglogrepl.LSN) error { _, err := c.db.ExecContext(ctx, `INSERT INTO wippy_cdc_offsets (slot, lsn, updated_at) VALUES ($1, $2, now()) - ON CONFLICT (slot) DO UPDATE SET lsn = EXCLUDED.lsn, updated_at = now()`, + ON CONFLICT (slot) DO UPDATE + SET lsn = EXCLUDED.lsn, updated_at = now() + WHERE wippy_cdc_offsets.lsn::pg_lsn <= EXCLUDED.lsn::pg_lsn`, slot, lsn.String()) if err != nil { return fmt.Errorf("save offset: %w", err) diff --git a/service/cdc/postgres/checkpoint_test.go b/service/cdc/postgres/checkpoint_test.go index 50ea937aa..990f6d479 100644 --- a/service/cdc/postgres/checkpoint_test.go +++ b/service/cdc/postgres/checkpoint_test.go @@ -39,6 +39,17 @@ func TestMemoryCheckpointerRoundtrip(t *testing.T) { assert.False(t, ok) } +func TestMemoryCheckpointerIsMonotonic(t *testing.T) { + cp := NewMemoryCheckpointer() + ctx := context.Background() + require.NoError(t, cp.Save(ctx, "slot", pglogrepl.LSN(0x200))) + require.NoError(t, cp.Save(ctx, "slot", pglogrepl.LSN(0x100))) + lsn, ok, err := cp.Load(ctx, "slot") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, pglogrepl.LSN(0x200), lsn) +} + func TestMemoryCheckpointerDelete(t *testing.T) { cp := NewMemoryCheckpointer() ctx := context.Background() diff --git a/service/cdc/postgres/config_decode_test.go b/service/cdc/postgres/config_decode_test.go index 7d7921782..7200cebc3 100644 --- a/service/cdc/postgres/config_decode_test.go +++ b/service/cdc/postgres/config_decode_test.go @@ -27,17 +27,21 @@ func decodeConfig(t *testing.T, raw map[string]any) *config.Config { func TestConfigWireFormatMapsAndBuildsDSN(t *testing.T) { cfg := decodeConfig(t, map[string]any{ - "host": "db.internal", - "port": 5432, - "username": "cdc_repl", - "password": "secret", - "database": "appdb", - "slot_name": "wippy_slot", - "publication": "wippy_pub", - "snapshot": true, - "standby_interval": "5s", - "status_interval": "1m", - "tables": []any{"public.accounts", "public.orders"}, + "host": "db.internal", + "port": 5432, + "username": "cdc_repl", + "password": "secret", + "database": "appdb", + "slot_name": "wippy_slot", + "publication": "wippy_pub", + "snapshot": true, + "max_transaction_changes": 1234, + "max_transaction_bytes": 65536, + "max_inflight_changes": 2345, + "max_inflight_bytes": 131072, + "standby_interval": "5s", + "status_interval": "1m", + "tables": []any{"public.accounts", "public.orders"}, }) require.NoError(t, cfg.Validate()) @@ -48,6 +52,10 @@ func TestConfigWireFormatMapsAndBuildsDSN(t *testing.T) { assert.Equal(t, "wippy_slot", cfg.SlotName) assert.Equal(t, "wippy_pub", cfg.Publication) assert.True(t, cfg.Snapshot) + assert.Equal(t, 1234, cfg.MaxTransactionChanges) + assert.Equal(t, int64(65536), cfg.MaxTransactionBytes) + assert.Equal(t, 2345, cfg.MaxInflightChanges) + assert.Equal(t, int64(131072), cfg.MaxInflightBytes) assert.Equal(t, "5s", cfg.StandbyInterval) assert.Equal(t, []string{"public.accounts", "public.orders"}, cfg.Tables) diff --git a/service/cdc/postgres/decoder.go b/service/cdc/postgres/decoder.go index ba0ee3e44..37cb7e5ce 100644 --- a/service/cdc/postgres/decoder.go +++ b/service/cdc/postgres/decoder.go @@ -11,27 +11,56 @@ import ( type bufferedChange struct { rc RowChange subxid uint32 + bytes int64 +} + +// decodeResult describes the progress made by one logical replication +// message. Changes are returned only at a transaction boundary. A safe result +// means the WAL position after this message can be acknowledged without +// reconstructing decoder state after a restart. +type decodeResult struct { + changes []RowChange + safe bool } type decoder struct { - rels *relationCache - buffer map[uint32][]bufferedChange - commitLSN pglogrepl.LSN - xid uint32 - curTopXid uint32 - streaming bool - inStream bool + rels *relationCache + buffer map[uint32][]bufferedChange + usage map[uint32]int64 + limits decoderLimits + commitLSN pglogrepl.LSN + inflightChanges int + inflightBytes int64 + xid uint32 + curTopXid uint32 + streaming bool + inStream bool + txActive bool } -func newDecoder() *decoder { - return &decoder{rels: newRelationCache()} +func newDecoder(limits ...decoderLimits) *decoder { + return newDecoderWithMode(false, limits...) } -func newStreamingDecoder() *decoder { - return &decoder{rels: newRelationCache(), streaming: true, buffer: map[uint32][]bufferedChange{}} +func newStreamingDecoder(limits ...decoderLimits) *decoder { + return newDecoderWithMode(true, limits...) } -func (d *decoder) decode(walData []byte, walStart pglogrepl.LSN) ([]RowChange, error) { +func newDecoderWithMode(streaming bool, limits ...decoderLimits) *decoder { + configured := defaultDecoderLimits() + if len(limits) > 0 { + configured = normalizeDecoderLimits(limits[0]) + } + return &decoder{ + rels: newRelationCache(), + buffer: make(map[uint32][]bufferedChange), + streaming: streaming, + limits: configured, + usage: make(map[uint32]int64), + } +} + +func (d *decoder) decodeResult(walData []byte, walStart pglogrepl.LSN) (decodeResult, error) { var ( msg pglogrepl.Message err error @@ -42,65 +71,216 @@ func (d *decoder) decode(walData []byte, walStart pglogrepl.LSN) ([]RowChange, e msg, err = pglogrepl.Parse(walData) } if err != nil { - return nil, fmt.Errorf("parse logical message: %w", err) + return decodeResult{}, fmt.Errorf("parse logical message: %w", err) } - return d.apply(msg, walStart) + return d.applyResult(msg, walStart) } +// apply is retained as the small decoder test seam. It deliberately returns +// no rows until Commit or StreamCommit; callers that need checkpoint progress +// use applyResult/decodeResult instead. func (d *decoder) apply(msg pglogrepl.Message, walStart pglogrepl.LSN) ([]RowChange, error) { + result, err := d.applyResult(msg, walStart) + if err != nil { + return nil, err + } + return result.changes, nil +} + +func (d *decoder) applyResult(msg pglogrepl.Message, walStart pglogrepl.LSN) (decodeResult, error) { switch m := msg.(type) { case *pglogrepl.RelationMessage: d.rels.put(m) + return decodeResult{safe: !d.inTransaction()}, nil + + case *pglogrepl.OriginMessage: + // Origin is transaction metadata. The row API has no origin field, but + // the message is valid and must not interrupt an otherwise valid stream. + return decodeResult{safe: !d.inTransaction()}, nil + + case *pglogrepl.TypeMessage: + // Type definitions are metadata for output plugins. Tuple decoding is + // intentionally text-preserving, so there is no row-level state to + // update here. + return decodeResult{safe: !d.inTransaction()}, nil + + case *pglogrepl.LogicalDecodingMessage: + // Logical messages are valid pgoutput records but are not row changes. + // They remain commit-gated by the decoder state and are therefore safe + // to ignore without advancing a checkpoint inside a transaction. + return decodeResult{safe: !d.inTransaction()}, nil + case *pglogrepl.BeginMessage: + // Protocol v2 may interleave an ordinary (small) transaction with + // streamed transactions whose segments have already been stopped. The + // ordinary transaction has its own buffer (key 0), so an existing + // streamed buffer is not a nested Begin. + if d.txActive || d.inStream { + return decodeResult{}, fmt.Errorf("%w: nested begin", ErrInvalidTransaction) + } d.commitLSN = m.FinalLSN d.xid = m.Xid + d.txActive = true + d.buffer[0] = nil + return decodeResult{}, nil + case *pglogrepl.CommitMessage: - d.commitLSN = 0 - d.xid = 0 + if !d.txActive { + return decodeResult{}, fmt.Errorf("%w: commit without begin", ErrInvalidTransaction) + } + return decodeResult{changes: d.flushTransaction(m.CommitLSN), safe: len(d.buffer) == 0}, nil + case *pglogrepl.InsertMessage: return d.one(OpInsert, m.RelationID, nil, m.Tuple, walStart) + case *pglogrepl.UpdateMessage: return d.one(OpUpdate, m.RelationID, m.OldTuple, m.NewTuple, walStart) + case *pglogrepl.DeleteMessage: return d.one(OpDelete, m.RelationID, m.OldTuple, nil, walStart) + case *pglogrepl.TruncateMessage: - return d.truncate(m, walStart) + return d.truncateResult(m.RelationIDs, walStart) + case *pglogrepl.RelationMessageV2: d.rels.put(&m.RelationMessage) + return decodeResult{safe: !d.inTransaction()}, nil + + case *pglogrepl.TypeMessageV2: + return decodeResult{safe: !d.inTransaction()}, nil + + case *pglogrepl.LogicalDecodingMessageV2: + return decodeResult{safe: !d.inTransaction()}, nil + case *pglogrepl.StreamStartMessageV2: + if !d.streaming { + return decodeResult{}, fmt.Errorf("%w: stream start in protocol v1", ErrInvalidTransaction) + } + if d.inStream || d.txActive { + return decodeResult{}, fmt.Errorf("%w: nested stream start", ErrInvalidTransaction) + } d.inStream = true d.curTopXid = m.Xid if _, ok := d.buffer[m.Xid]; !ok { d.buffer[m.Xid] = nil } + return decodeResult{}, nil + case *pglogrepl.StreamStopMessageV2: + if !d.inStream { + return decodeResult{}, fmt.Errorf("%w: stream stop without start", ErrInvalidTransaction) + } d.inStream = false + return decodeResult{}, nil + case *pglogrepl.StreamCommitMessageV2: - return d.flushStream(m.Xid, m.CommitLSN), nil + if d.inStream { + return decodeResult{}, fmt.Errorf("%w: stream commit before stream stop", ErrInvalidTransaction) + } + if _, ok := d.buffer[m.Xid]; !ok { + return decodeResult{}, fmt.Errorf("%w: stream commit for unknown xid %d", ErrInvalidTransaction, m.Xid) + } + changes := d.flushStream(m.Xid, m.CommitLSN) + return decodeResult{changes: changes, safe: len(d.buffer) == 0}, nil + case *pglogrepl.StreamAbortMessageV2: + if d.inStream { + return decodeResult{}, fmt.Errorf("%w: stream abort before stream stop", ErrInvalidTransaction) + } + if _, ok := d.buffer[m.Xid]; !ok { + return decodeResult{}, fmt.Errorf("%w: stream abort for unknown xid %d", ErrInvalidTransaction, m.Xid) + } d.abortStream(m.Xid, m.SubXid) + return decodeResult{safe: len(d.buffer) == 0}, nil + case *pglogrepl.InsertMessageV2: if d.inStream { - return nil, d.bufferOne(m.Xid, OpInsert, m.RelationID, nil, m.Tuple, walStart) + return decodeResult{}, d.bufferOne(m.Xid, OpInsert, m.RelationID, nil, m.Tuple, walStart) } return d.one(OpInsert, m.RelationID, nil, m.Tuple, walStart) + case *pglogrepl.UpdateMessageV2: if d.inStream { - return nil, d.bufferOne(m.Xid, OpUpdate, m.RelationID, m.OldTuple, m.NewTuple, walStart) + return decodeResult{}, d.bufferOne(m.Xid, OpUpdate, m.RelationID, m.OldTuple, m.NewTuple, walStart) } return d.one(OpUpdate, m.RelationID, m.OldTuple, m.NewTuple, walStart) + case *pglogrepl.DeleteMessageV2: if d.inStream { - return nil, d.bufferOne(m.Xid, OpDelete, m.RelationID, m.OldTuple, nil, walStart) + return decodeResult{}, d.bufferOne(m.Xid, OpDelete, m.RelationID, m.OldTuple, nil, walStart) } return d.one(OpDelete, m.RelationID, m.OldTuple, nil, walStart) + case *pglogrepl.TruncateMessageV2: if d.inStream { - return nil, d.bufferTruncate(m, walStart) + return decodeResult{}, d.bufferTruncate(m, walStart) } - return d.truncate(&m.TruncateMessage, walStart) + return d.truncateResult(m.RelationIDs, walStart) + + default: + return decodeResult{}, fmt.Errorf("%w: %T", ErrUnsupportedMessage, msg) + } +} + +func (d *decoder) inTransaction() bool { + return d.txActive || d.inStream || len(d.buffer) > 0 +} + +func (d *decoder) flushTransaction(commitLSN pglogrepl.LSN) []RowChange { + if commitLSN == 0 { + commitLSN = d.commitLSN + } + buffered := d.buffer[0] + d.releaseBuffer(0) + changes := make([]RowChange, 0, len(buffered)) + for i := range buffered { + buffered[i].rc.CommitLSN = commitLSN.String() + changes = append(changes, buffered[i].rc) } - return nil, nil + d.commitLSN = 0 + d.xid = 0 + d.txActive = false + return changes +} + +func (d *decoder) one(op Op, relID uint32, oldT, newT *pglogrepl.TupleData, walStart pglogrepl.LSN) (decodeResult, error) { + if !d.txActive { + return decodeResult{}, fmt.Errorf("%w: row without begin", ErrInvalidTransaction) + } + bytes, err := d.reserveRow(0, relID, oldT, newT) + if err != nil { + return decodeResult{}, err + } + rc, err := d.changeFor(op, relID, oldT, newT, walStart) + if err != nil { + d.releaseReservation(0, 1, bytes) + return decodeResult{}, err + } + rc.XID = d.xid + rc.CommitLSN = d.commitLSN.String() + d.buffer[0] = append(d.buffer[0], bufferedChange{rc: rc, subxid: d.xid, bytes: bytes}) + return decodeResult{}, nil +} + +func (d *decoder) truncateResult(relationIDs []uint32, walStart pglogrepl.LSN) (decodeResult, error) { + if !d.txActive { + return decodeResult{}, fmt.Errorf("%w: truncate without begin", ErrInvalidTransaction) + } + relations, bytes, err := d.truncateBudget(0, relationIDs) + if err != nil { + return decodeResult{}, err + } + for i, rel := range relations { + d.buffer[0] = append(d.buffer[0], bufferedChange{rc: RowChange{ + Op: OpTruncate, + Schema: rel.Namespace, + Table: rel.RelationName, + LSN: walStart.String(), + CommitLSN: d.commitLSN.String(), + XID: d.xid, + }, subxid: d.xid, bytes: bytes[i]}) + } + return decodeResult{}, nil } func (d *decoder) changeFor(op Op, relID uint32, oldT, newT *pglogrepl.TupleData, walStart pglogrepl.LSN) (RowChange, error) { @@ -118,32 +298,33 @@ func (d *decoder) changeFor(op Op, relID uint32, oldT, newT *pglogrepl.TupleData }, nil } -func (d *decoder) one(op Op, relID uint32, oldT, newT *pglogrepl.TupleData, walStart pglogrepl.LSN) ([]RowChange, error) { - rc, err := d.changeFor(op, relID, oldT, newT, walStart) +func (d *decoder) bufferOne(subxid uint32, op Op, relID uint32, oldT, newT *pglogrepl.TupleData, walStart pglogrepl.LSN) error { + if !d.inStream { + return fmt.Errorf("%w: streamed row without stream start", ErrInvalidTransaction) + } + bytes, err := d.reserveRow(d.curTopXid, relID, oldT, newT) if err != nil { - return nil, err + return err } - rc.XID = d.xid - rc.CommitLSN = d.commitLSN.String() - return []RowChange{rc}, nil -} - -func (d *decoder) bufferOne(subxid uint32, op Op, relID uint32, oldT, newT *pglogrepl.TupleData, walStart pglogrepl.LSN) error { rc, err := d.changeFor(op, relID, oldT, newT, walStart) if err != nil { + d.releaseReservation(d.curTopXid, 1, bytes) return err } rc.XID = d.curTopXid - d.buffer[d.curTopXid] = append(d.buffer[d.curTopXid], bufferedChange{rc: rc, subxid: subxid}) + d.buffer[d.curTopXid] = append(d.buffer[d.curTopXid], bufferedChange{rc: rc, subxid: subxid, bytes: bytes}) return nil } func (d *decoder) bufferTruncate(m *pglogrepl.TruncateMessageV2, walStart pglogrepl.LSN) error { - for _, relID := range m.RelationIDs { - rel, ok := d.rels.get(relID) - if !ok { - return fmt.Errorf("%w: %d", ErrUnknownRelation, relID) - } + if !d.inStream { + return fmt.Errorf("%w: streamed truncate without stream start", ErrInvalidTransaction) + } + relations, bytes, err := d.truncateBudget(d.curTopXid, m.RelationIDs) + if err != nil { + return err + } + for i, rel := range relations { rc := RowChange{ Op: OpTruncate, Schema: rel.Namespace, @@ -151,15 +332,16 @@ func (d *decoder) bufferTruncate(m *pglogrepl.TruncateMessageV2, walStart pglogr LSN: walStart.String(), XID: d.curTopXid, } - d.buffer[d.curTopXid] = append(d.buffer[d.curTopXid], bufferedChange{rc: rc, subxid: m.Xid}) + d.buffer[d.curTopXid] = append(d.buffer[d.curTopXid], bufferedChange{rc: rc, subxid: m.Xid, bytes: bytes[i]}) } return nil } func (d *decoder) flushStream(topXid uint32, commitLSN pglogrepl.LSN) []RowChange { buffered := d.buffer[topXid] - delete(d.buffer, topXid) + d.releaseBuffer(topXid) d.inStream = false + d.curTopXid = 0 out := make([]RowChange, 0, len(buffered)) for i := range buffered { @@ -171,40 +353,161 @@ func (d *decoder) flushStream(topXid uint32, commitLSN pglogrepl.LSN) []RowChang func (d *decoder) abortStream(topXid, subXid uint32) { d.inStream = false + d.curTopXid = 0 if topXid == subXid { - delete(d.buffer, topXid) + d.releaseBuffer(topXid) return } + + // PostgreSQL's logical apply worker does not remove only changes whose + // XID equals subXid. It records the first streamed offset for each + // subtransaction and truncates the transaction at the aborted subxact's + // offset. This also discards nested subtransactions (and the remainder of + // that stream segment); changes after the rollback are sent in a later + // stream segment. The decoder has the same ordering in its buffer, so the + // first change for subXid is the equivalent truncation point. src := d.buffer[topXid] - n := 0 - for _, bc := range src { - if bc.subxid != subXid { - src[n] = bc - n++ + cut := -1 + for i, bc := range src { + if bc.subxid == subXid { + cut = i + break } } - if n == 0 { + if cut < 0 { + // Empty subtransactions are valid and have no buffered offset. There + // is nothing to truncate in that case. + return + } + oldChanges := len(src) + oldBytes := d.usage[topXid] + d.inflightChanges -= oldChanges + d.inflightBytes -= oldBytes + // Clear the discarded tail before reslicing so decoded row maps and their + // byte slices are no longer retained by the backing array. + clear(src[cut:]) + src = src[:cut] + if len(src) == 0 { delete(d.buffer, topXid) + delete(d.usage, topXid) return } - d.buffer[topXid] = src[:n] + d.buffer[topXid] = src + var bytes int64 + for _, bc := range src { + bytes += bc.bytes + } + d.usage[topXid] = bytes + d.inflightChanges += len(src) + d.inflightBytes += bytes +} + +func (d *decoder) releaseBuffer(key uint32) { + buffered, exists := d.buffer[key] + if !exists { + return + } + d.inflightChanges -= len(buffered) + d.inflightBytes -= d.usage[key] + delete(d.buffer, key) + delete(d.usage, key) } -func (d *decoder) truncate(m *pglogrepl.TruncateMessage, walStart pglogrepl.LSN) ([]RowChange, error) { - changes := make([]RowChange, 0, len(m.RelationIDs)) - for _, relID := range m.RelationIDs { +func (d *decoder) releaseReservation(key uint32, changes int, bytes int64) { + d.usage[key] -= bytes + d.inflightChanges -= changes + d.inflightBytes -= bytes + if d.usage[key] == 0 { + delete(d.usage, key) + } +} + +func (d *decoder) reserveRow(key, relID uint32, oldT, newT *pglogrepl.TupleData) (int64, error) { + rel, ok := d.rels.get(relID) + if !ok { + return 0, fmt.Errorf("%w: %d", ErrUnknownRelation, relID) + } + bytes := estimateChangeBytes(rel, oldT, newT) + if err := d.reserve(key, 1, bytes); err != nil { + return 0, err + } + return bytes, nil +} + +func (d *decoder) truncateBudget(key uint32, relationIDs []uint32) ([]*pglogrepl.RelationMessage, []int64, error) { + relations := make([]*pglogrepl.RelationMessage, len(relationIDs)) + bytes := make([]int64, len(relationIDs)) + var total int64 + for i, relID := range relationIDs { rel, ok := d.rels.get(relID) if !ok { - return nil, fmt.Errorf("%w: %d", ErrUnknownRelation, relID) + return nil, nil, fmt.Errorf("%w: %d", ErrUnknownRelation, relID) + } + relations[i] = rel + bytes[i] = estimateRelationChangeBytes(rel) + total += bytes[i] + } + if err := d.reserve(key, len(relationIDs), total); err != nil { + return nil, nil, err + } + return relations, bytes, nil +} + +func (d *decoder) reserve(key uint32, changes int, bytes int64) error { + if changes < 0 || bytes < 0 { + return fmt.Errorf("%w: invalid estimated transaction size", ErrTransactionLimit) + } + currentChanges := len(d.buffer[key]) + if changes > d.limits.maxChanges-currentChanges { + return fmt.Errorf("%w: changes=%d limit=%d", ErrTransactionLimit, currentChanges+changes, d.limits.maxChanges) + } + currentBytes := d.usage[key] + if bytes > d.limits.maxBytes-currentBytes { + return fmt.Errorf("%w: bytes=%d limit=%d", ErrTransactionLimit, currentBytes+bytes, d.limits.maxBytes) + } + if changes > d.limits.maxInflightChanges-d.inflightChanges { + return fmt.Errorf("%w: inflight_changes=%d limit=%d", ErrTransactionLimit, + d.inflightChanges+changes, d.limits.maxInflightChanges) + } + if bytes > d.limits.maxInflightBytes-d.inflightBytes { + return fmt.Errorf("%w: inflight_bytes=%d limit=%d", ErrTransactionLimit, + d.inflightBytes+bytes, d.limits.maxInflightBytes) + } + d.usage[key] = currentBytes + bytes + d.inflightChanges += changes + d.inflightBytes += bytes + return nil +} + +const ( + changeEstimateBase int64 = 256 + columnEstimateBase int64 = 64 + stringCopyEstimate int64 = 1 +) + +func estimateChangeBytes(rel *pglogrepl.RelationMessage, oldT, newT *pglogrepl.TupleData) int64 { + return changeEstimateBase + int64(len(rel.Namespace)+len(rel.RelationName)) + + estimateTupleBytes(rel, oldT) + estimateTupleBytes(rel, newT) +} + +func estimateRelationChangeBytes(rel *pglogrepl.RelationMessage) int64 { + return changeEstimateBase + int64(len(rel.Namespace)+len(rel.RelationName)) +} + +func estimateTupleBytes(rel *pglogrepl.RelationMessage, tuple *pglogrepl.TupleData) int64 { + if tuple == nil { + return 0 + } + bytes := columnEstimateBase + for i, col := range tuple.Columns { + bytes += columnEstimateBase + int64(len(col.Data)) + if col.DataType != pglogrepl.TupleDataTypeNull && col.DataType != pglogrepl.TupleDataTypeToast { + // tupleToMap converts values to strings, retaining a second copy. + bytes += stringCopyEstimate * int64(len(col.Data)) + } + if i < len(rel.Columns) { + bytes += int64(len(rel.Columns[i].Name)) } - changes = append(changes, RowChange{ - Op: OpTruncate, - Schema: rel.Namespace, - Table: rel.RelationName, - LSN: walStart.String(), - CommitLSN: d.commitLSN.String(), - XID: d.xid, - }) } - return changes, nil + return bytes } diff --git a/service/cdc/postgres/decoder_stream_test.go b/service/cdc/postgres/decoder_stream_test.go index d5de5f6f6..9b11bf81e 100644 --- a/service/cdc/postgres/decoder_stream_test.go +++ b/service/cdc/postgres/decoder_stream_test.go @@ -45,11 +45,31 @@ func TestStreamingDecoderBuffersUntilCommit(t *testing.T) { assert.Equal(t, "a@w.ai", changes[0].After["email"]) } +func TestStreamingDecoderRequiresStopBeforeCommitOrAbort(t *testing.T) { + for _, tc := range []struct { + msg pglogrepl.Message + name string + }{ + {name: "commit", msg: &pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x99}}, + {name: "abort", msg: &pglogrepl.StreamAbortMessageV2{Xid: 100, SubXid: 100}}, + } { + t.Run(tc.name, func(t *testing.T) { + d := newStreamingDecoder() + _, err := d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + require.NoError(t, err) + + _, err = d.apply(tc.msg, 0) + require.ErrorIs(t, err, ErrInvalidTransaction) + }) + } +} + func TestStreamingDecoderTopLevelAbortDiscards(t *testing.T) { d := newStreamingDecoder() _, _ = d.apply(relV2(), 0) _, _ = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) _, _ = d.apply(insertV2(100, "1", "a@w.ai"), 0x20) + _, _ = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) _, err := d.apply(&pglogrepl.StreamAbortMessageV2{Xid: 100, SubXid: 100}, 0) require.NoError(t, err) @@ -64,6 +84,7 @@ func TestStreamingDecoderSubtransactionAbortDropsOnlyThatSubxid(t *testing.T) { _, _ = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) _, _ = d.apply(insertV2(100, "1", "keep@w.ai"), 0x20) _, _ = d.apply(insertV2(200, "2", "drop@w.ai"), 0x30) + _, _ = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) _, err := d.apply(&pglogrepl.StreamAbortMessageV2{Xid: 100, SubXid: 200}, 0) require.NoError(t, err) @@ -74,6 +95,28 @@ func TestStreamingDecoderSubtransactionAbortDropsOnlyThatSubxid(t *testing.T) { assert.Equal(t, "keep@w.ai", changes[0].After["email"]) } +func TestStreamingDecoderSubtransactionAbortDropsNestedDescendants(t *testing.T) { + d := newStreamingDecoder() + _, _ = d.apply(relV2(), 0) + _, _ = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + _, _ = d.apply(insertV2(100, "1", "keep@w.ai"), 0x20) + _, _ = d.apply(insertV2(200, "2", "drop-parent@w.ai"), 0x30) + _, _ = d.apply(insertV2(300, "3", "drop-child@w.ai"), 0x40) + _, _ = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + + // A subtransaction abort includes all of its nested subtransactions. + // PostgreSQL represents this by truncating the streamed changes at the + // aborted subtransaction's first offset, not by sending a parent XID on + // every descendant row. + _, err := d.apply(&pglogrepl.StreamAbortMessageV2{Xid: 100, SubXid: 200}, 0) + require.NoError(t, err) + + changes, err := d.apply(&pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x99}, 0) + require.NoError(t, err) + require.Len(t, changes, 1) + assert.Equal(t, "keep@w.ai", changes[0].After["email"]) +} + func TestStreamingDecoderInterleavedTransactions(t *testing.T) { d := newStreamingDecoder() _, _ = d.apply(relV2(), 0) @@ -97,25 +140,97 @@ func TestStreamingDecoderInterleavedTransactions(t *testing.T) { assert.Equal(t, "tx100@w.ai", c100[0].After["email"]) } -func TestStreamingDecoderNonStreamedV2EmitsImmediately(t *testing.T) { +func TestStreamingDecoderNonStreamedV2EmitsAtCommit(t *testing.T) { d := newStreamingDecoder() _, _ = d.apply(relV2(), 0) _, _ = d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) changes, err := d.apply(insertV2(0, "1", "v2small@w.ai"), 0x20) require.NoError(t, err) - require.Len(t, changes, 1, "non-streamed v2 insert (inStream=false) must emit immediately, not buffer") + assert.Empty(t, changes, "non-streamed v2 rows must wait for commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x30}, 0) + require.NoError(t, err) + require.Len(t, changes, 1) assert.Equal(t, uint32(7), changes[0].XID) assert.Equal(t, "v2small@w.ai", changes[0].After["email"]) } -func TestStreamingDecoderNonStreamedStillWorks(t *testing.T) { +func TestStreamingDecoderNonStreamedStillWaitsForCommit(t *testing.T) { d := newStreamingDecoder() _, _ = d.apply(accountsRel(), 0) _, _ = d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) changes, err := d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("1", "small@w.ai")}, 0x20) require.NoError(t, err) - require.Len(t, changes, 1, "small (non-streamed) transactions must still emit immediately") + assert.Empty(t, changes, "non-streamed rows must wait for commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x30}, 0) + require.NoError(t, err) + require.Len(t, changes, 1) assert.Equal(t, uint32(7), changes[0].XID) } + +func TestStreamingDecoderDoesNotMarkInterleavedCommitSafe(t *testing.T) { + d := newStreamingDecoder() + _, _ = d.apply(relV2(), 0) + _, _ = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + _, _ = d.apply(insertV2(100, "1", "tx100@w.ai"), 0x20) + _, _ = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + _, _ = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 200, FirstSegment: 1}, 0) + _, _ = d.apply(insertV2(200, "2", "tx200@w.ai"), 0x30) + _, _ = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + + result, err := d.applyResult(&pglogrepl.StreamCommitMessageV2{Xid: 200, CommitLSN: 0x40}, 0) + require.NoError(t, err) + assert.False(t, result.safe, "an earlier streamed transaction is still open") + assert.Len(t, result.changes, 1) + + result, err = d.applyResult(&pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x50}, 0) + require.NoError(t, err) + assert.True(t, result.safe) + assert.Len(t, result.changes, 1) +} + +func TestStreamingDecoderAllowsOrdinaryTransactionAlongsideStream(t *testing.T) { + d := newStreamingDecoder() + _, _ = d.apply(relV2(), 0) + _, _ = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + _, _ = d.apply(insertV2(100, "1", "streamed@w.ai"), 0x20) + _, _ = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + + _, err := d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x30, Xid: 7}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(0, "2", "ordinary@w.ai"), 0x31) + require.NoError(t, err) + result, err := d.applyResult(&pglogrepl.CommitMessage{CommitLSN: 0x32}, 0) + require.NoError(t, err) + assert.False(t, result.safe, "streamed transaction is still buffered") + require.Len(t, result.changes, 1) + assert.Equal(t, "ordinary@w.ai", result.changes[0].After["email"]) + + result, err = d.applyResult(&pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x40}, 0) + require.NoError(t, err) + assert.True(t, result.safe) + require.Len(t, result.changes, 1) + assert.Equal(t, "streamed@w.ai", result.changes[0].After["email"]) +} + +func TestStreamingDecoderAcceptsMetadataMessages(t *testing.T) { + d := newStreamingDecoder() + metadata := []pglogrepl.Message{ + &pglogrepl.TypeMessageV2{}, + &pglogrepl.LogicalDecodingMessageV2{}, + } + for _, msg := range metadata { + result, err := d.applyResult(msg, 0) + require.NoError(t, err) + assert.True(t, result.safe) + } + + _, err := d.applyResult(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + require.NoError(t, err) + for _, msg := range metadata { + result, err := d.applyResult(msg, 0) + require.NoError(t, err) + assert.False(t, result.safe) + } +} diff --git a/service/cdc/postgres/decoder_test.go b/service/cdc/postgres/decoder_test.go index d642f5b96..24f3b8e92 100644 --- a/service/cdc/postgres/decoder_test.go +++ b/service/cdc/postgres/decoder_test.go @@ -44,6 +44,9 @@ func TestDecoderInsert(t *testing.T) { changes, err := d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("1", "a@w.ai")}, 0x20) require.NoError(t, err) + assert.Empty(t, changes, "rows are not visible before commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x30}, 0) + require.NoError(t, err) require.Len(t, changes, 1) c := changes[0] @@ -52,7 +55,7 @@ func TestDecoderInsert(t *testing.T) { assert.Equal(t, "accounts", c.Table) assert.Equal(t, uint32(7), c.XID) assert.Equal(t, "0/20", c.LSN) - assert.Equal(t, "0/10", c.CommitLSN) + assert.Equal(t, "0/30", c.CommitLSN) assert.Equal(t, map[string]any{"id": "1", "email": "a@w.ai"}, c.After) assert.Nil(t, c.Before) } @@ -67,6 +70,9 @@ func TestDecoderUpdate(t *testing.T) { NewTuple: textTuple("1", "new@w.ai"), }, 0x30) require.NoError(t, err) + assert.Empty(t, changes, "rows are not visible before commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x31}, 0) + require.NoError(t, err) require.Len(t, changes, 1) c := changes[0] @@ -81,6 +87,9 @@ func TestDecoderDelete(t *testing.T) { changes, err := d.apply(&pglogrepl.DeleteMessage{RelationID: 42, OldTuple: textTuple("1", "a@w.ai")}, 0x40) require.NoError(t, err) + assert.Empty(t, changes, "rows are not visible before commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x41}, 0) + require.NoError(t, err) require.Len(t, changes, 1) c := changes[0] @@ -95,6 +104,9 @@ func TestDecoderTruncate(t *testing.T) { changes, err := d.apply(&pglogrepl.TruncateMessage{RelationNum: 1, RelationIDs: []uint32{42}}, 0x50) require.NoError(t, err) + assert.Empty(t, changes, "rows are not visible before commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x51}, 0) + require.NoError(t, err) require.Len(t, changes, 1) c := changes[0] @@ -114,9 +126,14 @@ func TestDecoderTruncateMultipleRelations(t *testing.T) { Columns: []*pglogrepl.RelationMessageColumn{{Name: "id"}}, }, 0) require.NoError(t, err) + _, err = d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) + require.NoError(t, err) changes, err := d.apply(&pglogrepl.TruncateMessage{RelationNum: 2, RelationIDs: []uint32{42, 43}}, 0x60) require.NoError(t, err) + assert.Empty(t, changes, "rows are not visible before commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x61}, 0) + require.NoError(t, err) require.Len(t, changes, 2) assert.Equal(t, "accounts", changes[0].Table) assert.Equal(t, "orders", changes[1].Table) @@ -124,6 +141,8 @@ func TestDecoderTruncateMultipleRelations(t *testing.T) { func TestDecoderTruncateUnknownRelation(t *testing.T) { d := newDecoder() + _, err := d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) + require.NoError(t, err) changes, err := d.apply(&pglogrepl.TruncateMessage{RelationNum: 1, RelationIDs: []uint32{999}}, 0x50) require.ErrorIs(t, err, ErrUnknownRelation) assert.Nil(t, changes) @@ -131,6 +150,8 @@ func TestDecoderTruncateUnknownRelation(t *testing.T) { func TestDecoderUnknownRelation(t *testing.T) { d := newDecoder() + _, err := d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) + require.NoError(t, err) changes, err := d.apply(&pglogrepl.InsertMessage{RelationID: 99, Tuple: textTuple("1")}, 0x10) require.ErrorIs(t, err, ErrUnknownRelation) assert.Nil(t, changes) @@ -151,10 +172,189 @@ func TestDecoderCommitClearsTransactionState(t *testing.T) { require.NoError(t, err) changes, err := d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("9", "x@w.ai")}, 0x99) + require.ErrorIs(t, err, ErrInvalidTransaction) + assert.Nil(t, changes) +} + +func TestDecoderSafeProgressOnlyAtTransactionBoundary(t *testing.T) { + d := newDecoder() + result, err := d.applyResult(accountsRel(), 0) require.NoError(t, err) - require.Len(t, changes, 1) - assert.Equal(t, uint32(0), changes[0].XID, "xid must not leak from a committed transaction") - assert.Equal(t, "0/0", changes[0].CommitLSN) + assert.True(t, result.safe) + + result, err = d.applyResult(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) + require.NoError(t, err) + assert.False(t, result.safe) + result, err = d.applyResult(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("1", "a@w.ai")}, 0x20) + require.NoError(t, err) + assert.False(t, result.safe) + result, err = d.applyResult(&pglogrepl.CommitMessage{CommitLSN: 0x30}, 0) + require.NoError(t, err) + assert.True(t, result.safe) + assert.Len(t, result.changes, 1) +} + +func TestDecoderAcceptsPgoutputMetadataMessages(t *testing.T) { + d := newDecoder() + metadata := []pglogrepl.Message{ + &pglogrepl.OriginMessage{}, + &pglogrepl.TypeMessage{}, + &pglogrepl.LogicalDecodingMessage{}, + } + for _, msg := range metadata { + result, err := d.applyResult(msg, 0) + require.NoError(t, err) + assert.True(t, result.safe) + } + + _, err := d.applyResult(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) + require.NoError(t, err) + for _, msg := range metadata { + result, err := d.applyResult(msg, 0) + require.NoError(t, err) + assert.False(t, result.safe, "metadata inside an open transaction cannot advance the checkpoint") + } + result, err := d.applyResult(&pglogrepl.CommitMessage{CommitLSN: 0x20}, 0) + require.NoError(t, err) + assert.True(t, result.safe) +} + +func TestStreamingDecoderAcceptsPgoutputMetadataMessages(t *testing.T) { + d := newStreamingDecoder() + metadata := []pglogrepl.Message{ + &pglogrepl.TypeMessageV2{}, + &pglogrepl.LogicalDecodingMessageV2{}, + } + for _, msg := range metadata { + result, err := d.applyResult(msg, 0) + require.NoError(t, err) + assert.True(t, result.safe) + } + + _, err := d.applyResult(&pglogrepl.StreamStartMessageV2{Xid: 7, FirstSegment: 1}, 0) + require.NoError(t, err) + for _, msg := range metadata { + result, err := d.applyResult(msg, 0) + require.NoError(t, err) + assert.False(t, result.safe, "metadata inside a stream cannot advance the checkpoint") + } + _, err = d.applyResult(&pglogrepl.StreamStopMessageV2{}, 0) + require.NoError(t, err) + _, err = d.applyResult(&pglogrepl.StreamAbortMessageV2{Xid: 7, SubXid: 7}, 0) + require.NoError(t, err) +} + +func TestDecoderEnforcesTransactionChangeLimit(t *testing.T) { + d := newDecoder(decoderLimits{maxChanges: 1, maxBytes: defaultMaxTransactionBytes}) + seedRelAndBegin(t, d) + + _, err := d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("1", "a@w.ai")}, 0x20) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("2", "b@w.ai")}, 0x21) + require.ErrorIs(t, err, ErrTransactionLimit) +} + +func TestDecoderEnforcesTransactionByteLimitForStreamedSegments(t *testing.T) { + d := newStreamingDecoder(decoderLimits{maxChanges: 100, maxBytes: 1}) + _, err := d.apply(relV2(), 0) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(100, "1", "a@w.ai"), 0x20) + require.ErrorIs(t, err, ErrTransactionLimit) +} + +func TestStreamingDecoderEnforcesAggregateChangeLimitAcrossXIDs(t *testing.T) { + d := newStreamingDecoder(decoderLimits{ + maxChanges: 100, + maxBytes: defaultMaxTransactionBytes, + maxInflightChanges: 1, + maxInflightBytes: defaultMaxInflightBytes, + }) + _, err := d.apply(relV2(), 0) + require.NoError(t, err) + + _, err = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(100, "1", "first@w.ai"), 0x20) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + require.NoError(t, err) + + _, err = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 200, FirstSegment: 1}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(200, "2", "second@w.ai"), 0x30) + require.ErrorIs(t, err, ErrTransactionLimit) + assert.Equal(t, 1, d.inflightChanges) + + _, err = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x40}, 0) + require.NoError(t, err) + assert.Zero(t, d.inflightChanges) +} + +func TestStreamingDecoderEnforcesAggregateByteLimitAcrossXIDs(t *testing.T) { + rel := relV2() + rowBytes := estimateChangeBytes(&rel.RelationMessage, textTuple("1", "first@w.ai"), nil) + d := newStreamingDecoder(decoderLimits{ + maxChanges: 100, + maxBytes: defaultMaxTransactionBytes, + maxInflightChanges: 100, + maxInflightBytes: rowBytes, + }) + _, err := d.apply(rel, 0) + require.NoError(t, err) + + _, err = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(100, "1", "first@w.ai"), 0x20) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + require.NoError(t, err) + + _, err = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 200, FirstSegment: 1}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(200, "2", "second@w.ai"), 0x30) + require.ErrorIs(t, err, ErrTransactionLimit) + assert.Equal(t, rowBytes, d.inflightBytes) +} + +func TestStreamingDecoderEnforcesAggregateLimitAcrossOrdinaryAndStreamedTransactions(t *testing.T) { + d := newStreamingDecoder(decoderLimits{ + maxChanges: 100, + maxBytes: defaultMaxTransactionBytes, + maxInflightChanges: 2, + maxInflightBytes: defaultMaxInflightBytes, + }) + _, err := d.apply(relV2(), 0) + require.NoError(t, err) + + _, err = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(100, "1", "stream@w.ai"), 0x20) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + require.NoError(t, err) + + // Protocol v2 may interleave a regular transaction while a streamed + // transaction remains buffered. Both must count against the same bound. + _, err = d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x30, Xid: 7}, 0) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("2", "ordinary@w.ai")}, 0x31) + require.NoError(t, err) + assert.Equal(t, 2, d.inflightChanges) + + _, err = d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("3", "over-limit@w.ai")}, 0x32) + require.ErrorIs(t, err, ErrTransactionLimit) + assert.Equal(t, 2, d.inflightChanges) + + _, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x40}, 0) + require.NoError(t, err) + assert.Equal(t, 1, d.inflightChanges) + _, err = d.apply(&pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x50}, 0) + require.NoError(t, err) + assert.Zero(t, d.inflightChanges) } func TestTupleToMapNullAndToast(t *testing.T) { diff --git a/service/cdc/postgres/driver.go b/service/cdc/postgres/driver.go new file mode 100644 index 000000000..aacd50545 --- /dev/null +++ b/service/cdc/postgres/driver.go @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: MPL-2.0 + +package postgres + +import ( + "context" + "net" + "strconv" + "strings" + "sync" + + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/cdc" + "github.com/wippyai/runtime/api/supervisor" + cdcservice "github.com/wippyai/runtime/service/cdc" + entryutil "github.com/wippyai/runtime/system/entry" + "go.uber.org/zap" +) + +// Driver wires PostgreSQL CDC into the driver-neutral CDC manager. It only +// constructs a source; registry visibility, replacement, and supervisor +// lifecycle remain owned by service/cdc. +type Driver struct{} + +func NewDriver() cdcservice.Driver { return Driver{} } + +func (Driver) Kind() registry.Kind { return config.Postgres } + +func (Driver) Create(ctx context.Context, entry registry.Entry, deps cdcservice.Dependencies) (cdcservice.ManagedSource, error) { + if deps.Transcoder == nil { + return nil, ErrTranscoderRequired + } + cfg, err := entryutil.DecodeEntryConfig[config.Config](ctx, deps.Transcoder, entry) + if err != nil { + return nil, NewInvalidConfigError(err) + } + if err := cfg.Validate(); err != nil { + return nil, NewInvalidConfigError(err) + } + if err := validateConfigIdentifiers(cfg); err != nil { + return nil, NewInvalidConfigError(err) + } + standby, _ := cfg.StandbyDuration() + status, _ := cfg.StatusDuration() + replDSN, adminDSN, err := buildDSNs(cfg) + if err != nil { + return nil, err + } + log := deps.Logger + if log == nil { + log = zap.NewNop() + } + opts := SourceOptions{ + ReplDSN: replDSN, + AdminDSN: adminDSN, + Name: entry.ID.String(), + Slot: cfg.SlotName, + Publication: cfg.Publication, + Tables: cfg.Tables, + Temporary: cfg.Temporary, + Snapshot: cfg.Snapshot, + Streaming: cfg.Streaming, + Failover: cfg.Failover, + StandbyInterval: standby, + StatusInterval: status, + SnapshotFetchSize: cfg.SnapshotFetchSize, + MaxTransactionChanges: cfg.MaxTransactionChanges, + MaxTransactionBytes: cfg.MaxTransactionBytes, + MaxInflightChanges: cfg.MaxInflightChanges, + MaxInflightBytes: cfg.MaxInflightBytes, + Log: log.With(zap.String("id", entry.ID.String())), + } + return &sourceAdapter{ + source: NewSource(opts), + lifecycle: cfg.Lifecycle, + exclusiveKey: postgresExclusiveKey(cfg), + }, nil +} + +// sourceAdapter preserves the existing PostgreSQL source implementation while +// exposing the common context-aware CDC contract. The old source stream API +// remains private to this adapter, so no driver-specific shape leaks into the +// common manager or dispatcher. +type sourceAdapter struct { + source *Source + exclusiveKey string + lifecycle supervisor.LifecycleConfig + mu sync.RWMutex +} + +func (s *sourceAdapter) Info() config.SourceInfo { + s.mu.RLock() + source := s.source + s.mu.RUnlock() + source.mu.Lock() + state := source.state + sourceErr := source.sourceErr + source.mu.Unlock() + + info := config.SourceInfo{ + Kind: config.Postgres, + Name: source.name, + Slot: source.slot, + Publication: source.publication, + Tables: append([]string(nil), source.tables...), + // Streaming is a legacy field describing the configured pgoutput + // protocol mode, not the current lifecycle state. State is exposed by + // SourceState above. + Streaming: source.streaming, + Failover: source.failover, + Temporary: source.temporary, + Snapshot: source.snapshot, + State: postgresSourceState(state), + Capabilities: config.Capabilities{ + // Snapshot is an atomic per-subscriber handoff. The source Snapshot + // field is only the entry default for that capability. + Snapshot: true, + Durable: !source.temporary, + Replayable: false, + CapturesExternalWrites: true, + BeforeImages: false, + }, + } + if state == sourceFailed { + info.Faulted = true + } + if sourceErr != nil { + info.Error = sourceErr.Error() + } + return info +} + +func (s *sourceAdapter) Subscribe(ctx context.Context, opts config.StreamOptions) (config.Stream, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + if err := opts.Validate(); err != nil { + return nil, err + } + if opts.After != "" { + return nil, config.ErrUnsupported + } + s.mu.RLock() + source := s.source + s.mu.RUnlock() + return source.subscribe(ctx, opts) +} + +func (s *sourceAdapter) Start(ctx context.Context) (<-chan any, error) { + s.mu.RLock() + source := s.source + s.mu.RUnlock() + return source.Start(ctx) +} + +func (s *sourceAdapter) Stop(ctx context.Context) error { + s.mu.RLock() + source := s.source + s.mu.RUnlock() + return source.Stop(ctx) +} + +// Dispose is used only for a committed dynamic delete. The generic manager +// keeps a non-subscribable tombstone until this completes, so retries can +// finish cleanup. Ordinary Stop/replacement never drops the slot. +func (s *sourceAdapter) Dispose(ctx context.Context) error { + s.mu.RLock() + source := s.source + s.mu.RUnlock() + source.MarkForSlotDrop() + return source.Stop(ctx) +} + +func (s *sourceAdapter) LifecycleConfig() supervisor.LifecycleConfig { + return s.lifecycle +} + +// PostgreSQL replication slots are exclusive resources. The stable manager +// slot uses this key to perform a stop/start handoff for updates that retain +// the same slot, avoiding a concurrent replication-slot ownership error. +func (s *sourceAdapter) ExclusiveResourceKey() string { + s.mu.RLock() + key := s.exclusiveKey + s.mu.RUnlock() + return key +} + +func postgresExclusiveKey(cfg *config.Config) string { + host := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(cfg.Host), ".")) + endpoint := net.JoinHostPort(host, strconv.Itoa(cfg.Port)) + return "postgres/" + endpoint + "/slot/" + cfg.SlotName +} + +func postgresSourceState(state sourceState) config.SourceState { + switch state { + case sourceStarting: + return config.SourceStateStarting + case sourceRunning: + return config.SourceStateRunning + case sourceFailed: + return config.SourceStateFaulted + case sourceStopped: + return config.SourceStateStopped + default: + return config.SourceStateUnknown + } +} + +var _ cdcservice.ManagedSource = (*sourceAdapter)(nil) +var _ cdcservice.ExclusiveResource = (*sourceAdapter)(nil) +var _ cdcservice.Disposable = (*sourceAdapter)(nil) diff --git a/service/cdc/postgres/driver_test.go b/service/cdc/postgres/driver_test.go new file mode 100644 index 000000000..a618a1a27 --- /dev/null +++ b/service/cdc/postgres/driver_test.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: MPL-2.0 + +package postgres + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cconfig "github.com/wippyai/runtime/api/service/cdc" +) + +func TestSourceAdapterInfoReportsConservativeCapabilities(t *testing.T) { + source := NewSource(SourceOptions{ + Name: "app:events", + Slot: "events_slot", + Snapshot: true, + Streaming: true, + Temporary: true, + }) + terminalErr := errors.New("replication connection lost") + source.mu.Lock() + source.state = sourceFailed + source.sourceErr = terminalErr + source.mu.Unlock() + + adapter := &sourceAdapter{source: source} + info := adapter.Info() + + assert.Equal(t, cconfig.SourceStateFaulted, info.State) + assert.True(t, info.Faulted) + assert.Equal(t, terminalErr.Error(), info.Error) + assert.True(t, info.Snapshot, "entry snapshot field preserves the configured subscriber default") + assert.True(t, info.Streaming, "legacy streaming field preserves configured protocol mode") + assert.True(t, info.Capabilities.Snapshot, "per-consumer snapshots use the atomic handoff") + assert.False(t, info.Capabilities.Replayable, "After cursors are unsupported") + assert.False(t, info.Capabilities.Durable, "temporary slots are not durable") + assert.False(t, info.Capabilities.BeforeImages) +} + +func TestSourceAdapterSnapshotRequiresRunningGeneration(t *testing.T) { + source := NewSource(SourceOptions{Slot: "events_slot"}) + adapter := &sourceAdapter{source: source} + _, err := adapter.Subscribe(context.Background(), cconfig.StreamOptions{Snapshot: true}) + assert.ErrorIs(t, err, cconfig.ErrSourceNotReady) +} + +func TestSourceStopAfterDisposeCleanupIsIdempotent(t *testing.T) { + source := NewSource(SourceOptions{Slot: "events_slot"}) + source.dropDone.Store(true) + source.MarkForSlotDrop() + + // A completed cleanup is a tombstone: retries must not reopen a connection + // or attempt to drop the same slot again. + require.NoError(t, source.Stop(nil)) + require.NoError(t, source.Stop(nil)) +} + +func TestPostgresExclusiveKeyIsClusterWide(t *testing.T) { + first := &cconfig.Config{Host: "db.internal", Port: 5432, Database: "one", SlotName: "events"} + second := &cconfig.Config{Host: "db.internal", Port: 5432, Database: "two", SlotName: "events"} + + assert.Equal(t, postgresExclusiveKey(first), postgresExclusiveKey(second)) + assert.NotContains(t, postgresExclusiveKey(first), "password") +} diff --git a/service/cdc/postgres/errors.go b/service/cdc/postgres/errors.go index 43174e65c..2baccc89b 100644 --- a/service/cdc/postgres/errors.go +++ b/service/cdc/postgres/errors.go @@ -12,8 +12,14 @@ import ( var ( ErrUnknownRelation = errors.New("cdc: unknown relation id") + ErrInvalidTransaction = errors.New("cdc: invalid transaction message sequence") + ErrTransactionLimit = errors.New("cdc: transaction buffer limit exceeded") + ErrUnsupportedMessage = errors.New("cdc: unsupported logical replication message") ErrSourceClosed = errors.New("cdc: source is closed") + ErrSourceRunning = errors.New("cdc: source is already running") + ErrSourceStopping = errors.New("cdc: source is stopping") ErrNoPublication = errors.New("cdc: no publication and no tables configured") + ErrInvalidIdentifier = errors.New("cdc: invalid PostgreSQL identifier") ErrTranscoderRequired = apierror.New(apierror.Invalid, "transcoder is required").WithRetryable(apierror.False) ErrEventBusRequired = apierror.New(apierror.Invalid, "event bus is required").WithRetryable(apierror.False) ErrNoSourceStreamer = apierror.New(apierror.Internal, "cdc source streamer not available").WithRetryable(apierror.False) diff --git a/service/cdc/postgres/identifiers.go b/service/cdc/postgres/identifiers.go new file mode 100644 index 000000000..bb7e33823 --- /dev/null +++ b/service/cdc/postgres/identifiers.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: MPL-2.0 + +package postgres + +import ( + "fmt" + "strings" + "unicode" + "unicode/utf8" + + "github.com/lib/pq" + config "github.com/wippyai/runtime/api/service/cdc" +) + +// PostgreSQL stores ordinary identifiers in NameData, whose default +// NAMEDATALEN leaves 63 bytes for the identifier. Replication slot and +// publication names are identifiers in the replication command grammar. +const postgresIdentifierMaxBytes = 63 + +// PostgreSQL replication slot names use a narrower grammar than ordinary +// identifiers. The server validates them as lowercase ASCII names composed +// only of letters, digits, and underscores; quoting does not broaden that +// rule. Keep this check separate from publication/table identifier quoting. +func quoteReplicationSlotName(value string) (string, error) { + if value == "" || len(value) > postgresIdentifierMaxBytes { + return "", fmt.Errorf("%w: slot_name", ErrInvalidIdentifier) + } + for i := 0; i < len(value); i++ { + c := value[i] + if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '_' { + return "", fmt.Errorf("%w: slot_name", ErrInvalidIdentifier) + } + } + return pq.QuoteIdentifier(value), nil +} + +func quotePostgresIdentifier(value, field string) (string, error) { + if value == "" || value != strings.TrimSpace(value) || + len(value) > postgresIdentifierMaxBytes || !utf8.ValidString(value) { + return "", fmt.Errorf("%w: %s", ErrInvalidIdentifier, field) + } + for _, r := range value { + if unicode.IsControl(r) { + return "", fmt.Errorf("%w: %s", ErrInvalidIdentifier, field) + } + } + return pq.QuoteIdentifier(value), nil +} + +func validatePostgresIdentifier(value, field string) error { + _, err := quotePostgresIdentifier(value, field) + return err +} + +// validateConfigIdentifiers applies the same SQL-grammar checks to both the +// driver-backed source and the retained legacy manager path. Keeping this +// policy in one helper prevents either construction path from creating an +// auto-publication before rejecting its slot name. +func validateConfigIdentifiers(cfg *config.Config) error { + if _, err := quoteReplicationSlotName(cfg.SlotName); err != nil { + return err + } + if cfg.Publication != "" { + return validatePostgresIdentifier(cfg.Publication, "publication") + } + for _, table := range cfg.Tables { + if _, err := quoteQualifiedIdent(table); err != nil { + return err + } + } + _, err := quotePostgresIdentifier(cfg.SlotName+"_pub", "publication") + return err +} + +func quotePostgresLiteral(value, field string) (string, error) { + if value == "" || !utf8.ValidString(value) { + return "", fmt.Errorf("%w: %s", ErrInvalidIdentifier, field) + } + for _, r := range value { + if r == 0 || unicode.IsControl(r) { + return "", fmt.Errorf("%w: %s", ErrInvalidIdentifier, field) + } + } + return pq.QuoteLiteral(value), nil +} diff --git a/service/cdc/postgres/identifiers_test.go b/service/cdc/postgres/identifiers_test.go new file mode 100644 index 000000000..02f251a94 --- /dev/null +++ b/service/cdc/postgres/identifiers_test.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MPL-2.0 + +package postgres + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestQuoteReplicationSlotNameUsesServerGrammar(t *testing.T) { + for _, name := range []string{"events_2026_08", "slot0", strings.Repeat("x", postgresIdentifierMaxBytes)} { + quoted, err := quoteReplicationSlotName(name) + require.NoError(t, err, "name %q", name) + assert.Equal(t, `"`+name+`"`, quoted) + } + + for _, name := range []string{ + "", "Events", "events-name", `events"name`, "events name", + "événements", "slot\nname", string([]byte{0xff}), + strings.Repeat("x", postgresIdentifierMaxBytes+1), + } { + _, err := quoteReplicationSlotName(name) + assert.ErrorIs(t, err, ErrInvalidIdentifier, "name %q", name) + } +} + +func TestQuotePostgresIdentifierUsesServerIdentifierQuoting(t *testing.T) { + quoted, err := quotePostgresIdentifier(`publication"name`, "publication") + require.NoError(t, err) + assert.Equal(t, `"publication""name"`, quoted) + + literal, err := quotePostgresLiteral(`publication'name`, "publication") + require.NoError(t, err) + assert.Equal(t, `'publication''name'`, literal) + + qualified, err := quoteQualifiedIdent("public.accounts") + require.NoError(t, err) + assert.Equal(t, `"public"."accounts"`, qualified) +} + +func TestQuoteQualifiedIdentRejectsMalformedTable(t *testing.T) { + for _, table := range []string{"public.accounts.extra", ".accounts", "public."} { + _, err := quoteQualifiedIdent(table) + assert.ErrorIs(t, err, ErrInvalidIdentifier, "table %q", table) + } +} diff --git a/service/cdc/postgres/integration_lifecycle_test.go b/service/cdc/postgres/integration_lifecycle_test.go new file mode 100644 index 000000000..affb7bbfb --- /dev/null +++ b/service/cdc/postgres/integration_lifecycle_test.go @@ -0,0 +1,127 @@ +//go:build integration + +package postgres + +import ( + "context" + "database/sql" + "testing" + "time" + + "github.com/lib/pq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + freshFailureSlot = "wippy_cdc_fresh_failure" + autoPublicationSlot = "wippy_cdc_auto_pub" +) + +func TestFreshSlotIsDroppedWhenReplicationStartFails(t *testing.T) { + repl, admin := dsns(t) + db, err := sql.Open("postgres", admin) + require.NoError(t, err) + defer func() { _ = db.Close() }() + setupSchema(t, db) + dropNamedSlot(t, repl, freshFailureSlot) + defer dropNamedSlot(t, repl, freshFailureSlot) + + src := NewSource(SourceOptions{ + ReplDSN: repl, AdminDSN: admin, Slot: freshFailureSlot, + Publication: "wippy_cdc_missing_publication", + StandbyInterval: time.Millisecond, StatusInterval: time.Hour, + }) + status, err := src.Start(context.Background()) + require.NoError(t, err, "startup returns before the replication command is issued") + + select { + case <-statusClosed(status): + case <-time.After(10 * time.Second): + t.Fatal("source did not terminate after invalid replication publication") + } + assert.Eventually(t, func() bool { + var count int + if err := db.QueryRow(`SELECT count(*) FROM pg_replication_slots WHERE slot_name=$1`, freshFailureSlot).Scan(&count); err != nil { + return false + } + return count == 0 + }, 5*time.Second, 100*time.Millisecond, "fresh slot must be cleaned after StartReplication failure") +} + +func TestMissingSlotDeletesStaleCheckpointBeforeRecreate(t *testing.T) { + repl, admin := dsns(t) + db, err := sql.Open("postgres", admin) + require.NoError(t, err) + defer func() { _ = db.Close() }() + setupSchema(t, db) + dropNamedSlot(t, repl, freshFailureSlot) + defer dropNamedSlot(t, repl, freshFailureSlot) + + _, err = db.Exec(`INSERT INTO wippy_cdc_offsets (slot, lsn) VALUES ($1, $2)`, freshFailureSlot, "F/FFFFFFF") + require.NoError(t, err) + + src := NewSource(SourceOptions{ + ReplDSN: repl, AdminDSN: admin, Slot: freshFailureSlot, + Publication: "wippy_cdc_pub", StandbyInterval: 200 * time.Millisecond, + StatusInterval: time.Hour, + }) + status, err := src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + var count int + require.NoError(t, db.QueryRow(`SELECT count(*) FROM wippy_cdc_offsets WHERE slot=$1`, freshFailureSlot).Scan(&count)) + assert.Zero(t, count, "an offset from a removed slot must not survive recreation") + _ = status +} + +func TestAutoPublicationReconcilesTableMembership(t *testing.T) { + repl, admin := dsns(t) + db, err := sql.Open("postgres", admin) + require.NoError(t, err) + defer func() { _ = db.Close() }() + setupSchema(t, db) + const extraTable = "wippy_cdc_auto_extra" + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS ` + pq.QuoteIdentifier(extraTable) + ` (id bigint PRIMARY KEY)`) + require.NoError(t, err) + pubName := autoPublicationSlot + "_pub" + _, err = db.Exec(`DROP PUBLICATION IF EXISTS ` + pq.QuoteIdentifier(pubName)) + require.NoError(t, err) + dropNamedSlot(t, repl, autoPublicationSlot) + defer func() { + dropNamedSlot(t, repl, autoPublicationSlot) + _, _ = db.Exec(`DROP PUBLICATION IF EXISTS ` + pq.QuoteIdentifier(pubName)) + _, _ = db.Exec(`DROP TABLE IF EXISTS ` + pq.QuoteIdentifier(extraTable)) + }() + + first := NewSource(SourceOptions{ + ReplDSN: repl, AdminDSN: admin, Slot: autoPublicationSlot, + Tables: []string{"public.accounts"}, StandbyInterval: 200 * time.Millisecond, + StatusInterval: time.Hour, + }) + _, err = first.Start(context.Background()) + require.NoError(t, err) + require.NoError(t, first.Stop(context.Background())) + + second := NewSource(SourceOptions{ + ReplDSN: repl, AdminDSN: admin, Slot: autoPublicationSlot, + Tables: []string{extraTable}, StandbyInterval: 200 * time.Millisecond, + StatusInterval: time.Hour, + }) + _, err = second.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = second.Stop(context.Background()) }() + + rows, err := db.Query(`SELECT schemaname || '.' || tablename FROM pg_publication_tables WHERE pubname=$1 ORDER BY 1`, pubName) + require.NoError(t, err) + defer func() { _ = rows.Close() }() + var got []string + for rows.Next() { + var table string + require.NoError(t, rows.Scan(&table)) + got = append(got, table) + } + require.NoError(t, rows.Err()) + assert.Equal(t, []string{"public." + extraTable}, got) +} diff --git a/service/cdc/postgres/integration_lua_test.go b/service/cdc/postgres/integration_lua_test.go index 3312706a3..4690729ae 100644 --- a/service/cdc/postgres/integration_lua_test.go +++ b/service/cdc/postgres/integration_lua_test.go @@ -85,11 +85,10 @@ func TestLuaSeesRealRunningSourceAndItsChanges(t *testing.T) { require.NoError(t, sup.Start(supCtx)) manager := &Manager{ - bus: bus, - log: zap.NewNop(), - sources: map[registry.ID]*Source{}, - infos: map[registry.ID]cdcapi.SourceInfo{}, - infosByKey: map[string]registry.ID{}, + bus: bus, + log: zap.NewNop(), + sources: map[registry.ID]*Source{}, + infos: map[registry.ID]cdcapi.SourceInfo{}, } entryID := registry.NewID("test", "cdc-lua-e2e") diff --git a/service/cdc/postgres/integration_snapshot_test.go b/service/cdc/postgres/integration_snapshot_test.go index a9dd05d24..549da4aee 100644 --- a/service/cdc/postgres/integration_snapshot_test.go +++ b/service/cdc/postgres/integration_snapshot_test.go @@ -6,11 +6,13 @@ import ( "context" "database/sql" "errors" + "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + cdcapi "github.com/wippyai/runtime/api/service/cdc" ) func waitForSnapshotEmail(t *testing.T, b *changeCapture, email string, op Op, timeout time.Duration) RowChange { @@ -29,6 +31,31 @@ func waitForSnapshotEmail(t *testing.T, b *changeCapture, email string, op Op, t } } +func attachSnapshotCapture(t *testing.T, ctx context.Context, src *Source, capture *changeCapture, capacity ...int) { + t.Helper() + size := 8192 + if len(capacity) > 0 && capacity[0] > 0 { + size = capacity[0] + } + stream := src.Subscribe(cdcapi.StreamOptions{Snapshot: true, Buffer: size}) + require.NotNil(t, stream) + t.Cleanup(stream.Close) + go func() { + for { + select { + case change, ok := <-stream.Changes(): + if !ok { + return + } + capture.send(rowChangeFromAPI(change)) + case <-ctx.Done(): + stream.Close() + return + } + } + }() +} + func TestSnapshotBootstrapsExistingRows(t *testing.T) { repl, admin := dsns(t) db, err := sql.Open("postgres", admin) @@ -50,10 +77,9 @@ func TestSnapshotBootstrapsExistingRows(t *testing.T) { }) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - attachCapture(t, ctx, src, capture) - _, err = src.Start(ctx) require.NoError(t, err) + attachSnapshotCapture(t, ctx, src, capture) seen := map[string]Op{} deadline := time.After(15 * time.Second) @@ -113,9 +139,9 @@ func TestSnapshotPreservesNull(t *testing.T) { }) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - attachCapture(t, ctx, src, capture) _, err = src.Start(ctx) require.NoError(t, err) + attachSnapshotCapture(t, ctx, src, capture) rc := waitForSnapshotEmail(t, capture, "null@w.ai", OpSnapshot, 15*time.Second) assert.Nil(t, rc.After["note"], "NULL column must map to nil in snapshot row") @@ -125,7 +151,7 @@ func TestSnapshotPreservesNull(t *testing.T) { stopCancel() } -func TestSnapshotSkippedOnResume(t *testing.T) { +func TestSnapshotDefaultAppliesPerSubscriberAfterResume(t *testing.T) { repl, admin := dsns(t) db, err := sql.Open("postgres", admin) require.NoError(t, err) @@ -148,9 +174,9 @@ func TestSnapshotSkippedOnResume(t *testing.T) { } src := mk(capture) ctx, cancel := context.WithCancel(context.Background()) - attachCapture(t, ctx, src, capture) _, err = src.Start(ctx) require.NoError(t, err) + attachSnapshotCapture(t, ctx, src, capture) waitForSnapshotEmail(t, capture, "resume-base@w.ai", OpSnapshot, 15*time.Second) require.Eventually(t, func() bool { var raw string @@ -166,22 +192,25 @@ func TestSnapshotSkippedOnResume(t *testing.T) { src2 := mk(capture2) ctx2, cancel2 := context.WithCancel(context.Background()) defer cancel2() - attachCapture(t, ctx2, src2, capture2) _, err = src2.Start(ctx2) require.NoError(t, err) + attachSnapshotCapture(t, ctx2, src2, capture2) _, err = db.Exec(`INSERT INTO accounts (email, balance) VALUES ('resume-new@w.ai', 2)`) require.NoError(t, err) deadline := time.After(15 * time.Second) - got := false - for !got { + gotNew := false + for !gotNew { select { case rc := <-capture2.ch: - assert.NotEqual(t, OpSnapshot, rc.Op, "resume must not re-snapshot existing rows") - if em, _ := rc.After["email"].(string); em == "resume-new@w.ai" { + em, _ := rc.After["email"].(string) + if em == "resume-base@w.ai" { + assert.Equal(t, OpSnapshot, rc.Op, "entry snapshot is per subscriber, including resumed sources") + } + if em == "resume-new@w.ai" { assert.Equal(t, OpInsert, rc.Op) - got = true + gotNew = true } case <-deadline: t.Fatal("resumed source did not stream the new insert") @@ -212,23 +241,26 @@ func TestSnapshotFailureDropsSlotForCleanRetry(t *testing.T) { src := NewSource(SourceOptions{ ReplDSN: repl, AdminDSN: admin, Slot: itSlot, Publication: "wippy_cdc_pub", - Snapshot: true, StandbyInterval: 200 * time.Millisecond, StatusInterval: time.Hour, + StandbyInterval: 200 * time.Millisecond, StatusInterval: time.Hour, }) ctx, cancel := context.WithCancel(context.Background()) - status, err := src.Start(ctx) + _, err = src.Start(ctx) require.NoError(t, err) - + stream := src.Subscribe(cdcapi.StreamOptions{Snapshot: true, Buffer: 8}) + require.NotNil(t, stream) select { - case <-statusClosed(status): + case _, ok := <-stream.Changes(): + require.False(t, ok) case <-time.After(15 * time.Second): - t.Fatal("run did not exit after injected snapshot failure") + t.Fatal("subscriber snapshot did not fail") } + assert.ErrorContains(t, stream.Err(), "injected snapshot failure") + stream.Close() + assert.Equal(t, 1, slotCount(t, db, itSlot), "subscriber snapshot failure must not drop the source slot") cancel() - - assert.Equal(t, 0, slotCount(t, db, itSlot), "snapshot failure must drop the fresh slot for a clean retry") - var offsets int - require.NoError(t, db.QueryRow(`SELECT count(*) FROM wippy_cdc_offsets WHERE slot=$1`, itSlot).Scan(&offsets)) - assert.Equal(t, 0, offsets, "snapshot failure must delete the checkpoint") + stopCtx, stopCancel := context.WithTimeout(context.Background(), 5*time.Second) + require.NoError(t, src.Stop(stopCtx)) + stopCancel() snapshotFailpoint = nil capture2 := newChangeCapture() @@ -238,12 +270,80 @@ func TestSnapshotFailureDropsSlotForCleanRetry(t *testing.T) { }) ctx2, cancel2 := context.WithCancel(context.Background()) defer cancel2() - attachCapture(t, ctx2, src2, capture2) _, err = src2.Start(ctx2) require.NoError(t, err) + attachSnapshotCapture(t, ctx2, src2, capture2) waitForSnapshotEmail(t, capture2, "retry@w.ai", OpSnapshot, 15*time.Second) stopCtx, sc := context.WithTimeout(context.Background(), 5*time.Second) require.NoError(t, src2.Stop(stopCtx)) sc() } + +func TestPerSubscriberSnapshotHandoffUsesCommitFence(t *testing.T) { + repl, admin := dsns(t) + db, err := sql.Open("postgres", admin) + require.NoError(t, err) + defer func() { _ = db.Close() }() + setupSchema(t, db) + const slot = "wippy_cdc_dynamic_snapshot" + dropNamedSlot(t, repl, slot) + defer dropNamedSlot(t, repl, slot) + + _, err = db.Exec(`DELETE FROM accounts`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO accounts (email, balance) VALUES ('before@w.ai', 1)`) + require.NoError(t, err) + + src := NewSource(SourceOptions{ + ReplDSN: repl, AdminDSN: admin, Slot: slot, Publication: "wippy_cdc_pub", + StandbyInterval: 200 * time.Millisecond, StatusInterval: time.Hour, + }) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + fenceReady := make(chan struct{}) + releaseSnapshot := make(chan struct{}) + var once sync.Once + snapshotFailpoint = func() error { + once.Do(func() { close(fenceReady) }) + <-releaseSnapshot + return nil + } + defer func() { snapshotFailpoint = nil }() + + stream := src.Subscribe(cdcapi.StreamOptions{Snapshot: true, Buffer: 64}) + require.NotNil(t, stream) + defer stream.Close() + select { + case <-fenceReady: + case <-time.After(15 * time.Second): + t.Fatal("subscriber snapshot did not establish its exported fence") + } + _, err = db.Exec(`INSERT INTO accounts (email, balance) VALUES ('after@w.ai', 2)`) + require.NoError(t, err) + close(releaseSnapshot) + + seenBefore := false + seenAfter := false + deadline := time.After(15 * time.Second) + for !seenBefore || !seenAfter { + select { + case change, ok := <-stream.Changes(): + require.True(t, ok, "snapshot stream closed: %v", stream.Err()) + email, _ := change.After["email"].(string) + switch email { + case "before@w.ai": + require.Equal(t, OpSnapshot, Op(change.Op)) + seenBefore = true + case "after@w.ai": + require.Equal(t, OpInsert, Op(change.Op)) + require.NotEmpty(t, change.CommitLSN) + seenAfter = true + } + case <-deadline: + t.Fatalf("snapshot/live handoff incomplete: before=%v after=%v", seenBefore, seenAfter) + } + } +} diff --git a/service/cdc/postgres/limits.go b/service/cdc/postgres/limits.go new file mode 100644 index 000000000..38a6f97c8 --- /dev/null +++ b/service/cdc/postgres/limits.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MPL-2.0 + +package postgres + +import ( + config "github.com/wippyai/runtime/api/service/cdc" +) + +const ( + // These aliases keep the decoder package independent from entry parsing + // while making the finite defaults owned by the public CDC configuration. + defaultMaxTransactionChanges = config.DefaultPostgresMaxTransactionChanges + defaultMaxTransactionBytes = config.DefaultPostgresMaxTransactionBytes + defaultMaxInflightChanges = config.DefaultPostgresMaxInflightChanges + defaultMaxInflightBytes = config.DefaultPostgresMaxInflightBytes +) + +type decoderLimits struct { + maxChanges int + maxBytes int64 + maxInflightChanges int + maxInflightBytes int64 +} + +func defaultDecoderLimits() decoderLimits { + return decoderLimits{ + maxChanges: defaultMaxTransactionChanges, + maxBytes: defaultMaxTransactionBytes, + maxInflightChanges: defaultMaxInflightChanges, + maxInflightBytes: defaultMaxInflightBytes, + } +} + +func normalizeDecoderLimits(limits decoderLimits) decoderLimits { + defaults := defaultDecoderLimits() + if limits.maxChanges <= 0 { + limits.maxChanges = defaults.maxChanges + } + if limits.maxBytes <= 0 { + limits.maxBytes = defaults.maxBytes + } + if limits.maxInflightChanges <= 0 { + limits.maxInflightChanges = defaults.maxInflightChanges + } + if limits.maxInflightBytes <= 0 { + limits.maxInflightBytes = defaults.maxInflightBytes + } + return limits +} diff --git a/service/cdc/postgres/manager.go b/service/cdc/postgres/manager.go index 7ef913337..d76796c1d 100644 --- a/service/cdc/postgres/manager.go +++ b/service/cdc/postgres/manager.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "net/url" + "sort" "strconv" "sync" @@ -20,14 +21,17 @@ import ( "go.uber.org/zap" ) +// Manager is the legacy PostgreSQL-specific registry and lifecycle wrapper. +// +// Deprecated: use service/cdc.Manager with NewDriver so source identity and +// lifecycle are owned by the driver-neutral CDC manager. type Manager struct { - dtt payload.Transcoder - bus event.Bus - log *zap.Logger - sources map[registry.ID]*Source - infos map[registry.ID]config.SourceInfo - infosByKey map[string]registry.ID - mu sync.Mutex + dtt payload.Transcoder + bus event.Bus + log *zap.Logger + sources map[registry.ID]*Source + infos map[registry.ID]config.SourceInfo + mu sync.Mutex } func NewManager(dtt payload.Transcoder, bus event.Bus, log *zap.Logger) (*Manager, error) { @@ -41,12 +45,11 @@ func NewManager(dtt payload.Transcoder, bus event.Bus, log *zap.Logger) (*Manage log = zap.NewNop() } return &Manager{ - dtt: dtt, - bus: bus, - log: log, - sources: make(map[registry.ID]*Source), - infos: make(map[registry.ID]config.SourceInfo), - infosByKey: make(map[string]registry.ID), + dtt: dtt, + bus: bus, + log: log, + sources: make(map[registry.ID]*Source), + infos: make(map[registry.ID]config.SourceInfo), }, nil } @@ -68,7 +71,9 @@ func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { if err := cfg.Validate(); err != nil { return NewInvalidConfigError(err) } - + if err := validateConfigIdentifiers(cfg); err != nil { + return NewInvalidConfigError(err) + } standby, _ := cfg.StandbyDuration() status, _ := cfg.StatusDuration() replDSN, adminDSN, err := buildDSNs(cfg) @@ -76,20 +81,24 @@ func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { return err } src := NewSource(SourceOptions{ - ReplDSN: replDSN, - AdminDSN: adminDSN, - Name: entry.ID.String(), - Slot: cfg.SlotName, - Publication: cfg.Publication, - Tables: cfg.Tables, - Temporary: cfg.Temporary, - Snapshot: cfg.Snapshot, - Streaming: cfg.Streaming, - Failover: cfg.Failover, - StandbyInterval: standby, - StatusInterval: status, - SnapshotFetchSize: cfg.SnapshotFetchSize, - Log: m.log.With(zap.String("id", entry.ID.String())), + ReplDSN: replDSN, + AdminDSN: adminDSN, + Name: entry.ID.String(), + Slot: cfg.SlotName, + Publication: cfg.Publication, + Tables: cfg.Tables, + Temporary: cfg.Temporary, + Snapshot: cfg.Snapshot, + Streaming: cfg.Streaming, + Failover: cfg.Failover, + StandbyInterval: standby, + StatusInterval: status, + SnapshotFetchSize: cfg.SnapshotFetchSize, + MaxTransactionChanges: cfg.MaxTransactionChanges, + MaxTransactionBytes: cfg.MaxTransactionBytes, + MaxInflightChanges: cfg.MaxInflightChanges, + MaxInflightBytes: cfg.MaxInflightBytes, + Log: m.log.With(zap.String("id", entry.ID.String())), }) m.sources[entry.ID] = src @@ -116,7 +125,9 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { if err := cfg.Validate(); err != nil { return NewInvalidConfigError(err) } - + if err := validateConfigIdentifiers(cfg); err != nil { + return NewInvalidConfigError(err) + } replDSN, adminDSN, err := buildDSNs(cfg) if err != nil { return err @@ -132,20 +143,24 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { standby, _ := cfg.StandbyDuration() status, _ := cfg.StatusDuration() src := NewSource(SourceOptions{ - ReplDSN: replDSN, - AdminDSN: adminDSN, - Name: entry.ID.String(), - Slot: cfg.SlotName, - Publication: cfg.Publication, - Tables: cfg.Tables, - Temporary: cfg.Temporary, - Snapshot: cfg.Snapshot, - Streaming: cfg.Streaming, - Failover: cfg.Failover, - StandbyInterval: standby, - StatusInterval: status, - SnapshotFetchSize: cfg.SnapshotFetchSize, - Log: m.log.With(zap.String("id", entry.ID.String())), + ReplDSN: replDSN, + AdminDSN: adminDSN, + Name: entry.ID.String(), + Slot: cfg.SlotName, + Publication: cfg.Publication, + Tables: cfg.Tables, + Temporary: cfg.Temporary, + Snapshot: cfg.Snapshot, + Streaming: cfg.Streaming, + Failover: cfg.Failover, + StandbyInterval: standby, + StatusInterval: status, + SnapshotFetchSize: cfg.SnapshotFetchSize, + MaxTransactionChanges: cfg.MaxTransactionChanges, + MaxTransactionBytes: cfg.MaxTransactionBytes, + MaxInflightChanges: cfg.MaxInflightChanges, + MaxInflightBytes: cfg.MaxInflightBytes, + Log: m.log.With(zap.String("id", entry.ID.String())), }) m.sources[entry.ID] = src m.storeInfo(entry, cfg) @@ -181,16 +196,10 @@ func (m *Manager) storeInfo(entry registry.Entry, cfg *config.Config) { Snapshot: cfg.Snapshot, } m.infos[entry.ID] = info - m.infosByKey[info.Slot] = entry.ID } func (m *Manager) removeInfo(id registry.ID) { - if info, ok := m.infos[id]; ok { - if current, present := m.infosByKey[info.Slot]; present && current == id { - delete(m.infosByKey, info.Slot) - } - delete(m.infos, id) - } + delete(m.infos, id) } func (m *Manager) List() []config.SourceInfo { @@ -201,6 +210,7 @@ func (m *Manager) List() []config.SourceInfo { for _, info := range m.infos { out = append(out, info) } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) return out } @@ -208,15 +218,9 @@ func (m *Manager) Get(name string) (config.SourceInfo, bool) { m.mu.Lock() defer m.mu.Unlock() - if id, ok := m.infosByKey[name]; ok { - if info, present := m.infos[id]; present { - return info, true - } - } - for _, info := range m.infos { - if info.Name == name { - return info, true - } + id := registry.ParseID(name) + if info, ok := m.infos[id]; ok { + return info, true } return config.SourceInfo{}, false } @@ -228,21 +232,21 @@ func (m *Manager) Stream(_ context.Context, name string, opts config.StreamOptio if !ok { return nil, config.SourceInfo{}, NewServiceNotFoundError(registry.ParseID(name)) } - return src.Subscribe(opts), info, nil + stream := src.Subscribe(opts) + if stream == nil { + return nil, info, config.ErrSourceNotReady + } + return stream, info, nil } func (m *Manager) lookupSourceLocked(name string) (*Source, config.SourceInfo, bool) { - if id, ok := m.infosByKey[name]; ok { - if src := m.sources[id]; src != nil { - return src, m.infos[id], true - } + id := registry.ParseID(name) + info, ok := m.infos[id] + if !ok { + return nil, config.SourceInfo{}, false } - for id, info := range m.infos { - if info.Name == name { - if src := m.sources[id]; src != nil { - return src, info, true - } - } + if src := m.sources[id]; src != nil { + return src, info, true } return nil, config.SourceInfo{}, false } diff --git a/service/cdc/postgres/manager_test.go b/service/cdc/postgres/manager_test.go index 991733e88..d8d494a51 100644 --- a/service/cdc/postgres/manager_test.go +++ b/service/cdc/postgres/manager_test.go @@ -190,9 +190,8 @@ func TestBuildDSNs(t *testing.T) { func newInspectorManager() *Manager { return &Manager{ - sources: map[registry.ID]*Source{}, - infos: map[registry.ID]config.SourceInfo{}, - infosByKey: map[string]registry.ID{}, + sources: map[registry.ID]*Source{}, + infos: map[registry.ID]config.SourceInfo{}, } } @@ -215,12 +214,12 @@ func TestManagerStoreAndListInfos(t *testing.T) { sort.Strings(slots) assert.Equal(t, []string{"slot_a", "slot_b"}, slots) - a, ok := m.Get("slot_a") + a, ok := m.Get("test:id-a") require.True(t, ok) assert.Equal(t, "pub_a", a.Publication) assert.True(t, a.Streaming) - b, ok := m.Get("slot_b") + b, ok := m.Get("test:id-b") require.True(t, ok) assert.Equal(t, []string{"public.t"}, b.Tables) @@ -232,20 +231,14 @@ func TestManagerStoreAndListInfos(t *testing.T) { assert.Equal(t, infos[0].Slot, byID.Slot) } -func TestManagerStreamBySlotAndID(t *testing.T) { +func TestManagerStreamByID(t *testing.T) { m := newInspectorManager() id := registry.NewID("test", "id-stream") src := NewSource(SourceOptions{Name: id.String(), Slot: "slot_stream"}) m.sources[id] = src m.storeInfo(registry.Entry{ID: id, Kind: config.Postgres}, &config.Config{SlotName: "slot_stream", Tables: []string{"public.accounts"}}) - stream, info, err := m.Stream(context.Background(), "slot_stream", config.StreamOptions{Buffer: 2}) - require.NoError(t, err) - require.NotNil(t, stream) - assert.Equal(t, "slot_stream", info.Slot) - stream.Close() - - stream, info, err = m.Stream(context.Background(), id.String(), config.StreamOptions{}) + stream, info, err := m.Stream(context.Background(), id.String(), config.StreamOptions{Buffer: 2}) require.NoError(t, err) require.NotNil(t, stream) assert.Equal(t, id.String(), info.Name) @@ -260,11 +253,11 @@ func TestManagerRemoveInfo(t *testing.T) { m.removeInfo(idX) assert.Empty(t, m.List()) - _, ok := m.Get("slot_x") + _, ok := m.Get(idX.String()) assert.False(t, ok) } -func TestManagerCollidingSlotsDoNotLeakIndex(t *testing.T) { +func TestManagerCollidingSlotsRemainDistinctByID(t *testing.T) { m := newInspectorManager() id1 := registry.NewID("test", "id-1") id2 := registry.NewID("test", "id-2") @@ -273,14 +266,13 @@ func TestManagerCollidingSlotsDoNotLeakIndex(t *testing.T) { require.Len(t, m.List(), 2) - m.removeInfo(id1) - got, ok := m.Get("shared") + got, ok := m.Get(id1.String()) require.True(t, ok) - assert.Equal(t, id2.String(), got.Name) + assert.Equal(t, id1.String(), got.Name) - m.removeInfo(id2) - _, ok = m.Get("shared") - assert.False(t, ok) + got, ok = m.Get(id2.String()) + require.True(t, ok) + assert.Equal(t, id2.String(), got.Name) } func TestBuildDSNsCarriesOptions(t *testing.T) { diff --git a/service/cdc/postgres/service.go b/service/cdc/postgres/service.go index 09bfa7b1c..48b359fe3 100644 --- a/service/cdc/postgres/service.go +++ b/service/cdc/postgres/service.go @@ -4,6 +4,7 @@ package postgres import ( "context" + "crypto/sha256" "database/sql" "errors" "fmt" @@ -23,9 +24,10 @@ import ( ) const ( - retainedWALGauge = "wippy_cdc_retained_wal_bytes" - changesCounter = "wippy_cdc_changes_total" - errorsCounter = "wippy_cdc_errors_total" + retainedWALGauge = "wippy_cdc_retained_wal_bytes" + changesCounter = "wippy_cdc_changes_total" + errorsCounter = "wippy_cdc_errors_total" + transactionLimitCounter = "wippy_cdc_transaction_limit_total" ) const ( @@ -52,42 +54,93 @@ type SourceOptions struct { StatusInterval time.Duration SnapshotFetchSize int Temporary bool - Snapshot bool - Streaming bool - Failover bool + // Snapshot makes the atomic snapshot handoff the default for each + // subscriber. It is an entry default; Start never emits a source-global + // snapshot. + Snapshot bool + Streaming bool + Failover bool + // MaxTransactionChanges bounds the number of row changes retained before + // an ordinary or streamed transaction commits. Zero uses the safe default. + MaxTransactionChanges int + // MaxTransactionBytes bounds the estimated memory retained for one + // ordinary or streamed transaction. Zero uses the safe default. + MaxTransactionBytes int64 + // MaxInflightChanges bounds all uncommitted row changes across interleaved + // ordinary and streamed transactions. Zero uses the safe default. + MaxInflightChanges int + // MaxInflightBytes bounds estimated memory retained by all uncommitted + // ordinary and streamed transactions. Zero uses the safe default. + MaxInflightBytes int64 } type Source struct { - log *zap.Logger - coll metrics.Collector - injectedCP Checkpointer - cancel context.CancelFunc - done chan struct{} - subs map[uint64]*sourceSubscription - replDSN string - adminDSN string - name string - slot string - publication string - tables []string - standbyInterval time.Duration - statusInterval time.Duration - mu sync.Mutex - subMu sync.RWMutex - nextSubID uint64 - snapshotFetchSize int - temporary bool - snapshot bool - streaming bool - failover bool - stopped atomic.Bool - dropSlot atomic.Bool + coll metrics.Collector + injectedCP Checkpointer + sourceErr error + log *zap.Logger + cancel context.CancelFunc + done chan struct{} + subs map[uint64]*sourceSubscription + streamNotify chan struct{} + snapshotGate chan struct{} // one temporary logical snapshot per source + replDSN string + adminDSN string + name string + slot string + publication string + tables []string + standbyInterval time.Duration + statusInterval time.Duration + nextSubID uint64 + snapshotFetchSize int + maxTransactionChanges int + maxTransactionBytes int64 + maxInflightChanges int + maxInflightBytes int64 + snapshotWG sync.WaitGroup + streamPosition pglogrepl.LSN + subMu sync.RWMutex + mu sync.Mutex + dropMu sync.Mutex + dropSlot atomic.Bool + dropDone atomic.Bool + temporary bool + snapshot bool + streaming bool + failover bool + permanentlyClosed bool + state sourceState } +type sourceState uint8 + +const ( + sourceNew sourceState = iota + sourceStarting + sourceRunning + sourceStopping + sourceFailed + sourceStopped +) + var snapshotFailpoint func() error func (s *Source) MarkForSlotDrop() { s.dropSlot.Store(true) + s.mu.Lock() + s.permanentlyClosed = true + s.mu.Unlock() +} + +// Close permanently retires a source. Stop alone is restartable so a +// supervisor can recover a failed generation; callers removing a source from +// the registry should use Close when the instance must not be started again. +func (s *Source) Close(ctx context.Context) error { + s.mu.Lock() + s.permanentlyClosed = true + s.mu.Unlock() + return s.Stop(ctx) } func NewSource(opts SourceOptions) *Source { @@ -107,109 +160,234 @@ func NewSource(opts SourceOptions) *Source { if fetch <= 0 { fetch = defaultSnapshotFetchSize } + limits := normalizeDecoderLimits(decoderLimits{ + maxChanges: opts.MaxTransactionChanges, + maxBytes: opts.MaxTransactionBytes, + maxInflightChanges: opts.MaxInflightChanges, + maxInflightBytes: opts.MaxInflightBytes, + }) return &Source{ - log: log, - injectedCP: opts.Checkpoint, - replDSN: opts.ReplDSN, - adminDSN: opts.AdminDSN, - name: opts.Name, - slot: opts.Slot, - publication: opts.Publication, - tables: opts.Tables, - subs: make(map[uint64]*sourceSubscription), - temporary: opts.Temporary, - snapshot: opts.Snapshot, - streaming: opts.Streaming, - failover: opts.Failover, - standbyInterval: standby, - statusInterval: status, - snapshotFetchSize: fetch, + log: log, + injectedCP: opts.Checkpoint, + replDSN: opts.ReplDSN, + adminDSN: opts.AdminDSN, + name: opts.Name, + slot: opts.Slot, + publication: opts.Publication, + tables: append([]string(nil), opts.Tables...), + subs: make(map[uint64]*sourceSubscription), + temporary: opts.Temporary, + snapshot: opts.Snapshot, + streaming: opts.Streaming, + failover: opts.Failover, + standbyInterval: standby, + statusInterval: status, + snapshotFetchSize: fetch, + maxTransactionChanges: limits.maxChanges, + maxTransactionBytes: limits.maxBytes, + maxInflightChanges: limits.maxInflightChanges, + maxInflightBytes: limits.maxInflightBytes, + streamNotify: make(chan struct{}), + snapshotGate: make(chan struct{}, 1), } } func (s *Source) Start(ctx context.Context) (<-chan any, error) { - if s.stopped.Load() { + if ctx == nil { + ctx = context.Background() + } + runCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + + s.mu.Lock() + if s.permanentlyClosed { + s.mu.Unlock() + cancel() return nil, ErrSourceClosed } + switch s.state { + case sourceStarting, sourceRunning: + s.mu.Unlock() + cancel() + return nil, ErrSourceRunning + case sourceStopping: + s.mu.Unlock() + cancel() + return nil, ErrSourceStopping + default: + s.state = sourceStarting + s.cancel = cancel + s.done = done + // A failed snapshot/start may have dropped and checkpoint-cleaned the + // previous slot. This start may create a new slot generation, so the + // destructive cleanup marker must apply to that generation as well. + s.dropDone.Store(false) + } + s.mu.Unlock() + + failStart := func(startErr error) { + cancel() + s.mu.Lock() + if s.done == done { + switch s.state { + case sourceStopping: + s.state = sourceStopped + case sourceStarting: + s.state = sourceFailed + s.sourceErr = startErr + } + s.cancel = nil + close(done) + } + s.mu.Unlock() + } adminDB, err := sql.Open("postgres", s.adminDSN) if err != nil { - return nil, fmt.Errorf("open admin connection: %w", err) + startErr := fmt.Errorf("open admin connection: %w", err) + failStart(startErr) + return nil, startErr } adminDB.SetMaxOpenConns(2) adminDB.SetMaxIdleConns(1) - if err := adminDB.PingContext(ctx); err != nil { + if err := adminDB.PingContext(runCtx); err != nil { _ = adminDB.Close() - return nil, fmt.Errorf("ping admin connection: %w", err) + startErr := fmt.Errorf("ping admin connection: %w", err) + failStart(startErr) + return nil, startErr } cp := s.injectedCP if cp == nil { - dbcp, cpErr := NewDBCheckpointer(ctx, adminDB) + dbcp, cpErr := NewDBCheckpointer(runCtx, adminDB) if cpErr != nil { _ = adminDB.Close() + failStart(cpErr) return nil, cpErr } cp = dbcp } - publication, err := s.ensurePublication(ctx, adminDB) + publication, err := s.ensurePublication(runCtx, adminDB) if err != nil { _ = adminDB.Close() + failStart(err) return nil, err } - conn, err := pgconn.Connect(ctx, s.replDSN) + conn, err := pgconn.Connect(runCtx, s.replDSN) if err != nil { _ = adminDB.Close() - return nil, fmt.Errorf("replication connect: %w", err) + startErr := fmt.Errorf("replication connect: %w", err) + failStart(startErr) + return nil, startErr } - sysident, err := pglogrepl.IdentifySystem(ctx, conn) + sysident, err := pglogrepl.IdentifySystem(runCtx, conn) if err != nil { - _ = conn.Close(ctx) + _ = conn.Close(context.Background()) _ = adminDB.Close() - return nil, fmt.Errorf("identify system: %w", err) + startErr := fmt.Errorf("identify system: %w", err) + failStart(startErr) + return nil, startErr } - startLSN, snapshotName, err := s.prepareSlot(ctx, conn, adminDB, cp, sysident.XLogPos) + startLSN, slotCreated, err := s.prepareSlot(runCtx, conn, adminDB, cp, sysident.XLogPos) if err != nil { - _ = conn.Close(ctx) + _ = conn.Close(context.Background()) _ = adminDB.Close() + if slotCreated { + s.cleanupFreshSlot() + } + failStart(err) return nil, err } - runCtx, cancel := context.WithCancel(ctx) status := make(chan any, 8) - done := make(chan struct{}) s.mu.Lock() - s.cancel = cancel - s.done = done + if s.state != sourceStarting { + // Stop may have been requested while the synchronous connection and + // slot setup was in progress. Do not publish a source that is already + // being stopped. + s.mu.Unlock() + _ = conn.Close(context.Background()) + _ = adminDB.Close() + if slotCreated { + s.cleanupFreshSlot() + } + failStart(ErrSourceClosed) + return nil, ErrSourceClosed + } + s.state = sourceRunning + s.sourceErr = nil + s.publication = publication + s.streamPosition = startLSN + if s.streamNotify == nil { + s.streamNotify = make(chan struct{}) + } s.mu.Unlock() s.log.Info("cdc source started", zap.String("slot", s.slot), zap.String("publication", publication), zap.String("start_lsn", startLSN.String()), - zap.Bool("snapshot", snapshotName != "")) + zap.Bool("snapshot", s.snapshot)) select { case status <- "cdc replication started": default: } - s.coll = metrics.GetCollector(ctx) - go s.run(runCtx, conn, adminDB, cp, startLSN, snapshotName, publication, s.coll, status, done) + s.coll = metrics.GetCollector(runCtx) + go s.run(runCtx, conn, adminDB, cp, startLSN, slotCreated, publication, s.coll, status, done) return status, nil } func (s *Source) Stop(ctx context.Context) error { - if !s.stopped.CompareAndSwap(false, true) { - return nil + if ctx == nil { + ctx = context.Background() } - defer s.closeSubscriptions() s.mu.Lock() + if s.state == sourceStopped { + drop := s.dropSlot.Load() + s.mu.Unlock() + if drop { + return s.dropSlotAndCheckpoint(ctx) + } + return nil + } + if s.state == sourceNew || s.state == sourceFailed { + if s.state == sourceNew { + // Keep the generation stopping until all snapshot workers have + // joined. This prevents a concurrent Start from resetting state + // while a worker can still call WaitGroup.Done. + s.state = sourceStopping + s.cancel = nil + s.mu.Unlock() + s.closeSubscriptions() + if err := s.waitSnapshots(ctx); err != nil { + return err + } + s.mu.Lock() + if s.state == sourceStopping { + s.state = sourceStopped + } + s.mu.Unlock() + if s.dropSlot.Load() { + return s.dropSlotAndCheckpoint(ctx) + } + return nil + } + // A replication run marks the source failed before its deferred + // cleanup closes done. Keep the source stopping until that run has + // fully released its connections; replacement and slot deletion must + // not race the failed generation. + s.state = sourceStopping + } + if s.state == sourceStarting || s.state == sourceRunning { + s.state = sourceStopping + } cancel := s.cancel done := s.done s.mu.Unlock() @@ -217,81 +395,255 @@ func (s *Source) Stop(ctx context.Context) error { if cancel != nil { cancel() } + s.closeSubscriptions() if done != nil { select { case <-done: case <-ctx.Done(): return ctx.Err() } + } else { + s.mu.Lock() + if s.state == sourceStopping { + s.state = sourceStopped + s.cancel = nil + } + s.mu.Unlock() + } + if err := s.waitSnapshots(ctx); err != nil { + return err } + s.mu.Lock() + if s.state == sourceStopping { + s.state = sourceStopped + s.cancel = nil + } + s.mu.Unlock() - if s.dropSlot.Load() && !s.temporary { + if s.dropSlot.Load() { return s.dropSlotAndCheckpoint(ctx) } return nil } +func (s *Source) startSnapshot(ctx context.Context, sub *sourceSubscription) { + snapshotCtx, cancel := context.WithCancel(ctx) + s.snapshotWG.Add(1) + snapshotDone := make(chan struct{}) + if !sub.registerSnapshot(cancel, snapshotDone) { + s.snapshotWG.Done() + cancel() + return + } + go func() { + defer s.snapshotWG.Done() + defer sub.finishSnapshotWorker() + watchDone := make(chan struct{}) + go func() { + select { + case <-sub.done: + cancel() + case <-snapshotCtx.Done(): + case <-watchDone: + } + }() + defer close(watchDone) + + if err := s.acquireSnapshot(snapshotCtx); err != nil { + sub.finishSnapshot(0, err) + return + } + defer s.releaseSnapshot() + + fence, err := s.snapshotCurrentTo(snapshotCtx, sub) + if err != nil { + sub.finishSnapshot(0, err) + return + } + sub.finishSnapshot(fence, nil) + }() +} + +func (s *Source) acquireSnapshot(ctx context.Context) error { + select { + case s.snapshotGate <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *Source) releaseSnapshot() { + <-s.snapshotGate +} + +func (s *Source) waitSnapshots(ctx context.Context) error { + done := make(chan struct{}) + go func() { + s.snapshotWG.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// advanceStreamPosition publishes the replication receive watermark after a +// complete XLogData message has been decoded and emitted, or after a server +// keepalive reports its WAL end. Snapshot handoff waits for this watermark +// before releasing its pending live queue, so a change at or before the +// exported snapshot fence cannot arrive late and be duplicated after the +// handoff. It never updates the transaction-safe checkpoint. +func (s *Source) advanceStreamPosition(done chan struct{}, position pglogrepl.LSN) { + s.mu.Lock() + defer s.mu.Unlock() + if s.done != done || position <= s.streamPosition { + return + } + s.streamPosition = position + close(s.streamNotify) + s.streamNotify = make(chan struct{}) +} + +func (s *Source) observeKeepalive(done chan struct{}, keepalive pglogrepl.PrimaryKeepaliveMessage) { + s.advanceStreamPosition(done, keepalive.ServerWALEnd) +} + +func (s *Source) waitStreamPosition(ctx context.Context, fence pglogrepl.LSN) error { + for { + s.mu.Lock() + if s.streamPosition >= fence { + s.mu.Unlock() + return nil + } + notify := s.streamNotify + s.mu.Unlock() + select { + case <-notify: + case <-ctx.Done(): + return ctx.Err() + } + } +} + +func (s *Source) finishRunGeneration(done chan struct{}) (bool, []*sourceSubscription) { + s.mu.Lock() + defer s.mu.Unlock() + if s.done != done { + return false, nil + } + switch s.state { + case sourceRunning, sourceStarting: + s.state = sourceFailed + } + s.cancel = nil + return true, s.detachSubscriptionsLocked() +} + func (s *Source) run( ctx context.Context, conn *pgconn.PgConn, adminDB *sql.DB, cp Checkpointer, startLSN pglogrepl.LSN, - snapshotName string, + slotCreated bool, publication string, mc metrics.Collector, status chan any, done chan struct{}, ) { - defer close(done) + defer func() { + current, subs := s.finishRunGeneration(done) + if current { + s.closeDetachedSubscriptions(subs, nil) + } + close(done) + }() defer close(status) - defer s.closeSubscriptions() defer func() { _ = adminDB.Close() }() defer func() { _ = conn.Close(context.Background()) }() - if snapshotName != "" { - if err := s.snapshotExisting(ctx, adminDB, publication, snapshotName); err != nil { - s.abortFreshSnapshot(conn) - s.fail(ctx, status, err) - return - } - } - protoVersion := config.ProtocolVersion if s.streaming { protoVersion = config.StreamingProtocolVersion } + publicationLiteral, err := quotePostgresLiteral(publication, "publication") + if err != nil { + s.abortFreshSlot(conn, slotCreated) + s.fail(ctx, status, err) + return + } pluginArgs := []string{ fmt.Sprintf("proto_version '%d'", protoVersion), - fmt.Sprintf("publication_names '%s'", publication), + fmt.Sprintf("publication_names %s", publicationLiteral), } if s.streaming { pluginArgs = append(pluginArgs, "streaming 'on'") } - if err := pglogrepl.StartReplication(ctx, conn, s.slot, startLSN, + slotIdentifier, err := quoteReplicationSlotName(s.slot) + if err != nil { + s.abortFreshSlot(conn, slotCreated) + s.fail(ctx, status, err) + return + } + if err := pglogrepl.StartReplication(ctx, conn, slotIdentifier, startLSN, pglogrepl.StartReplicationOptions{PluginArgs: pluginArgs}); err != nil { + s.abortFreshSlot(conn, slotCreated) s.fail(ctx, status, err) return } - dec := newDecoder() + limits := decoderLimits{ + maxChanges: s.maxTransactionChanges, + maxBytes: s.maxTransactionBytes, + maxInflightChanges: s.maxInflightChanges, + maxInflightBytes: s.maxInflightBytes, + } + dec := newDecoder(limits) if s.streaming { - dec = newStreamingDecoder() + dec = newStreamingDecoder(limits) } var opLabels map[Op]metrics.Labels if mc != nil { opLabels = map[Op]metrics.Labels{ - OpInsert: {"slot": s.slot, "op": string(OpInsert)}, - OpUpdate: {"slot": s.slot, "op": string(OpUpdate)}, - OpDelete: {"slot": s.slot, "op": string(OpDelete)}, - OpTruncate: {"slot": s.slot, "op": string(OpTruncate)}, - OpSnapshot: {"slot": s.slot, "op": string(OpSnapshot)}, + OpInsert: {"source": s.name, "op": string(OpInsert)}, + OpUpdate: {"source": s.name, "op": string(OpUpdate)}, + OpDelete: {"source": s.name, "op": string(OpDelete)}, + OpTruncate: {"source": s.name, "op": string(OpTruncate)}, + OpSnapshot: {"source": s.name, "op": string(OpSnapshot)}, } } - clientPos := startLSN - lastSaved := pglogrepl.LSN(0) + // safePos is the furthest position that can be replayed without losing + // decoder state. It advances only after a complete transaction boundary; + // the server WAL end in a keepalive is never a safe checkpoint. + safePos := startLSN + lastSaved := startLSN + defer func() { + if safePos <= lastSaved { + return + } + flushCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := cp.Save(flushCtx, s.slot, safePos); err != nil { + s.log.Warn("failed to persist final cdc checkpoint", + zap.String("slot", s.slot), zap.String("lsn", safePos.String()), zap.Error(err)) + } + }() + saveSafe := func() error { + if safePos <= lastSaved { + return nil + } + if err := cp.Save(ctx, s.slot, safePos); err != nil { + return err + } + lastSaved = safePos + return nil + } now := time.Now() nextStandby := now.Add(s.standbyInterval) nextStatus := now.Add(s.statusInterval) @@ -303,15 +655,16 @@ func (s *Source) run( now = time.Now() if !now.Before(nextStandby) { - if clientPos > lastSaved { - if err := cp.Save(ctx, s.slot, clientPos); err != nil { - s.fail(ctx, status, err) - return - } - lastSaved = clientPos + if err := saveSafe(); err != nil { + s.fail(ctx, status, err) + return } if err := pglogrepl.SendStandbyStatusUpdate(ctx, conn, - pglogrepl.StandbyStatusUpdate{WALWritePosition: clientPos}); err != nil { + pglogrepl.StandbyStatusUpdate{ + WALWritePosition: safePos, + WALFlushPosition: safePos, + WALApplyPosition: safePos, + }); err != nil { s.fail(ctx, status, err) return } @@ -340,6 +693,10 @@ func (s *Source) run( if !ok { continue } + if len(cd.Data) == 0 { + s.fail(ctx, status, fmt.Errorf("%w: empty CopyData payload", ErrUnsupportedMessage)) + return + } switch cd.Data[0] { case pglogrepl.PrimaryKeepaliveMessageByteID: @@ -348,12 +705,21 @@ func (s *Source) run( s.fail(ctx, status, kaErr) return } - if ka.ServerWALEnd > clientPos { - clientPos = ka.ServerWALEnd - } + // ServerWALEnd is the receive watermark used by an in-flight + // snapshot handoff. It is deliberately independent from safePos: + // keepalives must not advance the transaction-safe checkpoint. + s.observeKeepalive(done, ka) if ka.ReplyRequested { + if err := saveSafe(); err != nil { + s.fail(ctx, status, err) + return + } if err := pglogrepl.SendStandbyStatusUpdate(ctx, conn, - pglogrepl.StandbyStatusUpdate{WALWritePosition: clientPos}); err != nil { + pglogrepl.StandbyStatusUpdate{ + WALWritePosition: safePos, + WALFlushPosition: safePos, + WALApplyPosition: safePos, + }); err != nil { s.fail(ctx, status, err) return } @@ -364,20 +730,26 @@ func (s *Source) run( s.fail(ctx, status, xErr) return } - changes, dErr := dec.decode(xld.WALData, xld.WALStart) + result, dErr := dec.decodeResult(xld.WALData, xld.WALStart) if dErr != nil { s.fail(ctx, status, dErr) return } - for i := range changes { - s.emitChange(ctx, changes[i]) + for i := range result.changes { + s.emitChange(ctx, result.changes[i]) if mc != nil { - mc.CounterInc(changesCounter, opLabels[changes[i].Op]) + mc.CounterInc(changesCounter, opLabels[result.changes[i].Op]) } } - if end := xld.WALStart + pglogrepl.LSN(len(xld.WALData)); end > clientPos { - clientPos = end + if result.safe { + if end := xld.WALStart + pglogrepl.LSN(len(xld.WALData)); end > safePos { + safePos = end + } } + s.advanceStreamPosition(done, xld.WALStart+pglogrepl.LSN(len(xld.WALData))) + default: + s.fail(ctx, status, fmt.Errorf("%w: copy data kind %q", ErrUnsupportedMessage, cd.Data[0])) + return } } } @@ -407,15 +779,28 @@ func (s *Source) reportLag(ctx context.Context, adminDB *sql.DB, mc metrics.Coll return } if mc != nil { - mc.GaugeSet(retainedWALGauge, float64(retained), metrics.Labels{"slot": s.slot}) + mc.GaugeSet(retainedWALGauge, float64(retained), metrics.Labels{"source": s.name}) } } func (s *Source) fail(_ context.Context, status chan any, err error) { + if err == nil { + err = ErrSourceClosed + } + s.mu.Lock() + s.sourceErr = err + if s.state == sourceRunning || s.state == sourceStarting { + s.state = sourceFailed + } + s.mu.Unlock() s.log.Error("cdc stream error", zap.String("slot", s.slot), zap.Error(err)) if s.coll != nil { - s.coll.CounterInc(errorsCounter, metrics.Labels{"slot": s.slot}) + s.coll.CounterInc(errorsCounter, metrics.Labels{"source": s.name}) + if errors.Is(err, ErrTransactionLimit) { + s.coll.CounterInc(transactionLimitCounter, metrics.Labels{"source": s.name}) + } } + s.closeSubscriptionsWithError(err) select { case status <- err: default: @@ -428,11 +813,11 @@ func (s *Source) prepareSlot( adminDB *sql.DB, cp Checkpointer, fallback pglogrepl.LSN, -) (pglogrepl.LSN, string, error) { +) (pglogrepl.LSN, bool, error) { var start pglogrepl.LSN resumed := false if cpLSN, ok, err := cp.Load(ctx, s.slot); err != nil { - return 0, "", err + return 0, false, err } else if ok { start = cpLSN resumed = true @@ -443,47 +828,78 @@ func (s *Source) prepareSlot( var err error exists, err = slotExists(ctx, adminDB, s.slot) if err != nil { - return 0, "", err + return 0, false, err + } + if !exists && resumed { + // A local offset is meaningful only for the server-side slot + // incarnation that produced it. If that slot disappeared, do not + // reuse the old LSN for a newly-created slot. + if err := cp.Delete(ctx, s.slot); err != nil { + return 0, false, fmt.Errorf("delete stale cdc checkpoint: %w", err) + } + start = 0 + resumed = false + } + if exists && !resumed { + // A persistent slot is the server-side durable cursor. Never fall + // back to the current system WAL position when local checkpoint + // state is missing; doing so can skip retained logical changes. + confirmed, valid, err := slotConfirmedFlush(ctx, adminDB, s.slot) + if err != nil { + return 0, false, err + } + if valid { + start = confirmed + } } + } else if resumed { + // Temporary slots are destroyed with their replication connection, so + // any persisted offset belongs to an older slot incarnation. + if err := cp.Delete(ctx, s.slot); err != nil { + return 0, false, fmt.Errorf("delete stale cdc checkpoint: %w", err) + } + start = 0 } - snapshotName := "" + slotCreated := false if !exists { - opts := pglogrepl.CreateReplicationSlotOptions{Temporary: s.temporary} - wantSnapshot := s.snapshot && !resumed - if wantSnapshot { - opts.SnapshotAction = "EXPORT_SNAPSHOT" + slotIdentifier, err := quoteReplicationSlotName(s.slot) + if err != nil { + return 0, false, err } - res, err := pglogrepl.CreateReplicationSlot(ctx, conn, s.slot, config.OutputPlugin, opts) + opts := pglogrepl.CreateReplicationSlotOptions{Temporary: s.temporary} + res, err := pglogrepl.CreateReplicationSlot(ctx, conn, slotIdentifier, config.OutputPlugin, opts) if err != nil { - return 0, "", fmt.Errorf("create replication slot: %w", err) + return 0, false, fmt.Errorf("create replication slot: %w", err) } + slotCreated = true cpoint, err := pglogrepl.ParseLSN(res.ConsistentPoint) if err != nil { - return 0, "", fmt.Errorf("parse consistent point %q: %w", res.ConsistentPoint, err) + return 0, slotCreated, fmt.Errorf("parse consistent point %q: %w", res.ConsistentPoint, err) } if cpoint > start { start = cpoint } - if wantSnapshot { - snapshotName = res.SnapshotName - } } if s.failover && !s.temporary { if err := s.setSlotFailover(ctx, conn); err != nil { - return 0, "", err + return 0, slotCreated, err } } if start == 0 { start = fallback } - return start, snapshotName, nil + return start, slotCreated, nil } func (s *Source) setSlotFailover(ctx context.Context, conn *pgconn.PgConn) error { - cmd := fmt.Sprintf("ALTER_REPLICATION_SLOT %s ( FAILOVER )", s.slot) + slotIdentifier, err := quoteReplicationSlotName(s.slot) + if err != nil { + return err + } + cmd := fmt.Sprintf("ALTER_REPLICATION_SLOT %s ( FAILOVER )", slotIdentifier) if err := conn.Exec(ctx, cmd).Close(); err != nil { return fmt.Errorf("set slot failover: %w", err) } @@ -500,7 +916,7 @@ func (t tableRef) quoted() string { return pq.QuoteIdentifier(t.schema) + "." + pq.QuoteIdentifier(t.name) } -func (s *Source) snapshotExisting(ctx context.Context, adminDB *sql.DB, publication, snapshotName string) error { +func (s *Source) snapshotWithSink(ctx context.Context, adminDB *sql.DB, publication, snapshotName string, sink snapshotSink) error { conn, err := adminDB.Conn(ctx) if err != nil { return fmt.Errorf("snapshot connection: %w", err) @@ -542,7 +958,7 @@ func (s *Source) snapshotExisting(ctx context.Context, adminDB *sql.DB, publicat total := 0 for _, tbl := range tables { - n, err := s.snapshotTable(ctx, conn, tbl) + n, err := s.snapshotTableWithSink(ctx, conn, tbl, sink) if err != nil { return err } @@ -558,7 +974,82 @@ func (s *Source) snapshotExisting(ctx context.Context, adminDB *sql.DB, publicat return nil } -func (s *Source) snapshotTable(ctx context.Context, conn *sql.Conn, tbl tableRef) (int, error) { +// snapshotCurrent establishes an exported logical-decoding snapshot for one +// subscriber. The temporary slot's consistent point is the exact WAL fence; +// unlike a SQL-only pg_current_wal_lsn query, it cannot race a concurrent +// commit between snapshot acquisition and fence capture. +func (s *Source) snapshotCurrentTo(ctx context.Context, sub *sourceSubscription) (pglogrepl.LSN, error) { + replConn, err := pgconn.Connect(ctx, s.replDSN) + if err != nil { + return 0, fmt.Errorf("snapshot replication connection: %w", err) + } + defer func() { _ = replConn.Close(context.Background()) }() + if _, err := pglogrepl.IdentifySystem(ctx, replConn); err != nil { + return 0, fmt.Errorf("identify snapshot replication system: %w", err) + } + + snapshotSlot := subscriberSnapshotSlot(s.slot, sub.id) + snapshotSlotID, err := quoteReplicationSlotName(snapshotSlot) + if err != nil { + return 0, err + } + result, err := pglogrepl.CreateReplicationSlot(ctx, replConn, snapshotSlotID, config.OutputPlugin, + pglogrepl.CreateReplicationSlotOptions{Temporary: true, SnapshotAction: "EXPORT_SNAPSHOT"}) + if err != nil { + return 0, fmt.Errorf("create subscriber snapshot slot: %w", err) + } + fence, err := pglogrepl.ParseLSN(result.ConsistentPoint) + if err != nil { + return 0, fmt.Errorf("parse subscriber snapshot fence %q: %w", result.ConsistentPoint, err) + } + if result.SnapshotName == "" { + return 0, errors.New("subscriber snapshot slot returned no exported snapshot") + } + if err := s.waitStreamPosition(ctx, fence); err != nil { + return 0, fmt.Errorf("wait for replication fence: %w", err) + } + + adminDB, err := sql.Open("postgres", s.adminDSN) + if err != nil { + return 0, fmt.Errorf("open snapshot connection: %w", err) + } + defer func() { _ = adminDB.Close() }() + adminDB.SetMaxOpenConns(1) + adminDB.SetMaxIdleConns(1) + if err := adminDB.PingContext(ctx); err != nil { + return 0, fmt.Errorf("ping snapshot connection: %w", err) + } + + err = s.snapshotWithSink(ctx, adminDB, s.publication, result.SnapshotName, func(rc RowChange) error { + if !sub.matchesSnapshot(config.Change{Table: rc.Table, Relation: rc.Relation()}) { + return nil + } + change := config.Change{ + Source: s.name, + Op: string(OpSnapshot), + Schema: rc.Schema, + Table: rc.Table, + Relation: rc.Relation(), + CommitLSN: fence.String(), + Before: rc.Before, + After: rc.After, + } + return sub.sendSnapshot(change, config.EstimateChangeBytes(change)) + }) + if err != nil { + return 0, fmt.Errorf("subscriber snapshot scan: %w", err) + } + return fence, nil +} + +func subscriberSnapshotSlot(slot string, id uint64) string { + digest := sha256.Sum256([]byte(slot)) + return fmt.Sprintf("wippy_snap_%x_%d", digest[:8], id) +} + +type snapshotSink func(RowChange) error + +func (s *Source) snapshotTableWithSink(ctx context.Context, conn *sql.Conn, tbl tableRef, sink snapshotSink) (int, error) { if _, err := conn.ExecContext(ctx, "DECLARE "+snapshotCursor+" NO SCROLL CURSOR FOR SELECT * FROM "+tbl.quoted()); err != nil { return 0, fmt.Errorf("declare cursor %s.%s: %w", tbl.schema, tbl.name, err) @@ -568,7 +1059,7 @@ func (s *Source) snapshotTable(ctx context.Context, conn *sql.Conn, tbl tableRef fetchSQL := fmt.Sprintf("FETCH %d FROM %s", s.snapshotFetchSize, snapshotCursor) n := 0 for { - got, err := s.fetchSnapshotBatch(ctx, conn, tbl, fetchSQL) + got, err := s.fetchSnapshotBatch(ctx, conn, tbl, fetchSQL, sink) if err != nil { return n, err } @@ -579,7 +1070,7 @@ func (s *Source) snapshotTable(ctx context.Context, conn *sql.Conn, tbl tableRef } } -func (s *Source) fetchSnapshotBatch(ctx context.Context, conn *sql.Conn, tbl tableRef, fetchSQL string) (int, error) { +func (s *Source) fetchSnapshotBatch(ctx context.Context, conn *sql.Conn, tbl tableRef, fetchSQL string, sink snapshotSink) (int, error) { rows, err := conn.QueryContext(ctx, fetchSQL) if err != nil { return 0, fmt.Errorf("fetch %s.%s: %w", tbl.schema, tbl.name, err) @@ -610,7 +1101,9 @@ func (s *Source) fetchSnapshotBatch(ctx context.Context, conn *sql.Conn, tbl tab after[c] = nil } } - s.emitChange(ctx, RowChange{Op: OpSnapshot, Schema: tbl.schema, Table: tbl.name, After: after}) + if err := sink(RowChange{Op: OpSnapshot, Schema: tbl.schema, Table: tbl.name, After: after}); err != nil { + return got, err + } got++ } return got, rows.Err() @@ -646,14 +1139,56 @@ func slotExists(ctx context.Context, adminDB *sql.DB, slot string) (bool, error) return n > 0, nil } +func slotConfirmedFlush(ctx context.Context, adminDB *sql.DB, slot string) (pglogrepl.LSN, bool, error) { + var raw sql.NullString + err := adminDB.QueryRowContext(ctx, + `SELECT confirmed_flush_lsn::text + FROM pg_replication_slots + WHERE slot_name = $1`, slot).Scan(&raw) + if err != nil { + return 0, false, fmt.Errorf("read slot confirmed flush position: %w", err) + } + if !raw.Valid || raw.String == "" { + return 0, false, nil + } + lsn, err := pglogrepl.ParseLSN(raw.String) + if err != nil { + return 0, false, fmt.Errorf("parse slot confirmed flush position %q: %w", raw.String, err) + } + return lsn, true, nil +} + func (s *Source) ensurePublication(ctx context.Context, adminDB *sql.DB) (string, error) { if s.publication != "" { + if err := validatePostgresIdentifier(s.publication, "publication"); err != nil { + return "", err + } return s.publication, nil } if len(s.tables) == 0 { return "", ErrNoPublication } name := s.slot + "_pub" + quotedName, err := quotePostgresIdentifier(name, "publication") + if err != nil { + return "", err + } + quotedTables := make([]string, 0, len(s.tables)) + seenTables := make(map[string]struct{}, len(s.tables)) + for _, table := range s.tables { + quotedTable, err := quoteQualifiedIdent(table) + if err != nil { + return "", err + } + if _, exists := seenTables[quotedTable]; exists { + continue + } + seenTables[quotedTable] = struct{}{} + quotedTables = append(quotedTables, quotedTable) + } + if len(quotedTables) == 0 { + return "", ErrNoPublication + } var n int if err := adminDB.QueryRowContext(ctx, @@ -661,33 +1196,51 @@ func (s *Source) ensurePublication(ctx context.Context, adminDB *sql.DB) (string return "", fmt.Errorf("check publication: %w", err) } if n == 0 { - quoted := make([]string, len(s.tables)) - for i, t := range s.tables { - quoted[i] = quoteQualifiedIdent(t) - } stmt := fmt.Sprintf("CREATE PUBLICATION %s FOR TABLE %s", - pq.QuoteIdentifier(name), strings.Join(quoted, ", ")) + quotedName, strings.Join(quotedTables, ", ")) if _, err := adminDB.ExecContext(ctx, stmt); err != nil { return "", fmt.Errorf("create publication: %w", err) } + } else { + // The generated name is owned by this source configuration. Reconcile + // its membership exactly on every start so an update cannot silently + // continue publishing an old table set. User-supplied publications take + // the early return above and are never altered or dropped. + stmt := fmt.Sprintf("ALTER PUBLICATION %s SET TABLE %s", + quotedName, strings.Join(quotedTables, ", ")) + if _, err := adminDB.ExecContext(ctx, stmt); err != nil { + return "", fmt.Errorf("reconcile publication: %w", err) + } } return name, nil } -func (s *Source) abortFreshSnapshot(conn *pgconn.PgConn) { - _ = conn.Close(context.Background()) - if s.temporary { +func (s *Source) abortFreshSlot(conn *pgconn.PgConn, created bool) { + if conn != nil { + _ = conn.Close(context.Background()) + } + if !created { return } + s.cleanupFreshSlot() +} + +func (s *Source) cleanupFreshSlot() { cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := s.dropSlotAndCheckpoint(cleanupCtx); err != nil { - s.log.Warn("cdc cleanup after snapshot failure failed", + s.log.Warn("cdc cleanup after fresh slot failure failed", zap.String("slot", s.slot), zap.Error(err)) } } func (s *Source) dropSlotAndCheckpoint(ctx context.Context) error { + s.dropMu.Lock() + defer s.dropMu.Unlock() + if s.dropDone.Load() { + return nil + } + adminDB, err := sql.Open("postgres", s.adminDSN) if err != nil { return fmt.Errorf("open admin connection for slot drop: %w", err) @@ -704,11 +1257,13 @@ func (s *Source) dropSlotAndCheckpoint(ctx context.Context) error { if err := s.injectedCP.Delete(ctx, s.slot); err != nil { return fmt.Errorf("delete checkpoint: %w", err) } + s.dropDone.Store(true) return nil } if _, err := adminDB.ExecContext(ctx, `DELETE FROM wippy_cdc_offsets WHERE slot = $1`, s.slot); err != nil { return fmt.Errorf("delete checkpoint: %w", err) } + s.dropDone.Store(true) return nil } @@ -722,6 +1277,11 @@ func dropReplicationSlot(ctx context.Context, adminDB *sql.DB, slot string) erro lastErr = err var pqErr *pq.Error + if errors.As(err, &pqErr) && string(pqErr.Code) == "42704" { + // Delete is intentionally idempotent. A source can have already + // dropped its slot during Stop before the manager retries Dispose. + return nil + } if !errors.As(err, &pqErr) || string(pqErr.Code) != slotActiveSQLState { return err } @@ -734,10 +1294,18 @@ func dropReplicationSlot(ctx context.Context, adminDB *sql.DB, slot string) erro return lastErr } -func quoteQualifiedIdent(name string) string { +func quoteQualifiedIdent(name string) (string, error) { parts := strings.Split(name, ".") + if len(parts) < 1 || len(parts) > 2 { + return "", fmt.Errorf("%w: table", ErrInvalidIdentifier) + } + quoted := make([]string, len(parts)) for i, p := range parts { - parts[i] = pq.QuoteIdentifier(p) + quotedPart, err := quotePostgresIdentifier(p, "table") + if err != nil { + return "", err + } + quoted[i] = quotedPart } - return strings.Join(parts, ".") + return strings.Join(quoted, "."), nil } diff --git a/service/cdc/postgres/service_metrics_test.go b/service/cdc/postgres/service_metrics_test.go index 31a1cff31..447604978 100644 --- a/service/cdc/postgres/service_metrics_test.go +++ b/service/cdc/postgres/service_metrics_test.go @@ -15,12 +15,12 @@ import ( func TestSource_FailEmitsErrorCounter(t *testing.T) { rec := telemetrytest.NewRecorder() - s := &Source{log: zap.NewNop(), slot: "test_slot", coll: rec} + s := &Source{log: zap.NewNop(), name: "test:source", slot: "test_slot", coll: rec} status := make(chan any, 1) s.fail(context.Background(), status, errors.New("boom")) - assert.Equal(t, 1.0, rec.CounterValue(errorsCounter, metrics.Labels{"slot": "test_slot"})) + assert.Equal(t, 1.0, rec.CounterValue(errorsCounter, metrics.Labels{"source": "test:source"})) } func TestSource_FailNilCollector(t *testing.T) { @@ -28,3 +28,13 @@ func TestSource_FailNilCollector(t *testing.T) { status := make(chan any, 1) s.fail(context.Background(), status, errors.New("boom")) } + +func TestSource_FailEmitsTransactionLimitCounter(t *testing.T) { + rec := telemetrytest.NewRecorder() + s := &Source{log: zap.NewNop(), name: "test:source", coll: rec} + + status := make(chan any, 1) + s.fail(context.Background(), status, ErrTransactionLimit) + + assert.Equal(t, 1.0, rec.CounterValue(transactionLimitCounter, metrics.Labels{"source": "test:source"})) +} diff --git a/service/cdc/postgres/service_test.go b/service/cdc/postgres/service_test.go index 8fe312044..760dbaa73 100644 --- a/service/cdc/postgres/service_test.go +++ b/service/cdc/postgres/service_test.go @@ -16,18 +16,30 @@ func TestNewSourceDefaults(t *testing.T) { assert.Equal(t, defaultStandbyInterval, s.standbyInterval) assert.Equal(t, defaultStatusInterval, s.statusInterval) assert.Equal(t, defaultSnapshotFetchSize, s.snapshotFetchSize) + assert.Equal(t, defaultMaxTransactionChanges, s.maxTransactionChanges) + assert.Equal(t, int64(defaultMaxTransactionBytes), s.maxTransactionBytes) + assert.Equal(t, defaultMaxInflightChanges, s.maxInflightChanges) + assert.Equal(t, int64(defaultMaxInflightBytes), s.maxInflightBytes) assert.NotNil(t, s.log) } func TestNewSourceHonorsOverrides(t *testing.T) { s := NewSource(SourceOptions{ - StandbyInterval: 1 * time.Second, - StatusInterval: 2 * time.Second, - SnapshotFetchSize: 4096, + StandbyInterval: 1 * time.Second, + StatusInterval: 2 * time.Second, + SnapshotFetchSize: 4096, + MaxTransactionChanges: 123, + MaxTransactionBytes: 456, + MaxInflightChanges: 789, + MaxInflightBytes: 101112, }) assert.Equal(t, 1*time.Second, s.standbyInterval) assert.Equal(t, 2*time.Second, s.statusInterval) assert.Equal(t, 4096, s.snapshotFetchSize) + assert.Equal(t, 123, s.maxTransactionChanges) + assert.Equal(t, int64(456), s.maxTransactionBytes) + assert.Equal(t, 789, s.maxInflightChanges) + assert.Equal(t, int64(101112), s.maxInflightBytes) } func TestStopBeforeStartIsSafe(t *testing.T) { @@ -37,3 +49,85 @@ func TestStopBeforeStartIsSafe(t *testing.T) { require.NoError(t, s.Stop(ctx)) require.NoError(t, s.Stop(ctx)) } + +func TestFailedSourceCanBeStoppedAndRetried(t *testing.T) { + s := NewSource(SourceOptions{}) + s.mu.Lock() + s.state = sourceFailed + s.mu.Unlock() + + require.NoError(t, s.Stop(context.Background())) + + // A canceled start fails during setup, but it must not be rejected as a + // permanently closed source. Supervisors use this path after a fault. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := s.Start(ctx) + require.Error(t, err) + assert.NotErrorIs(t, err, ErrSourceClosed) +} + +func TestFailedSourceStopWaitsForRunCleanup(t *testing.T) { + source := NewSource(SourceOptions{}) + runDone := make(chan struct{}) + cancelCalled := make(chan struct{}) + releaseRun := make(chan struct{}) + source.mu.Lock() + source.state = sourceFailed + source.done = runDone + source.cancel = func() { close(cancelCalled) } + source.mu.Unlock() + go func() { + <-cancelCalled + <-releaseRun + close(runDone) + }() + + stopDone := make(chan error, 1) + go func() { stopDone <- source.Stop(context.Background()) }() + select { + case err := <-stopDone: + t.Fatalf("Stop returned before failed run cleanup: %v", err) + case <-cancelCalled: + } + close(releaseRun) + require.NoError(t, <-stopDone) + + source.mu.Lock() + assert.Equal(t, sourceStopped, source.state) + source.mu.Unlock() +} + +func TestFailedSourceStopCancellationIsRetryableAndIsolated(t *testing.T) { + first := NewSource(SourceOptions{Name: "db-one"}) + firstDone := make(chan struct{}) + first.mu.Lock() + first.state = sourceFailed + first.done = firstDone + first.mu.Unlock() + + second := NewSource(SourceOptions{Name: "db-two"}) + second.mu.Lock() + second.state = sourceFailed + second.mu.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + assert.ErrorIs(t, first.Stop(ctx), context.DeadlineExceeded) + first.mu.Lock() + assert.Equal(t, sourceStopping, first.state) + first.mu.Unlock() + + // A blocked source must not hold lifecycle state for an independent + // database/source instance. + require.NoError(t, second.Stop(context.Background())) + close(firstDone) + require.NoError(t, first.Stop(context.Background())) +} + +func TestClosePermanentlyRetiresSource(t *testing.T) { + s := NewSource(SourceOptions{}) + require.NoError(t, s.Close(context.Background())) + _, err := s.Start(context.Background()) + require.ErrorIs(t, err, ErrSourceClosed) +} diff --git a/service/cdc/postgres/stream.go b/service/cdc/postgres/stream.go index 4f5a36a60..4dc938812 100644 --- a/service/cdc/postgres/stream.go +++ b/service/cdc/postgres/stream.go @@ -4,10 +4,11 @@ package postgres import ( "context" + "errors" "strings" "sync" - "sync/atomic" + "github.com/jackc/pglogrepl" config "github.com/wippyai/runtime/api/service/cdc" ) @@ -16,19 +17,91 @@ const ( maxStreamBuffer = 65536 ) +// errSubscriberOverflow is terminal for one subscription only. A consumer +// that cannot keep up must not back-pressure the replication receive loop or +// unrelated subscribers. +var errSubscriberOverflow = errors.New("postgres cdc subscriber backlog overflow") +var errSnapshotNotActive = errors.New("postgres cdc snapshot is no longer active") + type sourceSubscription struct { - source *Source - in chan config.Change - out chan config.Change - done chan struct{} - tables map[string]struct{} - ops map[string]struct{} - id uint64 - once sync.Once - closed atomic.Bool -} - -func (s *Source) Subscribe(opts config.StreamOptions) config.ChangeStream { + err error + tables map[string]struct{} + source *Source + done chan struct{} + notify chan struct{} + relayDone chan struct{} + snapshotDone chan struct{} + snapshotCancel context.CancelFunc + ops map[string]struct{} + out chan config.Change + queue []queuedChange + pending []queuedChange + maxBytes int64 + maxChanges int + id uint64 + queuedBytes int64 + mu sync.Mutex + closed bool + snapshotting bool +} + +type queuedChange struct { + change config.Change + bytes int64 +} + +func (s *Source) Subscribe(opts config.StreamOptions) config.Stream { + if err := opts.Validate(); err != nil { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + if s.state != sourceNew && s.state != sourceStarting && s.state != sourceRunning { + return nil + } + effectiveSnapshot := opts.Snapshot || s.snapshot + if effectiveSnapshot && s.state != sourceRunning { + return nil + } + opts.Snapshot = effectiveSnapshot + sub := s.newSubscription(opts) + if effectiveSnapshot { + s.startSnapshot(context.Background(), sub) + } + return sub +} + +// subscribe is the driver-facing subscription path. It holds the source +// lifecycle lock while registering the child, so Stop/fault cannot transition +// the source and close its current subscriptions between the state check and +// registration. +func (s *Source) subscribe(ctx context.Context, opts config.StreamOptions) (config.Stream, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + + s.mu.Lock() + defer s.mu.Unlock() + if (s.state != sourceNew && s.state != sourceStarting && s.state != sourceRunning) || + s.permanentlyClosed || s.sourceErr != nil { + return nil, config.ErrSourceNotReady + } + effectiveSnapshot := opts.Snapshot || s.snapshot + if effectiveSnapshot && s.state != sourceRunning { + return nil, config.ErrSourceNotReady + } + opts.Snapshot = effectiveSnapshot + sub := s.newSubscription(opts) + if effectiveSnapshot { + s.startSnapshot(ctx, sub) + } + return sub, nil +} + +func (s *Source) newSubscription(opts config.StreamOptions) *sourceSubscription { buffer := opts.Buffer if buffer <= 0 { buffer = defaultStreamBuffer @@ -42,11 +115,18 @@ func (s *Source) Subscribe(opts config.StreamOptions) config.ChangeStream { sub := &sourceSubscription{ source: s, id: s.nextSubID, - in: make(chan config.Change, buffer), - out: make(chan config.Change, buffer), - done: make(chan struct{}), - tables: filterSet(opts.Tables), - ops: filterSet(opts.Ops), + // queue is the sole driver-owned backlog. out is an unbuffered + // delivery handoff, so bytes are released exactly after a consumer + // receives the change rather than when it is merely enqueued. + out: make(chan config.Change), + done: make(chan struct{}), + notify: make(chan struct{}, 1), + maxChanges: buffer, + maxBytes: opts.EffectiveMaxBytes(), + snapshotting: opts.Snapshot, + relayDone: make(chan struct{}), + tables: filterSet(opts.Tables), + ops: filterSet(opts.Ops), } s.subs[sub.id] = sub s.subMu.Unlock() @@ -65,8 +145,14 @@ func (s *Source) publishChange(ctx context.Context, change config.Change) { } s.subMu.RUnlock() + if len(subs) == 0 { + return + } + // Estimate the retained size once for this source event. Fan-out must not + // repeat a recursive walk for every subscriber. + bytes := config.EstimateChangeBytes(change) for _, sub := range subs { - sub.send(ctx, change) + sub.send(ctx, change, bytes) } } @@ -77,6 +163,30 @@ func (s *Source) removeSubscription(id uint64) { } func (s *Source) closeSubscriptions() { + s.closeSubscriptionsWithError(nil) +} + +func (s *Source) closeSubscriptionsWithError(err error) { + subs := s.detachSubscriptions() + s.closeDetachedSubscriptions(subs, err) +} + +// detachSubscriptionsLocked must be called while s.mu is held. Subscribe +// takes s.mu before subMu, so this ordering makes generation cleanup atomic +// with the lifecycle transition and prevents an old run from taking a new +// generation's subscribers. +func (s *Source) detachSubscriptionsLocked() []*sourceSubscription { + s.subMu.Lock() + subs := make([]*sourceSubscription, 0, len(s.subs)) + for id, sub := range s.subs { + subs = append(subs, sub) + delete(s.subs, id) + } + s.subMu.Unlock() + return subs +} + +func (s *Source) detachSubscriptions() []*sourceSubscription { s.subMu.Lock() subs := make([]*sourceSubscription, 0, len(s.subs)) for id, sub := range s.subs { @@ -84,9 +194,18 @@ func (s *Source) closeSubscriptions() { delete(s.subs, id) } s.subMu.Unlock() + return subs +} +func (s *Source) closeDetachedSubscriptions(subs []*sourceSubscription, err error) { + for _, sub := range subs { + sub.closeWithError(err) + } for _, sub := range subs { - sub.Close() + sub.waitSnapshot() + } + for _, sub := range subs { + sub.waitRelay() } } @@ -95,47 +214,216 @@ func (s *sourceSubscription) Changes() <-chan config.Change { } func (s *sourceSubscription) Close() { - s.once.Do(func() { - s.closed.Store(true) - if s.source != nil { - s.source.removeSubscription(s.id) - } - close(s.done) - }) + s.closeWithError(nil) + s.waitSnapshot() + s.waitRelay() +} + +func (s *sourceSubscription) Err() error { + s.mu.Lock() + err := s.err + s.mu.Unlock() + return err +} + +func (s *sourceSubscription) closeWithError(err error) { + s.mu.Lock() + parent, id := s.closeLocked(err) + cancel := s.snapshotCancel + s.mu.Unlock() + if cancel != nil { + cancel() + } + if parent != nil { + parent.removeSubscription(id) + } +} + +func (s *sourceSubscription) registerSnapshot(cancel context.CancelFunc, done chan struct{}) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return false + } + s.snapshotCancel = cancel + s.snapshotDone = done + return true +} + +func (s *sourceSubscription) finishSnapshotWorker() { + s.mu.Lock() + done := s.snapshotDone + s.snapshotDone = nil + s.snapshotCancel = nil + s.mu.Unlock() + if done != nil { + close(done) + } +} + +func (s *sourceSubscription) waitSnapshot() { + s.mu.Lock() + done := s.snapshotDone + s.mu.Unlock() + if done != nil { + <-done + } +} + +func (s *sourceSubscription) waitRelay() { + <-s.relayDone +} + +func (s *sourceSubscription) closeLocked(err error) (*Source, uint64) { + if s.closed { + return nil, 0 + } + s.closed = true + s.err = err + s.queue = nil + s.pending = nil + s.queuedBytes = 0 + close(s.done) + return s.source, s.id } func (s *sourceSubscription) run() { + defer close(s.relayDone) defer close(s.out) for { - select { - case <-s.done: - return - default: - } - select { - case change := <-s.in: - select { - case <-s.done: + s.mu.Lock() + if len(s.queue) == 0 { + if s.closed { + s.mu.Unlock() return - case s.out <- change: } - case <-s.done: + notify := s.notify + done := s.done + s.mu.Unlock() + select { + case <-notify: + case <-done: + } + continue + } + item := s.queue[0] + done := s.done + s.mu.Unlock() + + select { + case <-done: return + case s.out <- item.change: + s.mu.Lock() + if len(s.queue) > 0 { + s.queuedBytes -= s.queue[0].bytes + s.queue[0] = queuedChange{} + s.queue = s.queue[1:] + } + s.mu.Unlock() + } + } +} + +func (s *sourceSubscription) send(_ context.Context, change config.Change, bytes int64) { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + if len(s.queue)+len(s.pending) >= s.maxChanges || bytes > s.maxBytes-s.queuedBytes { + parent, id := s.closeLocked(errSubscriberOverflow) + s.mu.Unlock() + if parent != nil { + parent.removeSubscription(id) } + return + } + item := queuedChange{change: change, bytes: bytes} + if s.snapshotting { + s.pending = append(s.pending, item) + } else { + s.queue = append(s.queue, item) + } + s.queuedBytes += bytes + s.mu.Unlock() + select { + case s.notify <- struct{}{}: + default: } } -func (s *sourceSubscription) send(ctx context.Context, change config.Change) { - if s.closed.Load() { +func (s *sourceSubscription) sendSnapshot(change config.Change, bytes int64) error { + s.mu.Lock() + if s.closed { + err := s.err + if err == nil { + err = context.Canceled + } + s.mu.Unlock() + return err + } + if !s.snapshotting { + s.mu.Unlock() + return errSnapshotNotActive + } + if len(s.queue)+len(s.pending) >= s.maxChanges || bytes > s.maxBytes-s.queuedBytes { + parent, id := s.closeLocked(errSubscriberOverflow) + s.mu.Unlock() + if parent != nil { + parent.removeSubscription(id) + } + return errSubscriberOverflow + } + s.queue = append(s.queue, queuedChange{change: change, bytes: bytes}) + s.queuedBytes += bytes + s.mu.Unlock() + select { + case s.notify <- struct{}{}: + default: + } + return nil +} + +func (s *sourceSubscription) finishSnapshot(fence pglogrepl.LSN, err error) { + if err != nil { + s.closeWithError(err) + return + } + s.mu.Lock() + if s.closed || !s.snapshotting { + s.mu.Unlock() return } + for _, item := range s.pending { + if !changeAfterSnapshotFence(item.change, fence) { + s.queuedBytes -= item.bytes + continue + } + s.queue = append(s.queue, item) + } + s.pending = nil + s.snapshotting = false + s.mu.Unlock() select { - case s.in <- change: - case <-s.done: - case <-ctx.Done(): + case s.notify <- struct{}{}: + default: } } +func changeAfterSnapshotFence(change config.Change, fence pglogrepl.LSN) bool { + if change.CommitLSN == "" { + return true + } + commit, err := pglogrepl.ParseLSN(change.CommitLSN) + if err != nil { + // A malformed cursor cannot be proven to be represented by the + // snapshot. Retain it rather than silently dropping a change. + return true + } + return commit > fence +} + func (s *sourceSubscription) matches(change config.Change) bool { if len(s.ops) > 0 { if _, ok := s.ops[strings.ToLower(change.Op)]; !ok { @@ -154,6 +442,17 @@ func (s *sourceSubscription) matches(change config.Change) bool { return true } +func (s *sourceSubscription) matchesSnapshot(change config.Change) bool { + if len(s.tables) == 0 { + return true + } + if _, ok := s.tables[strings.ToLower(change.Relation)]; ok { + return true + } + _, ok := s.tables[strings.ToLower(change.Table)] + return ok +} + func filterSet(values []string) map[string]struct{} { if len(values) == 0 { return nil diff --git a/service/cdc/postgres/stream_test.go b/service/cdc/postgres/stream_test.go index 2342d833f..5f512e8d9 100644 --- a/service/cdc/postgres/stream_test.go +++ b/service/cdc/postgres/stream_test.go @@ -4,9 +4,11 @@ package postgres import ( "context" + "errors" "testing" "time" + "github.com/jackc/pglogrepl" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" cdcapi "github.com/wippyai/runtime/api/service/cdc" @@ -40,6 +42,228 @@ func TestSourceSubscribePublishesMatchingChanges(t *testing.T) { } } +func TestSourceSubscribeAllowsOrdinaryPreStartStream(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + stream, err := src.subscribe(context.Background(), cdcapi.StreamOptions{Buffer: 1}) + require.NoError(t, err) + require.NotNil(t, stream) + defer stream.Close() + + src.publishChange(context.Background(), cdcapi.Change{ + Op: "insert", + Table: "accounts", + After: map[string]any{"id": int64(1)}, + Source: "test:cdc", + }) + + select { + case got := <-stream.Changes(): + require.Equal(t, "insert", got.Op) + require.Equal(t, "accounts", got.Table) + case <-time.After(time.Second): + t.Fatal("pre-start subscription did not retain the ordinary stream") + } +} + +func TestSourceSnapshotDefaultRequiresRunningGeneration(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a", Snapshot: true}) + stream, err := src.subscribe(context.Background(), cdcapi.StreamOptions{}) + assert.ErrorIs(t, err, cdcapi.ErrSourceNotReady) + assert.Nil(t, stream) + assert.Nil(t, src.Subscribe(cdcapi.StreamOptions{})) +} + +func TestSnapshotHandoffWaitsForReplicationFence(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + done := make(chan struct{}) + src.mu.Lock() + src.done = done + src.streamPosition = 0 + src.mu.Unlock() + + fence, err := pglogrepl.ParseLSN("0/20") + require.NoError(t, err) + waited := make(chan error, 1) + go func() { waited <- src.waitStreamPosition(context.Background(), fence) }() + select { + case err := <-waited: + t.Fatalf("handoff released before replication reached fence: %v", err) + case <-time.After(20 * time.Millisecond): + } + + src.advanceStreamPosition(done, fence) + select { + case err := <-waited: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("handoff did not observe the replication fence") + } +} + +func TestIdleKeepaliveAdvancesSnapshotWatermark(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + done := make(chan struct{}) + src.mu.Lock() + src.done = done + src.streamPosition = 0 + src.mu.Unlock() + + fence, err := pglogrepl.ParseLSN("0/40") + require.NoError(t, err) + waited := make(chan error, 1) + go func() { waited <- src.waitStreamPosition(context.Background(), fence) }() + select { + case err := <-waited: + t.Fatalf("idle snapshot released before keepalive: %v", err) + case <-time.After(20 * time.Millisecond): + } + + src.observeKeepalive(done, pglogrepl.PrimaryKeepaliveMessage{ServerWALEnd: fence}) + select { + case err := <-waited: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("idle keepalive did not advance snapshot watermark") + } +} + +func TestSnapshotGateSerializesPerSourceAndIsolatesSources(t *testing.T) { + first := NewSource(SourceOptions{Name: "db-one", Slot: "slot_one"}) + second := NewSource(SourceOptions{Name: "db-two", Slot: "slot_two"}) + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, first.acquireSnapshot(ctx)) + + waiting := make(chan error, 1) + go func() { waiting <- first.acquireSnapshot(ctx) }() + select { + case err := <-waiting: + t.Fatalf("same-source snapshot gate was not serialized: %v", err) + case <-time.After(20 * time.Millisecond): + } + require.NoError(t, second.acquireSnapshot(context.Background())) + second.releaseSnapshot() + cancel() + select { + case err := <-waiting: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("cancelled snapshot did not leave the per-source gate") + } + first.releaseSnapshot() +} + +func TestSubscriptionCloseWaitsForSnapshotWorker(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + sub := src.newSubscription(cdcapi.StreamOptions{Snapshot: true}) + cancelled := make(chan struct{}) + workerDone := make(chan struct{}) + require.True(t, sub.registerSnapshot(func() { close(cancelled) }, workerDone)) + go func() { + <-cancelled + sub.finishSnapshotWorker() + }() + + sub.Close() + select { + case <-workerDone: + default: + t.Fatal("subscription Close returned before its snapshot worker joined") + } +} + +func TestStopJoinsSnapshotWorkerBeforeGenerationReset(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + sub := src.newSubscription(cdcapi.StreamOptions{Snapshot: true}) + cancelled := make(chan struct{}) + release := make(chan struct{}) + workerDone := make(chan struct{}) + require.True(t, sub.registerSnapshot(func() { close(cancelled) }, workerDone)) + src.snapshotWG.Add(1) + go func() { + <-cancelled + <-release + sub.finishSnapshotWorker() + src.snapshotWG.Done() + }() + + stopped := make(chan error, 1) + go func() { stopped <- src.Stop(context.Background()) }() + select { + case err := <-stopped: + t.Fatalf("Stop returned before snapshot worker joined: %v", err) + case <-time.After(20 * time.Millisecond): + } + src.mu.Lock() + assert.Equal(t, sourceStopping, src.state) + src.mu.Unlock() + close(release) + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("Stop did not complete after snapshot worker release") + } + src.mu.Lock() + assert.Equal(t, sourceStopped, src.state) + src.mu.Unlock() +} + +func TestOldGenerationCleanupDoesNotCloseReplacementSubscribers(t *testing.T) { + src := NewSource(SourceOptions{Name: "db-one", Slot: "slot_one"}) + oldDone := make(chan struct{}) + src.mu.Lock() + src.state = sourceRunning + src.done = oldDone + src.mu.Unlock() + oldSub := src.Subscribe(cdcapi.StreamOptions{Buffer: 2}) + require.NotNil(t, oldSub) + + current, detached := src.finishRunGeneration(oldDone) + require.True(t, current) + src.closeDetachedSubscriptions(detached, nil) + close(oldDone) + + newDone := make(chan struct{}) + src.mu.Lock() + src.state = sourceRunning + src.done = newDone + src.mu.Unlock() + newSub := src.Subscribe(cdcapi.StreamOptions{Buffer: 2}) + require.NotNil(t, newSub) + defer newSub.Close() + + select { + case _, ok := <-oldSub.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("old generation subscriber was not closed") + } + src.publishChange(context.Background(), cdcapi.Change{Op: "insert", Table: "users"}) + select { + case change := <-newSub.Changes(): + require.Equal(t, "insert", change.Op) + case <-time.After(time.Second): + t.Fatal("replacement subscriber was closed by old generation cleanup") + } + + other := NewSource(SourceOptions{Name: "db-two", Slot: "slot_two"}) + otherDone := make(chan struct{}) + other.mu.Lock() + other.state = sourceRunning + other.done = otherDone + other.mu.Unlock() + otherSub := other.Subscribe(cdcapi.StreamOptions{Buffer: 2}) + require.NotNil(t, otherSub) + defer otherSub.Close() + other.publishChange(context.Background(), cdcapi.Change{Op: "insert", Table: "isolated"}) + select { + case change := <-otherSub.Changes(): + require.Equal(t, "isolated", change.Table) + case <-time.After(time.Second): + t.Fatal("independent source subscriber was affected by generation cleanup") + } +} + func TestSourceSubscribeFiltersChanges(t *testing.T) { src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) stream := src.Subscribe(cdcapi.StreamOptions{ @@ -76,7 +300,261 @@ func TestSourceSubscriptionCloseReleasesChannel(t *testing.T) { select { case _, ok := <-stream.Changes(): assert.False(t, ok) + default: + t.Fatal("stream channel was not closed synchronously") + } + assert.NoError(t, stream.Err()) +} + +func TestSourceSubscriptionRetainsTerminalError(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + stream := src.Subscribe(cdcapi.StreamOptions{}) + err := errors.New("replication failed") + src.closeSubscriptionsWithError(err) + + select { + case _, ok := <-stream.Changes(): + assert.False(t, ok) + default: + t.Fatal("stream channel was not closed synchronously") + } + assert.ErrorIs(t, stream.(interface{ Err() error }).Err(), err) +} + +func TestSourceSubscriptionChurnDetachesImmediately(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + for i := 0; i < 1000; i++ { + stream := src.Subscribe(cdcapi.StreamOptions{Buffer: 1}) + stream.Close() + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + default: + t.Fatal("churned stream channel was not closed") + } + } + src.subMu.RLock() + defer src.subMu.RUnlock() + assert.Empty(t, src.subs) +} + +func TestSourceSubscriptionIsPrunedWhenStopWins(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + src.mu.Lock() + src.state = sourceRunning + src.mu.Unlock() + + stream, err := src.subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + require.NotNil(t, stream) + + require.NoError(t, src.Stop(context.Background())) + assert.Eventually(t, func() bool { + src.subMu.RLock() + defer src.subMu.RUnlock() + return len(src.subs) == 0 + }, time.Second, time.Millisecond) + + select { + case _, ok := <-stream.Changes(): + assert.False(t, ok) + case <-time.After(time.Second): + t.Fatal("stopped source left a live subscription") + } + _, err = src.subscribe(context.Background(), cdcapi.StreamOptions{}) + assert.ErrorIs(t, err, cdcapi.ErrSourceNotReady) +} + +func TestSourceSubscriptionOverflowIsBoundedAndLocal(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + laggard := src.Subscribe(cdcapi.StreamOptions{Buffer: 1}) + reader := src.Subscribe(cdcapi.StreamOptions{Buffer: maxStreamBuffer}) + + done := make(chan struct{}) + go func() { + for i := 0; i < 1000; i++ { + src.publishChange(context.Background(), cdcapi.Change{ + Op: "insert", + Table: "accounts", + Relation: "public.accounts", + }) + } + close(done) + }() + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("replication fan-out blocked on a slow subscriber") + } + + assert.Eventually(t, func() bool { + return errors.Is(laggard.(interface{ Err() error }).Err(), errSubscriberOverflow) + }, time.Second, time.Millisecond, "laggard must terminate with an overflow error") + select { + case _, ok := <-reader.Changes(): + assert.True(t, ok, "an unrelated subscriber must remain active") + case <-time.After(time.Second): + t.Fatal("unrelated subscriber did not receive a change") + } + laggard.Close() + reader.Close() +} + +func TestSourceSubscriptionMaxBytesReleasesOnlyAfterDelivery(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + change := cdcapi.Change{ + Op: "insert", + Table: "users", + After: map[string]any{"payload": []byte("payload")}, + } + changeBytes := cdcapi.EstimateChangeBytes(change) + stream := src.newSubscription(cdcapi.StreamOptions{Buffer: 2, MaxBytes: changeBytes + 1}) + sub := stream + defer sub.Close() + + sub.send(context.Background(), change, cdcapi.EstimateChangeBytes(change)) + assert.Eventually(t, func() bool { + sub.mu.Lock() + defer sub.mu.Unlock() + return len(sub.queue) == 1 && sub.queuedBytes == changeBytes + }, time.Second, time.Millisecond) + + select { + case got := <-sub.Changes(): + assert.Equal(t, change.Table, got.Table) + case <-time.After(time.Second): + t.Fatal("timed out receiving queued change") + } + assert.Eventually(t, func() bool { + sub.mu.Lock() + defer sub.mu.Unlock() + return len(sub.queue) == 0 && sub.queuedBytes == 0 + }, time.Second, time.Millisecond) + + sub.send(context.Background(), change, cdcapi.EstimateChangeBytes(change)) + assert.NotErrorIs(t, sub.Err(), errSubscriberOverflow) + select { + case <-sub.Changes(): + case <-time.After(time.Second): + t.Fatal("released byte budget did not accept the next change") + } +} + +func TestSourceSubscriptionMaxBytesOverflowIsIsolated(t *testing.T) { + change := cdcapi.Change{ + Op: "insert", + Table: "users", + After: map[string]any{"payload": []byte("payload")}, + } + limit := cdcapi.EstimateChangeBytes(change) - 1 + first := NewSource(SourceOptions{Name: "test:first", Slot: "slot_first"}) + second := NewSource(SourceOptions{Name: "test:second", Slot: "slot_second"}) + firstStream := first.Subscribe(cdcapi.StreamOptions{MaxBytes: limit}) + secondStream := second.Subscribe(cdcapi.StreamOptions{MaxBytes: limit + 1}) + defer firstStream.Close() + defer secondStream.Close() + + first.publishChange(context.Background(), change) + assert.ErrorIs(t, firstStream.(interface{ Err() error }).Err(), errSubscriberOverflow) + second.publishChange(context.Background(), change) + assert.NotErrorIs(t, secondStream.(interface{ Err() error }).Err(), errSubscriberOverflow) + select { + case got := <-secondStream.Changes(): + assert.Equal(t, change.Table, got.Table) + case <-time.After(time.Second): + t.Fatal("independent source did not receive change") + } +} + +func TestSnapshotSubscriptionHandoffIsCommitLSNFenced(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + sub := src.newSubscription(cdcapi.StreamOptions{Snapshot: true, Buffer: 8}) + defer sub.Close() + + before := cdcapi.Change{Op: "insert", Table: "users", CommitLSN: "0/10"} + after := cdcapi.Change{Op: "insert", Table: "users", CommitLSN: "0/30"} + sub.send(context.Background(), before, cdcapi.EstimateChangeBytes(before)) + sub.send(context.Background(), after, cdcapi.EstimateChangeBytes(after)) + select { + case got := <-sub.Changes(): + t.Fatalf("live change escaped before snapshot completion: %#v", got) + case <-time.After(20 * time.Millisecond): + } + + fence, err := pglogrepl.ParseLSN("0/20") + require.NoError(t, err) + snapshot := cdcapi.Change{ + Op: "snapshot", + Table: "users", + CommitLSN: fence.String(), + After: map[string]any{"id": int64(1)}, + } + require.NoError(t, sub.sendSnapshot(snapshot, cdcapi.EstimateChangeBytes(snapshot))) + sub.finishSnapshot(fence, nil) + + select { + case got := <-sub.Changes(): + require.Equal(t, "snapshot", got.Op) + case <-time.After(time.Second): + t.Fatal("snapshot row was not delivered") + } + select { + case got := <-sub.Changes(): + require.Equal(t, after.CommitLSN, got.CommitLSN) + case <-time.After(time.Second): + t.Fatal("post-fence live row was not delivered") + } +} + +func TestSnapshotSubscriptionBoundsPendingLiveChanges(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + change := cdcapi.Change{ + Op: "insert", + Table: "users", + After: map[string]any{"payload": []byte("large")}, + } + bytes := cdcapi.EstimateChangeBytes(change) + sub := src.newSubscription(cdcapi.StreamOptions{Snapshot: true, MaxBytes: bytes}) + defer sub.Close() + sub.send(context.Background(), change, bytes) + sub.send(context.Background(), change, bytes) + assert.ErrorIs(t, sub.Err(), errSubscriberOverflow) + select { + case _, ok := <-sub.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("overflowed snapshot stream did not close") + } +} + +func TestSnapshotSubscriptionsDoNotSharePendingState(t *testing.T) { + firstSource := NewSource(SourceOptions{Name: "db-one", Slot: "slot_one"}) + secondSource := NewSource(SourceOptions{Name: "db-two", Slot: "slot_two"}) + first := firstSource.newSubscription(cdcapi.StreamOptions{Snapshot: true, Buffer: 4}) + second := secondSource.newSubscription(cdcapi.StreamOptions{Snapshot: true, Buffer: 4}) + defer first.Close() + defer second.Close() + + firstChange := cdcapi.Change{Op: "insert", Table: "first", CommitLSN: "0/30"} + secondChange := cdcapi.Change{Op: "insert", Table: "second", CommitLSN: "0/30"} + first.send(context.Background(), firstChange, cdcapi.EstimateChangeBytes(firstChange)) + second.send(context.Background(), secondChange, cdcapi.EstimateChangeBytes(secondChange)) + fence, err := pglogrepl.ParseLSN("0/20") + require.NoError(t, err) + first.finishSnapshot(fence, nil) + second.finishSnapshot(fence, nil) + + select { + case got := <-first.Changes(): + require.Equal(t, "first", got.Table) + case <-time.After(time.Second): + t.Fatal("first source did not deliver its pending change") + } + select { + case got := <-second.Changes(): + require.Equal(t, "second", got.Table) case <-time.After(time.Second): - t.Fatal("timed out waiting for closed cdc stream") + t.Fatal("second source did not deliver its pending change") } } diff --git a/service/cdc/slot.go b/service/cdc/slot.go new file mode 100644 index 000000000..edf001f3d --- /dev/null +++ b/service/cdc/slot.go @@ -0,0 +1,877 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "context" + "errors" + "strconv" + "sync" + + "github.com/wippyai/runtime/api/registry" + api "github.com/wippyai/runtime/api/service/cdc" + "github.com/wippyai/runtime/api/supervisor" + "go.uber.org/zap" +) + +var ( + ErrSourceClosed = errors.New("cdc source slot is closed") + ErrSourceBusy = errors.New("cdc source slot is stopping") +) + +type slotState uint8 + +const ( + slotIdle slotState = iota + slotStarting + slotRunning + slotStopping + slotFaulted + slotStopped +) + +// sourceSlot is the stable object placed in both the system registry and the +// supervisor. A driver replacement changes the delegated generation, never the +// supervisor object or registry pointer. +type sourceSlot struct { + runCtx context.Context + current ManagedSource + runCancel context.CancelFunc + log *zap.Logger + status chan any + retiredHook func(string, uint64) + id registry.ID + kind registry.Kind + retired []retiredSource + generation uint64 + mu sync.RWMutex + opMu sync.Mutex + state slotState + disposing bool + replacing bool + statusDone bool +} + +type retiredSource struct { + source ManagedSource + key string + token uint64 + destructive bool +} + +// leaseRef is passed through a replacement handoff so a failed private +// candidate can retain exactly the lease it reserved. The manager owns the +// lease map; the slot only reports successful retired cleanup through its +// hook. +type leaseRef struct { + key string + token uint64 + owned bool +} + +func newSourceSlot(id registry.ID, kind registry.Kind, source ManagedSource, logs ...*zap.Logger) *sourceSlot { + log := zap.NewNop() + if len(logs) > 0 && logs[0] != nil { + log = logs[0] + } + return &sourceSlot{ + id: canonicalID(id), + kind: kind, + current: source, + log: log, + generation: 1, + state: slotIdle, + } +} + +func (s *sourceSlot) Info() api.SourceInfo { + s.mu.RLock() + current := s.current + generation := s.generation + state := s.state + s.mu.RUnlock() + if isNilSource(current) { + return api.SourceInfo{ + ID: s.id, + Kind: s.kind, + Name: s.id.String(), + Generation: generationString(generation), + State: sourceState(state), + Streaming: state == slotRunning, + Faulted: state == slotFaulted, + Epoch: generationString(generation), + } + } + info := current.Info() + info.ID = s.id + info.Kind = s.kind + info.Name = s.id.String() + info.Generation = generationString(generation) + info.State = sourceState(state) + info.Streaming = state == slotRunning + info.Faulted = state == slotFaulted + if info.Generation != "" { + info.Epoch = info.Generation + } + return info +} + +// Subscribe delegates pre-start subscriptions to drivers that can retain the +// registration until Start establishes the generation. This is required for +// source-owned startup snapshots; drivers without that handoff return +// ErrSourceNotReady. The stable slot still rejects stopped, replacing, and +// disposing generations. +func (s *sourceSlot) Subscribe(ctx context.Context, opts api.StreamOptions) (api.Stream, error) { + if err := opts.Validate(); err != nil { + return nil, err + } + s.mu.RLock() + preStart := s.state == slotIdle || s.state == slotStarting + if (!preStart && s.state != slotRunning) || isNilSource(s.current) || s.disposing || s.replacing { + s.mu.RUnlock() + return nil, api.ErrSourceNotReady + } + current := s.current + generation := s.generation + s.mu.RUnlock() + stream, err := current.Subscribe(ctx, opts) + if err != nil { + return nil, err + } + if stream == nil { + return nil, errors.New("cdc source returned a nil stream") + } + + s.mu.RLock() + stillCurrent := (s.state == slotIdle || s.state == slotStarting || s.state == slotRunning) && + s.current == current && s.generation == generation && !s.replacing + s.mu.RUnlock() + if !stillCurrent { + stream.Close() + return nil, api.ErrSourceNotReady + } + return newStampedStream(s.id, generation, opts.Buffer, stream), nil +} + +// Start is idempotent while the active generation is running. This is what +// permits an update to synchronously start a candidate before the supervisor +// receives its unchanged stable slot pointer. +func (s *sourceSlot) Start(ctx context.Context) (<-chan any, error) { + if ctx == nil { + ctx = context.Background() + } + s.opMu.Lock() + defer s.opMu.Unlock() + + s.mu.Lock() + if s.disposing { + s.mu.Unlock() + return nil, ErrSourceBusy + } + if len(s.retired) > 0 { + s.mu.Unlock() + return nil, ErrSourceBusy + } + if s.state == slotRunning { + status := s.status + s.mu.Unlock() + return status, nil + } + if s.state == slotStopping { + s.mu.Unlock() + return nil, ErrSourceBusy + } + if isNilSource(s.current) { + s.mu.Unlock() + return nil, ErrSourceClosed + } + current := s.current + restart := s.state == slotStopped || s.state == slotFaulted + status := make(chan any, 8) + s.status = status + s.statusDone = false + s.state = slotStarting + runCtx, runCancel := detachedContext(ctx) + s.runCtx = runCtx + s.runCancel = runCancel + s.replacing = false + s.mu.Unlock() + + if err := s.retryRetired(ctx); err != nil { + _ = stopSource(ctx, current) + s.mu.Lock() + s.state = slotFaulted + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil + s.mu.Unlock() + return nil, err + } + + underlying, err := startSource(ctx, runCtx, current) + if err != nil { + _ = stopSource(ctx, current) + s.mu.Lock() + s.state = slotFaulted + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil + s.mu.Unlock() + return nil, err + } + + s.mu.Lock() + s.state = slotRunning + if restart { + s.generation++ + } + generation := s.generation + s.mu.Unlock() + s.watchStatus(current, generation, underlying) + return status, nil +} + +func (s *sourceSlot) Stop(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + s.opMu.Lock() + defer s.opMu.Unlock() + + s.mu.Lock() + if !s.disposing && s.state == slotStopped && len(s.retired) == 0 { + s.mu.Unlock() + return nil + } + s.state = slotStopping + current := s.current + cancel := s.runCancel + pendingCurrent := s.isRetiredLocked(current) + disposing := s.disposing + s.mu.Unlock() + + if cancel != nil { + cancel() + } + var err error + if disposing && !pendingCurrent { + if disposable, ok := current.(Disposable); ok { + err = disposable.Dispose(ctx) + } else { + err = stopSource(ctx, current) + } + } else { + err = stopSource(ctx, current) + } + err = errors.Join(err, s.retryRetired(ctx)) + + s.mu.Lock() + if err != nil { + s.state = slotFaulted + } else { + s.state = slotStopped + } + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil + s.replacing = false + s.mu.Unlock() + return err +} + +// Dispose performs a committed delete. It mirrors Stop's stable-slot state +// transition, but delegates the destructive hook to the active driver only +// on this path. Updates and supervisor restarts always use Stop instead. +func (s *sourceSlot) Dispose(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + s.opMu.Lock() + defer s.opMu.Unlock() + + s.mu.Lock() + s.disposing = true + s.state = slotStopping + current := s.current + cancel := s.runCancel + pendingCurrent := s.isRetiredLocked(current) + s.mu.Unlock() + + if cancel != nil { + cancel() + } + var err error + if isNilSource(current) { + err = ErrSourceClosed + } else if !pendingCurrent { + if disposable, ok := current.(Disposable); ok { + err = disposable.Dispose(ctx) + } else { + err = stopSource(ctx, current) + } + } else { + err = nil + } + err = errors.Join(err, s.retryRetired(ctx)) + + s.mu.Lock() + if err != nil { + s.state = slotFaulted + } else { + s.state = slotStopped + } + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil + s.replacing = false + s.mu.Unlock() + return err +} + +// Replace starts a candidate before changing visibility whenever the slot is +// running or the candidate is configured for auto-start. A failed handoff +// never publishes the candidate; any failed candidate cleanup is retained as +// retired work for a later Stop/Delete retry. +func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, oldLease, candidateLease leaseRef) error { + if isNilSource(candidate) { + return ErrDriverRequired + } + if ctx == nil { + ctx = context.Background() + } + s.opMu.Lock() + defer s.opMu.Unlock() + + s.mu.Lock() + old := s.current + oldState := s.state + oldRunCancel := s.runCancel + disposing := s.disposing + hasRetired := len(s.retired) > 0 + s.mu.Unlock() + + oldKey := exclusiveResourceKey(old) + candidateKey := exclusiveResourceKey(candidate) + differentResource := oldKey != candidateKey + if disposing || hasRetired || oldState == slotStopping { + cleanupErr := s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, false) + return errors.Join(ErrSourceBusy, cleanupErr) + } + + // Re-check under the slot lock after calculating resource identity. No + // other lifecycle operation can replace current while opMu is held, but a + // status watcher may still have changed the state. + s.mu.Lock() + if s.disposing || len(s.retired) > 0 || s.state == slotStopping { + s.replacing = false + s.mu.Unlock() + cleanupErr := s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, false) + return errors.Join(ErrSourceBusy, cleanupErr) + } + s.replacing = true + s.mu.Unlock() + + startCandidate := oldState == slotRunning || lifecycleAutoStart(candidate) + // A source configured with a startup snapshot may have accepted a + // pre-start subscription while the stable slot was idle. Stop that old + // generation on replacement so its driver can close the prepared stream; + // ordinary idle sources retain the historical no-op handoff. + oldHasStartupSnapshot := oldState == slotIdle && !isNilSource(old) && old.Info().Snapshot + shouldStopOld := !isNilSource(old) && + (oldState != slotStopped && oldState != slotIdle || differentResource || oldKey == "" || oldHasStartupSnapshot) + + var ( + underlying <-chan any + runCtx context.Context + runCancel context.CancelFunc + ) + speculative := differentResource && startCandidate + startCandidateGeneration := func() error { + runCtx, runCancel = detachedContext(ctx) + var err error + underlying, err = startSource(ctx, runCtx, candidate) + if err != nil { + runCancel() + cleanupErr := s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, true) + return errors.Join(err, cleanupErr) + } + return nil + } + if speculative { + // Different resource keys may be prepared in parallel. The candidate + // remains private until old Stop and Dispose both commit below. + if err := startCandidateGeneration(); err != nil { + return s.resetReplaceFailure(oldState, err) + } + } + + // The old source is stopped before destructive cleanup. A speculative + // candidate may already be running, but it is not visible through the slot + // until this handoff has completed. + oldStopped := !shouldStopOld + if shouldStopOld { + if err := stopGeneration(ctx, old, oldRunCancel); err != nil { + var cleanupErr error + if speculative { + runCancel() + cleanupErr = s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, true) + } else { + cleanupErr = s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, false) + } + s.mu.Lock() + s.state = slotFaulted + s.replacing = false + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil + s.mu.Unlock() + return errors.Join(err, cleanupErr) + } + oldStopped = true + s.mu.Lock() + s.state = slotStopped + s.runCtx = nil + s.runCancel = nil + s.closeStatusLocked() + s.mu.Unlock() + } + + if differentResource && !isNilSource(old) { + if disposable, ok := old.(Disposable); ok { + if err := disposable.Dispose(ctx); err != nil { + // Keep the old source current and retain its lease. Stop retries + // this pending destructive cleanup during shutdown or delete. + s.recordRetired(old, oldKey, oldLease.token, true) + s.mu.Lock() + s.state = slotFaulted + s.replacing = false + s.mu.Unlock() + var cleanupErr error + if speculative { + runCancel() + } + cleanupErr = s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, speculative) + return errors.Join(err, cleanupErr) + } + if oldKey != "" { + s.mu.RLock() + hook := s.retiredHook + s.mu.RUnlock() + if hook != nil { + hook(oldKey, oldLease.token) + } + } + } + } + + if startCandidate && !speculative { + if err := startCandidateGeneration(); err != nil { + _, oldDisposable := old.(Disposable) + restoreOld := oldState == slotRunning && (!differentResource || !oldDisposable) + if restoreOld && oldStopped { + // The old generation still owns the shared resource. Restore it + // before making the failed update visible to the caller. + restoreCtx, restoreCancel := detachedContext(ctx) + oldUpdates, restoreErr := startSource(ctx, restoreCtx, old) + if restoreErr == nil { + s.mu.Lock() + s.state = slotRunning + s.generation++ + s.runCtx = restoreCtx + s.runCancel = restoreCancel + s.replacing = false + generation := s.generation + if s.status == nil || s.statusDone { + s.status = make(chan any, 8) + s.statusDone = false + } + s.mu.Unlock() + s.watchStatus(old, generation, oldUpdates) + return err + } + restoreCancel() + return s.finishReplaceFailure(errors.Join(err, restoreErr)) + } + return s.finishReplaceFailure(err) + } + } + + s.mu.Lock() + if s.disposing || s.state == slotStopping { + s.replacing = false + s.state = slotFaulted + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil + s.mu.Unlock() + if runCancel != nil { + runCancel() + } + cleanupErr := s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, startCandidate) + return errors.Join(ErrSourceBusy, cleanupErr) + } + s.current = candidate + s.generation++ + if startCandidate { + if s.status == nil || s.statusDone { + s.status = make(chan any, 8) + s.statusDone = false + } + s.state = slotRunning + s.runCtx = runCtx + s.runCancel = runCancel + } else if oldState == slotIdle { + s.state = slotIdle + } else { + s.state = slotStopped + } + s.replacing = false + generation := s.generation + s.mu.Unlock() + + if startCandidate { + s.watchStatus(candidate, generation, underlying) + } + return nil +} + +func (s *sourceSlot) finishReplaceFailure(err error) error { + s.mu.Lock() + s.state = slotFaulted + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil + s.replacing = false + s.mu.Unlock() + return err +} + +func (s *sourceSlot) resetReplaceFailure(state slotState, err error) error { + s.mu.Lock() + s.state = state + s.replacing = false + s.mu.Unlock() + return err +} + +// stopUnstartedSource abandons a source returned by Driver.Create before its +// Start method has successfully handed ownership of a durable resource to the +// manager. Drivers must make Create side-effect-free; Stop is intentionally +// the only cleanup allowed on this path so an unstarted candidate cannot drop +// a shared replication slot/checkpoint. +func stopUnstartedSource(ctx context.Context, source api.Source) error { + return stopSource(ctx, source) +} + +// cleanupStartedSource cleans a candidate after Start was attempted. A +// different exclusive resource is owned solely by that candidate and may be +// destructively disposed. Same-key candidates share the old resource contract +// and must only be stopped; disposing them could drop the old generation's +// resource. +func cleanupStartedSource(ctx context.Context, source api.Source, destructive bool) error { + if destructive { + return disposeSource(ctx, source) + } + return stopSource(ctx, source) +} + +// cleanupCandidate performs the only cleanup of a private replacement +// generation. When cleanup itself fails, the candidate is retained in the +// slot's retired queue so Stop/Delete can retry it. A different resource keeps +// its candidate lease; a same-key candidate never owns the old lease and must +// not release it when its non-destructive Stop eventually succeeds. +func (s *sourceSlot) cleanupCandidate( + ctx context.Context, + source ManagedSource, + lease leaseRef, + differentResource bool, + started bool, +) error { + if isNilSource(source) { + return nil + } + var err error + if started { + err = cleanupStartedSource(ctx, source, differentResource) + } else { + err = stopUnstartedSource(ctx, source) + } + if err == nil { + return nil + } + if !differentResource && !lease.owned { + lease = leaseRef{} + } + s.recordRetired(source, lease.key, lease.token, started && differentResource) + return err +} + +func disposeSource(ctx context.Context, source api.Source) error { + if isNilSource(source) { + return nil + } + if disposable, ok := source.(Disposable); ok { + return disposable.Dispose(ctx) + } + return stopSource(ctx, source) +} + +func stopGeneration(ctx context.Context, source api.Source, cancel context.CancelFunc) error { + if cancel != nil { + cancel() + } + return stopSource(ctx, source) +} + +func (s *sourceSlot) recordRetired(source ManagedSource, key string, token uint64, destructive bool) { + if isNilSource(source) { + return + } + s.mu.Lock() + for _, existing := range s.retired { + if existing.source == source { + s.mu.Unlock() + return + } + } + s.retired = append(s.retired, retiredSource{ + source: source, + key: key, + token: token, + destructive: destructive, + }) + s.mu.Unlock() +} + +func (s *sourceSlot) isRetiredLocked(source ManagedSource) bool { + if isNilSource(source) { + return false + } + for _, retired := range s.retired { + if retired.source == source { + return true + } + } + return false +} + +func (s *sourceSlot) hasRetiredSource(source ManagedSource) bool { + s.mu.RLock() + defer s.mu.RUnlock() + for _, retired := range s.retired { + if retired.source == source { + return true + } + } + return false +} + +func (s *sourceSlot) retryRetired(ctx context.Context) error { + s.mu.RLock() + retired := append([]retiredSource(nil), s.retired...) + s.mu.RUnlock() + var errs []error + for _, item := range retired { + var err error + if item.destructive { + err = disposeSource(ctx, item.source) + } else { + err = stopSource(ctx, item.source) + } + if err != nil { + errs = append(errs, err) + continue + } + + s.mu.Lock() + for i, current := range s.retired { + if current.source == item.source { + s.retired = append(s.retired[:i], s.retired[i+1:]...) + break + } + } + hook := s.retiredHook + s.mu.Unlock() + if hook != nil && item.key != "" { + hook(item.key, item.token) + } + } + return errors.Join(errs...) +} + +func (s *sourceSlot) setRetiredCleanupHook(hook func(string, uint64)) { + s.mu.Lock() + s.retiredHook = hook + s.mu.Unlock() +} + +func (s *sourceSlot) hasRetiredKey(key string) bool { + if key == "" { + return false + } + s.mu.RLock() + defer s.mu.RUnlock() + for _, retired := range s.retired { + if retired.key == key { + return true + } + } + return false +} + +func (s *sourceSlot) resourceKeys() []string { + s.mu.RLock() + current := s.current + retired := append([]retiredSource(nil), s.retired...) + s.mu.RUnlock() + keys := make([]string, 0, len(retired)+1) + if key := exclusiveResourceKey(current); key != "" { + keys = append(keys, key) + } + for _, item := range retired { + if item.key == "" { + continue + } + seen := false + for _, key := range keys { + if key == item.key { + seen = true + break + } + } + if !seen { + keys = append(keys, item.key) + } + } + return keys +} + +func (s *sourceSlot) LifecycleConfig() supervisor.LifecycleConfig { + s.mu.RLock() + current := s.current + s.mu.RUnlock() + if isNilSource(current) { + return supervisor.LifecycleConfig{} + } + if configured, ok := current.(interface { + LifecycleConfig() supervisor.LifecycleConfig + }); ok { + return configured.LifecycleConfig() + } + return supervisor.LifecycleConfig{} +} + +func (s *sourceSlot) watchStatus(source ManagedSource, generation uint64, updates <-chan any) { + if updates == nil { + return + } + go func() { + for detail := range updates { + s.mu.RLock() + current := s.current == source && s.generation == generation && s.state == slotRunning && !s.replacing + status := s.status + s.mu.RUnlock() + if !current || status == nil { + continue + } + select { + case status <- detail: + default: + } + } + + s.mu.Lock() + if s.current == source && s.generation == generation && s.state == slotRunning && !s.replacing { + s.state = slotFaulted + s.closeStatusLocked() + } + s.mu.Unlock() + }() +} + +func exclusiveResourceKey(source ManagedSource) string { + if isNilSource(source) { + return "" + } + if keyed, ok := source.(ExclusiveResource); ok { + return keyed.ExclusiveResourceKey() + } + return "" +} + +func (s *sourceSlot) currentSource() ManagedSource { + s.mu.RLock() + source := s.current + s.mu.RUnlock() + return source +} + +func (s *sourceSlot) closeStatusLocked() { + if s.status != nil && !s.statusDone { + close(s.status) + s.statusDone = true + } +} + +func lifecycleAutoStart(source ManagedSource) bool { + configured, ok := source.(interface { + LifecycleConfig() supervisor.LifecycleConfig + }) + return ok && configured.LifecycleConfig().AutoStart +} + +func sourceState(state slotState) api.SourceState { + switch state { + case slotStarting: + return api.SourceStateStarting + case slotRunning: + return api.SourceStateRunning + case slotFaulted: + return api.SourceStateFaulted + case slotStopped: + return api.SourceStateStopped + default: + return api.SourceStateUnknown + } +} + +// startSource gives the startup operation its caller's cancellation while +// retaining a detached run context after successful startup. Supervisor +// start timeouts must still interrupt a blocked driver handshake, but a +// dynamic registry event must not cancel a source that it has just started. +func startSource(ctx context.Context, runCtx context.Context, source ManagedSource) (<-chan any, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + if runCtx == nil { + runCtx = context.WithoutCancel(ctx) + } + startCtx, cancelStart := context.WithCancel(runCtx) + stopPropagation := context.AfterFunc(ctx, cancelStart) + updates, err := source.Start(startCtx) + stopPropagation() + if err == nil { + err = ctx.Err() + } + if err != nil { + cancelStart() + } + return updates, err +} + +func detachedContext(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithCancel(context.WithoutCancel(ctx)) +} + +func generationString(generation uint64) string { + if generation == 0 { + return "" + } + return strconv.FormatUint(generation, 10) +} + +var _ ManagedSource = (*sourceSlot)(nil) +var _ Disposable = (*sourceSlot)(nil) diff --git a/service/cdc/sqlite/bench_test.go b/service/cdc/sqlite/bench_test.go new file mode 100644 index 000000000..3de73c668 --- /dev/null +++ b/service/cdc/sqlite/bench_test.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import "testing" + +func BenchmarkMapRow(b *testing.B) { + cols := []columnInfo{ + {name: "id"}, + {name: "email", text: true}, + {name: "balance"}, + {name: "blob"}, + } + vals := []any{int64(1), []byte("user@example.com"), 42.5, []byte{0x00, 0x01, 0x02}} + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = mapRow(cols, vals) + } +} diff --git a/service/cdc/sqlite/decode.go b/service/cdc/sqlite/decode.go new file mode 100644 index 000000000..9decc12d0 --- /dev/null +++ b/service/cdc/sqlite/decode.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "strconv" + "strings" + "unicode/utf8" +) + +type columnInfo struct { + name string + text bool +} + +func textAffinity(declType string) bool { + t := strings.ToUpper(declType) + if strings.Contains(t, "BLOB") || t == "" { + return false + } + return strings.Contains(t, "CHAR") || strings.Contains(t, "CLOB") || strings.Contains(t, "TEXT") +} + +func mapRow(cols []columnInfo, vals []any) map[string]any { + if vals == nil { + return nil + } + out := make(map[string]any, len(vals)) + for i, v := range vals { + name, text := columnAt(cols, i) + out[name] = normalizeValue(v, text) + } + return out +} + +func columnAt(cols []columnInfo, i int) (string, bool) { + if i < len(cols) { + return cols[i].name, cols[i].text + } + return "column" + strconv.Itoa(i), false +} + +func normalizeValue(v any, text bool) any { + b, ok := v.([]byte) + if !ok { + return v + } + if text && utf8.Valid(b) { + return string(b) + } + return b +} diff --git a/service/cdc/sqlite/decode_test.go b/service/cdc/sqlite/decode_test.go new file mode 100644 index 000000000..f1307111f --- /dev/null +++ b/service/cdc/sqlite/decode_test.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTextAffinity(t *testing.T) { + cases := map[string]bool{ + "TEXT": true, + "VARCHAR(255)": true, + "CLOB": true, + "nchar": true, + "INTEGER": false, + "REAL": false, + "BLOB": false, + "": false, + "NUMERIC": false, + } + for decl, want := range cases { + assert.Equalf(t, want, textAffinity(decl), "decl=%q", decl) + } +} + +func TestMapRowNilForMissingSide(t *testing.T) { + assert.Nil(t, mapRow([]columnInfo{{name: "id"}}, nil)) +} + +func TestMapRowDecodesTextBytesByAffinity(t *testing.T) { + cols := []columnInfo{ + {name: "id", text: false}, + {name: "email", text: true}, + {name: "payload", text: false}, + } + vals := []any{int64(7), []byte("a@b.com"), []byte{0x00, 0x01, 0x02}} + + out := mapRow(cols, vals) + + assert.Equal(t, int64(7), out["id"]) + assert.Equal(t, "a@b.com", out["email"]) + assert.Equal(t, []byte{0x00, 0x01, 0x02}, out["payload"]) +} + +func TestMapRowKeepsInvalidUTF8AsBytes(t *testing.T) { + cols := []columnInfo{{name: "data", text: true}} + invalid := []byte{0xff, 0xfe, 0xfd} + out := mapRow(cols, []any{invalid}) + assert.Equal(t, invalid, out["data"]) +} + +func TestMapRowFallbackColumnNames(t *testing.T) { + out := mapRow(nil, []any{int64(1), "x"}) + assert.Equal(t, int64(1), out["column0"]) + assert.Equal(t, "x", out["column1"]) +} + +func TestNormalizeValuePassThrough(t *testing.T) { + assert.Equal(t, int64(3), normalizeValue(int64(3), true)) + assert.Equal(t, 1.5, normalizeValue(1.5, true)) + assert.Nil(t, normalizeValue(nil, true)) +} diff --git a/service/cdc/sqlite/driver.go b/service/cdc/sqlite/driver.go new file mode 100644 index 000000000..7010a2e18 --- /dev/null +++ b/service/cdc/sqlite/driver.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "context" + + "go.uber.org/zap" + + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/resource" + config "github.com/wippyai/runtime/api/service/cdc" + "github.com/wippyai/runtime/api/supervisor" + cdcservice "github.com/wippyai/runtime/service/cdc" + entryutil "github.com/wippyai/runtime/system/entry" +) + +// managedSource is the narrow internal contract shared by build-tagged +// implementations. Keeping it in the untagged driver file makes the package +// fail closed without the SQLite preupdate build tag rather than exposing a +// half-working source. +type managedSource interface { + config.Source + supervisor.Service +} + +type sourceOptions struct { + res resource.Registry + log *zap.Logger + id registry.ID + dbResource registry.ID + name string + statusInterval string + tables []string + lifecycle supervisor.LifecycleConfig + snapshot bool +} + +// Driver wires the SQLite CDC implementation into the driver-neutral CDC +// manager. It owns no process-global SQL driver registration. +type Driver struct{} + +func NewDriver() cdcservice.Driver { return Driver{} } + +func (Driver) Kind() registry.Kind { return config.SQLite } + +func (Driver) Create(ctx context.Context, entry registry.Entry, deps cdcservice.Dependencies) (cdcservice.ManagedSource, error) { + if deps.Resources == nil { + return nil, ErrResourceRegRequired + } + cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, deps.Transcoder, entry) + if err != nil { + return nil, NewInvalidConfigError(err) + } + if err := cfg.Validate(); err != nil { + return nil, NewInvalidConfigError(err) + } + log := deps.Logger + if log == nil { + log = zap.NewNop() + } + return buildSource(sourceOptions{ + res: deps.Resources, + log: log.With(zap.String("id", entry.ID.String())), + id: entry.ID, + dbResource: registry.ParseID(cfg.DBResource), + name: entry.ID.String(), + statusInterval: cfg.StatusInterval, + tables: cfg.Tables, + snapshot: cfg.Snapshot, + lifecycle: cfg.Lifecycle, + }) +} + +var _ cdcservice.Driver = Driver{} diff --git a/service/cdc/sqlite/driver_test.go b/service/cdc/sqlite/driver_test.go new file mode 100644 index 000000000..92d769e07 --- /dev/null +++ b/service/cdc/sqlite/driver_test.go @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + config "github.com/wippyai/runtime/api/service/cdc" +) + +func TestDriverKind(t *testing.T) { + assert.Equal(t, config.SQLite, Driver{}.Kind()) + assert.Equal(t, config.SQLite, NewDriver().Kind()) +} diff --git a/service/cdc/sqlite/errors.go b/service/cdc/sqlite/errors.go new file mode 100644 index 000000000..714a00c1f --- /dev/null +++ b/service/cdc/sqlite/errors.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "errors" + + "github.com/wippyai/runtime/api/attrs" + apierror "github.com/wippyai/runtime/api/error" + "github.com/wippyai/runtime/api/registry" +) + +var ( + ErrSourceClosed = errors.New("cdc: source is closed") + ErrTranscoderRequired = apierror.New(apierror.Invalid, "transcoder is required").WithRetryable(apierror.False) + ErrEventBusRequired = apierror.New(apierror.Invalid, "event bus is required").WithRetryable(apierror.False) + ErrResourceRegRequired = apierror.New(apierror.Invalid, "resource registry is required").WithRetryable(apierror.False) + ErrPreupdateTagRequired = apierror.New(apierror.Internal, "sqlite cdc requires the sqlite_preupdate_hook build tag").WithRetryable(apierror.False) + ErrNoSourceStreamer = apierror.New(apierror.Internal, "cdc source streamer not available").WithRetryable(apierror.False) + ErrChangeBacklogOverflow = apierror.New(apierror.Unavailable, "sqlite cdc change backlog overflow").WithRetryable(apierror.True) +) + +func NewUnsupportedEntryKindError(kind registry.Kind) apierror.Error { + return apierror.New(apierror.Invalid, "unsupported entry kind"). + WithRetryable(apierror.False). + WithDetails(attrs.NewBagFrom(map[string]any{"kind": kind})) +} + +func NewServiceExistsError(id registry.ID) apierror.Error { + return apierror.New(apierror.Conflict, "cdc service already exists"). + WithRetryable(apierror.False). + WithDetails(attrs.NewBagFrom(map[string]any{"id": id.String()})) +} + +func NewServiceNotFoundError(id registry.ID) apierror.Error { + return apierror.New(apierror.NotFound, "cdc service not found"). + WithRetryable(apierror.False). + WithDetails(attrs.NewBagFrom(map[string]any{"id": id.String()})) +} + +func NewInvalidConfigError(err error) apierror.Error { + apiErr := apierror.New(apierror.Invalid, "invalid cdc configuration").WithRetryable(apierror.False) + if err != nil { + apiErr = apiErr.WithDetails(attrs.NewBagFrom(map[string]any{"cause": err.Error()})).WithCause(err) + } + return apiErr +} + +func NewSourceCreationError(err error) apierror.Error { + apiErr := apierror.New(apierror.Internal, "failed to create cdc source").WithRetryable(apierror.False) + if err != nil { + apiErr = apiErr.WithDetails(attrs.NewBagFrom(map[string]any{"cause": err.Error()})).WithCause(err) + } + return apiErr +} diff --git a/service/cdc/sqlite/helpers_test.go b/service/cdc/sqlite/helpers_test.go new file mode 100644 index 000000000..f0520c3d6 --- /dev/null +++ b/service/cdc/sqlite/helpers_test.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeSchema(t *testing.T) { + assert.Equal(t, "main", normalizeSchema("")) + assert.Equal(t, "temp", normalizeSchema("temp")) + assert.Equal(t, "audit", normalizeSchema("audit")) +} + +func TestValuesByColumnRequiresCapturedShape(t *testing.T) { + got, err := valuesByColumn([]string{"id", "name"}, []any{int64(1), "one"}) + require.NoError(t, err) + assert.Equal(t, map[string]any{"id": int64(1), "name": "one"}, got) + + _, err = valuesByColumn([]string{"id"}, []any{int64(1), "extra"}) + assert.Error(t, err) + _, err = valuesByColumn([]string{"id", "id"}, []any{int64(1), int64(2)}) + assert.Error(t, err) +} + +func TestValuesByColumnCopiesBytes(t *testing.T) { + value := []byte{1, 2, 3} + got, err := valuesByColumn([]string{"blob"}, []any{value}) + require.NoError(t, err) + value[0] = 9 + assert.Equal(t, []byte{1, 2, 3}, got["blob"]) +} diff --git a/service/cdc/sqlite/integration_live_test.go b/service/cdc/sqlite/integration_live_test.go new file mode 100644 index 000000000..3414b7ef7 --- /dev/null +++ b/service/cdc/sqlite/integration_live_test.go @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build integration && sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/wippyai/runtime/api/registry" + cdcapi "github.com/wippyai/runtime/api/service/cdc" + sqlapi "github.com/wippyai/runtime/api/service/sql" + sqlconfig "github.com/wippyai/runtime/api/service/sql" + sqliteengine "github.com/wippyai/runtime/service/sql/engine/sqlite" +) + +type integrationDB struct { + db *sql.DB + observer sqlapi.CommittedMutationSource + resources *testResourceRegistry + source *Source +} + +func openIntegrationDB(t *testing.T, opts sourceOptions) *integrationDB { + t.Helper() + ctx := context.Background() + file := filepath.Join(t.TempDir(), "cdc.db") + cfg := &sqlconfig.SQLiteConfig{File: file} + cfg.InitDefaults() + driver := sqliteengine.NewDriver() + opened, err := driver.Open(ctx, cfg) + require.NoError(t, err) + require.NoError(t, driver.Prepare(ctx, opened.DB, cfg)) + driver.Tune(opened.DB, cfg) + require.NotNil(t, opened.Observer) + + resources := &testResourceRegistry{observer: opened.Observer} + if opts.res == nil { + opts.res = resources + } + if opts.id.Name == "" { + opts.id = registry.NewID("app", "sqlite-cdc") + } + if opts.name == "" { + opts.name = opts.id.String() + } + sourceValue, err := buildSource(opts) + require.NoError(t, err) + source := sourceValue.(*Source) + + result := &integrationDB{db: opened.DB, observer: opened.Observer, resources: resources, source: source} + t.Cleanup(func() { + _ = source.Stop(context.Background()) + _ = opened.Observer.Close() + _ = opened.DB.Close() + }) + return result +} + +func requireNoChange(t *testing.T, stream cdcapi.Stream) { + t.Helper() + select { + case change, ok := <-stream.Changes(): + if !ok { + t.Fatalf("stream closed unexpectedly: %v", stream.Err()) + } + t.Fatalf("unexpected CDC change: %#v", change) + case <-time.After(100 * time.Millisecond): + } +} + +func TestIntegrationLiveCommitAndFilters(t *testing.T) { + db := openIntegrationDB(t, sourceOptions{tables: []string{"users"}}) + _, err := db.db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + _, err = db.db.Exec(`CREATE TABLE audit (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + require.NoError(t, startSourceForIntegration(db.source)) + + stream, err := db.source.Subscribe(context.Background(), cdcapi.StreamOptions{Ops: []string{"insert"}}) + require.NoError(t, err) + defer stream.Close() + + _, err = db.db.Exec(`INSERT INTO users (id, value) VALUES (1, 'one')`) + require.NoError(t, err) + change := receiveChange(t, stream) + assert.Equal(t, "insert", change.Op) + assert.Equal(t, "users", change.Table) + assert.Equal(t, int64(1), change.After["id"]) + + _, err = db.db.Exec(`UPDATE users SET value = 'two' WHERE id = 1`) + require.NoError(t, err) + requireNoChange(t, stream) + _, err = db.db.Exec(`INSERT INTO audit (id, value) VALUES (1, 'ignored')`) + require.NoError(t, err) + requireNoChange(t, stream) +} + +func startSourceForIntegration(source *Source) error { + _, err := source.Start(context.Background()) + return err +} + +func TestIntegrationRollbackAndSavepointPublishOnlyCommittedRows(t *testing.T) { + db := openIntegrationDB(t, sourceOptions{}) + _, err := db.db.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + require.NoError(t, startSourceForIntegration(db.source)) + stream, err := db.source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + defer stream.Close() + + tx, err := db.db.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (1, 'kept')`) + require.NoError(t, err) + _, err = tx.Exec(`SAVEPOINT nested`) + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (2, 'discarded')`) + require.NoError(t, err) + _, err = tx.Exec(`ROLLBACK TO SAVEPOINT nested`) + require.NoError(t, err) + _, err = tx.Exec(`RELEASE SAVEPOINT nested`) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + + change := receiveChange(t, stream) + assert.Equal(t, int64(1), change.After["id"]) + assert.Equal(t, []byte("kept"), change.After["value"]) + requireNoChange(t, stream) + + _, err = db.db.Exec(`INSERT INTO items (id, value) VALUES (3, 'rolled back')`) + require.NoError(t, err) + change = receiveChange(t, stream) + assert.Equal(t, int64(3), change.After["id"]) +} + +func TestIntegrationFailedStatementFailsClosed(t *testing.T) { + db := openIntegrationDB(t, sourceOptions{}) + _, err := db.db.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + require.NoError(t, err) + require.NoError(t, startSourceForIntegration(db.source)) + stream, err := db.source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + defer stream.Close() + + tx, err := db.db.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT OR FAIL INTO items (id, value) VALUES (1, 'same'), (2, 'same')`) + assert.Error(t, err) + require.NoError(t, tx.Commit()) + + var count int + require.NoError(t, db.db.QueryRow(`SELECT count(*) FROM items`).Scan(&count)) + assert.Equal(t, 1, count, "SQLite must retain the applied prefix") + streamErr := waitStreamClosed(t, stream) + require.Error(t, streamErr) + assert.Contains(t, streamErr.Error(), "cannot determine statement outcome") +} + +func TestIntegrationSubscriberOverflowDoesNotRollbackApplication(t *testing.T) { + db := openIntegrationDB(t, sourceOptions{}) + _, err := db.db.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + require.NoError(t, startSourceForIntegration(db.source)) + stream, err := db.source.Subscribe(context.Background(), cdcapi.StreamOptions{Buffer: 1}) + require.NoError(t, err) + + for i := 1; i <= 32; i++ { + _, err = db.db.Exec(`INSERT INTO items (id, value) VALUES (?, 'value')`, i) + require.NoError(t, err) + } + err = waitStreamClosed(t, stream) + assert.ErrorIs(t, err, errSubscriberOverflow) + var count int + require.NoError(t, db.db.QueryRow(`SELECT count(*) FROM items`).Scan(&count)) + assert.Equal(t, 32, count) +} + +func TestIntegrationSQLGenerationCloseFaultsSource(t *testing.T) { + db := openIntegrationDB(t, sourceOptions{}) + _, err := db.db.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + require.NoError(t, startSourceForIntegration(db.source)) + stream, err := db.source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + + require.NoError(t, db.observer.Close()) + assert.Error(t, waitStreamClosed(t, stream)) + assert.Equal(t, cdcapi.SourceStateFaulted, db.source.Info().State) +} + +func TestIntegrationSnapshotHandoffIsPerSubscriber(t *testing.T) { + db := openIntegrationDB(t, sourceOptions{}) + _, err := db.db.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + _, err = db.db.Exec(`INSERT INTO items (id, value) VALUES (1, 'existing')`) + require.NoError(t, err) + require.NoError(t, startSourceForIntegration(db.source)) + stream, err := db.source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + require.NoError(t, err) + defer stream.Close() + + snapshot := receiveChange(t, stream) + assert.Equal(t, "snapshot", snapshot.Op) + assert.Equal(t, int64(1), snapshot.After["id"]) + assert.NotEmpty(t, snapshot.Cursor) + + _, err = db.db.Exec(`INSERT INTO items (id, value) VALUES (2, 'live')`) + require.NoError(t, err) + live := receiveChange(t, stream) + assert.Equal(t, "insert", live.Op) + assert.Equal(t, int64(2), live.After["id"]) +} diff --git a/service/cdc/sqlite/source.go b/service/cdc/sqlite/source.go new file mode 100644 index 000000000..26accf32a --- /dev/null +++ b/service/cdc/sqlite/source.go @@ -0,0 +1,857 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "go.uber.org/zap" + + "github.com/wippyai/runtime/api/metrics" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/resource" + config "github.com/wippyai/runtime/api/service/cdc" + sqlapi "github.com/wippyai/runtime/api/service/sql" + "github.com/wippyai/runtime/api/supervisor" + sqlservice "github.com/wippyai/runtime/service/sql" +) + +const ( + defaultStatusInterval = 30 * time.Second + cleanupTimeout = 5 * time.Second + changesCounter = "wippy_cdc_changes_total" +) + +// Source is the SQLite CDC adapter. The SQL resource owns the SQLite +// connection, driver hooks, and observer lifetime. This adapter only borrows +// the resource long enough to subscribe to its committed-mutation capability; +// it never opens another connection and never installs hooks on a raw one. +type Source struct { + sourceErr error + res resource.Registry + observerSource sqlapi.CommittedMutationSource + observer sqlapi.MutationStream + status chan any + runCancel context.CancelFunc + stopGate chan struct{} + startCancel context.CancelFunc + subs *subscribers + log *zap.Logger + startDone chan struct{} + snapshotSubs map[*subscription]sqlapi.MutationStream + runDone chan struct{} + snapshotWait chan struct{} + snapshotAcq map[uint64]*snapshotAcquisition + id registry.ID + dbResID registry.ID + name string + state config.SourceState + generation string + tables []string + lifecycle configLifecycle + snapshotWG sync.WaitGroup + statusTick time.Duration + nextSnapshotID uint64 + mu sync.RWMutex + snapshot bool + statusClosed bool + stopping bool +} + +// configLifecycle is an alias kept local to this package so the Source does +// not expose configuration implementation details through its API. +type configLifecycle = supervisor.LifecycleConfig + +type snapshotAcquisition struct { + cancel context.CancelFunc +} + +func buildSource(opts sourceOptions) (managedSource, error) { + log := opts.log + if log == nil { + log = zap.NewNop() + } + interval := defaultStatusInterval + if opts.statusInterval != "" { + d, err := time.ParseDuration(opts.statusInterval) + if err != nil || d < 0 { + return nil, fmt.Errorf("invalid status_interval %q", opts.statusInterval) + } + if d > 0 { + interval = d + } + } + name := opts.name + if name == "" && (opts.id.NS != "" || opts.id.Name != "") { + name = opts.id.String() + } + if name == "" { + name = "sqlite" + } + stopGate := make(chan struct{}, 1) + stopGate <- struct{}{} + + return &Source{ + res: opts.res, + log: log, + id: opts.id, + name: name, + dbResID: opts.dbResource, + tables: append([]string(nil), opts.tables...), + statusTick: interval, + lifecycle: opts.lifecycle, + snapshot: opts.snapshot, + subs: newSubscribers(), + snapshotSubs: make(map[*subscription]sqlapi.MutationStream), + snapshotAcq: make(map[uint64]*snapshotAcquisition), + stopGate: stopGate, + state: config.SourceStateUnknown, + }, nil +} + +// Info reports the guarantees of this generation. SQLite observer positions +// are process-local and are not a durable LSN or resumable checkpoint. The +// SQL-owned snapshot stream provides an atomic snapshot/live handoff; writes +// made through a different unobserved database generation are not captured. +func (s *Source) Info() config.SourceInfo { + s.mu.RLock() + state := s.state + generation := s.generation + err := s.sourceErr + s.mu.RUnlock() + + info := config.SourceInfo{ + ID: s.id, + Kind: config.SQLite, + State: state, + Generation: generation, + Name: s.name, + Engine: "sqlite", + DBResource: s.dbResID.String(), + Tables: append([]string(nil), s.tables...), + Epoch: generation, + Capabilities: config.Capabilities{ + Snapshot: true, + Durable: false, + Replayable: false, + CapturesExternalWrites: false, + BeforeImages: true, + Coalesced: true, + }, + Snapshot: true, + Streaming: state == config.SourceStateRunning, + Faulted: state == config.SourceStateFaulted, + } + if err != nil { + info.Error = err.Error() + } + return info +} + +// LifecycleConfig lets the generic CDC manager register this source with the +// platform supervisor. The source itself does not emit lifecycle events. +func (s *Source) LifecycleConfig() supervisor.LifecycleConfig { return s.lifecycle } + +// Start subscribes to the SQL resource's observer and starts the forwarding +// loop. The resource borrow is released immediately after Subscribe succeeds; +// the SQL generation remains the owner of the observer and closes it when the +// database generation is replaced or stopped. +func (s *Source) Start(ctx context.Context) (<-chan any, error) { + if ctx == nil { + ctx = context.Background() + } + + s.mu.Lock() + if s.stopping { + s.mu.Unlock() + return nil, ErrSourceClosed + } + if s.state == config.SourceStateRunning { + status := s.status + s.mu.Unlock() + return status, nil + } + if s.state == config.SourceStateStarting { + s.mu.Unlock() + return nil, fmt.Errorf("%w: start already in progress", config.ErrSourceNotReady) + } + startCtx, startCancel := context.WithCancel(ctx) + startDone := make(chan struct{}) + status := make(chan any, 8) + s.state = config.SourceStateStarting + s.stopping = false + s.startCancel = startCancel + s.startDone = startDone + s.status = status + s.statusClosed = false + s.mu.Unlock() + defer close(startDone) + + observer, err := s.acquireObserver(startCtx) + if err != nil { + startCancel() + s.mu.Lock() + s.sourceErr = err + if !s.stopping { + s.state = config.SourceStateFaulted + } + s.startCancel = nil + if !s.stopping { + s.closeStatusLocked() + } + s.mu.Unlock() + return nil, err + } + + stream, err := observer.Subscribe(startCtx, sqlapi.MutationOptions{ + Tables: append([]string(nil), s.tables...), + }) + // Releasing the ordinary resource borrow is part of acquireObserver; the + // observer remains owned by the SQL resource generation. + if err != nil { + startCancel() + s.mu.Lock() + s.sourceErr = err + if !s.stopping { + s.state = config.SourceStateFaulted + } + s.startCancel = nil + if !s.stopping { + s.closeStatusLocked() + } + s.mu.Unlock() + return nil, fmt.Errorf("subscribe sqlite mutation observer: %w", err) + } + + runCtx, runCancel := context.WithCancel(startCtx) + runDone := make(chan struct{}) + + s.mu.Lock() + if s.stopping { + s.mu.Unlock() + runCancel() + _ = stream.Close() + startCancel() + return nil, ErrSourceClosed + } + s.observer = stream + s.observerSource = observer + s.sourceErr = nil + s.runCancel = runCancel + s.runDone = runDone + s.startCancel = nil + s.state = config.SourceStateRunning + s.mu.Unlock() + + select { + case status <- "sqlite cdc source started": + default: + } + go s.run(runCtx, stream, runDone) + return status, nil +} + +func (s *Source) acquireObserver(ctx context.Context) (sqlapi.CommittedMutationSource, error) { + if s.res == nil { + return nil, ErrResourceRegRequired + } + borrow, err := s.res.Acquire(ctx, s.dbResID, resource.ModeNormal) + if err != nil { + return nil, fmt.Errorf("acquire db resource: %w", err) + } + defer borrow.Release() + + value, err := borrow.Get() + if err != nil { + return nil, fmt.Errorf("get db resource: %w", err) + } + db, ok := value.(sqlservice.DBResource) + if !ok { + return nil, fmt.Errorf("resource %s is not a database", s.name) + } + if db.Type != sqlapi.SQLite { + return nil, fmt.Errorf("resource %s is not a sqlite database (kind %s)", s.name, db.Type) + } + if db.Observer == nil { + return nil, fmt.Errorf("resource %s does not expose committed mutation observation", s.name) + } + return db.Observer, nil +} + +func (s *Source) run(ctx context.Context, stream sqlapi.MutationStream, done chan struct{}) { + defer close(done) + defer func() { + s.mu.RLock() + running := s.state == config.SourceStateRunning && !s.stopping + s.mu.RUnlock() + if running { + s.fail(ErrSourceClosed) + } + }() + + collector := metrics.GetCollector(ctx) + ticker := time.NewTicker(s.statusTick) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + s.mu.RLock() + stopping := s.stopping || s.state == config.SourceStateStopped + s.mu.RUnlock() + if !stopping { + s.fail(ctx.Err()) + } + return + case <-ticker.C: + // The stream is deliberately passive. A status tick keeps the + // lifecycle channel alive without probing a second SQL connection. + case batch, ok := <-stream.Changes(): + if !ok { + err := stream.Err() + if err == nil { + err = ErrSourceClosed + } + s.fail(err) + return + } + if err := s.processBatch(batch, collector); err != nil { + s.fail(err) + return + } + } + } +} + +func (s *Source) processBatch(batch sqlapi.MutationBatch, collector metrics.Collector) error { + if batch.Transaction == "" { + return errors.New("sqlite mutation observer emitted a batch without a transaction identity") + } + for i, mutation := range batch.Changes { + change, err := s.changeFromMutation(batch, i, mutation) + if err != nil { + return err + } + s.subs.publish(change) + if collector != nil { + collector.CounterInc(changesCounter, metrics.Labels{"source": s.name, "op": change.Op}) + } + } + return nil +} + +func (s *Source) changeFromMutation(batch sqlapi.MutationBatch, index int, mutation sqlapi.Mutation) (config.Change, error) { + op := strings.ToLower(strings.TrimSpace(mutation.Op)) + if op != "insert" && op != "update" && op != "delete" && (!batch.Snapshot || op != "snapshot") { + return config.Change{}, fmt.Errorf("sqlite mutation observer emitted unsupported operation %q", mutation.Op) + } + if mutation.Table == "" { + return config.Change{}, errors.New("sqlite mutation observer emitted a mutation without a table") + } + if len(mutation.Columns) == 0 && (len(mutation.Before) != 0 || len(mutation.After) != 0) { + return config.Change{}, fmt.Errorf("sqlite mutation observer emitted %s.%s without captured columns", mutation.Schema, mutation.Table) + } + before, err := valuesByColumn(mutation.Columns, mutation.Before) + if err != nil { + return config.Change{}, fmt.Errorf("sqlite %s.%s before image: %w", mutation.Schema, mutation.Table, err) + } + after, err := valuesByColumn(mutation.Columns, mutation.After) + if err != nil { + return config.Change{}, fmt.Errorf("sqlite %s.%s after image: %w", mutation.Schema, mutation.Table, err) + } + + cursor := batch.Transaction + "/" + strconv.Itoa(index) + schema := normalizeSchema(mutation.Schema) + return config.Change{ + Before: before, + After: after, + Source: s.name, + SourceID: s.id, + Op: op, + Schema: schema, + Table: mutation.Table, + Relation: mutation.Table, + Cursor: cursor, + Transaction: batch.Transaction, + }, nil +} + +func valuesByColumn(columns []string, values []any) (map[string]any, error) { + if values == nil { + return nil, nil + } + if len(columns) != len(values) { + return nil, fmt.Errorf("captured %d values for %d columns", len(values), len(columns)) + } + out := make(map[string]any, len(values)) + for i, column := range columns { + column = strings.TrimSpace(column) + if column == "" { + return nil, fmt.Errorf("captured column %d has an empty name", i) + } + if _, exists := out[column]; exists { + return nil, fmt.Errorf("captured duplicate column %q", column) + } + out[column] = cloneValue(values[i]) + } + return out, nil +} + +func cloneValue(value any) any { + bytes, ok := value.([]byte) + if !ok { + return value + } + return append([]byte(nil), bytes...) +} + +func normalizeSchema(schema string) string { + schema = strings.TrimSpace(schema) + if schema == "" { + return "main" + } + return schema +} + +func (s *Source) fail(err error) { + if err == nil { + err = ErrSourceClosed + } + s.mu.Lock() + if s.state == config.SourceStateStopped || s.stopping { + s.mu.Unlock() + return + } + if s.state == config.SourceStateFaulted { + s.mu.Unlock() + return + } + s.state = config.SourceStateFaulted + s.sourceErr = err + stream := s.observer + s.observer = nil + s.observerSource = nil + snapshotSubscriptions := make([]*subscription, 0, len(s.snapshotSubs)) + snapshotSubs := make([]sqlapi.MutationStream, 0, len(s.snapshotSubs)) + snapshotCancels := s.snapshotAcquisitionCancelsLocked() + for sub, snapshotStream := range s.snapshotSubs { + snapshotSubscriptions = append(snapshotSubscriptions, sub) + snapshotSubs = append(snapshotSubs, snapshotStream) + delete(s.snapshotSubs, sub) + } + s.closeStatusLocked() + s.mu.Unlock() + + s.subs.closeWithError(err) + for _, cancel := range snapshotCancels { + cancel() + } + for _, sub := range snapshotSubscriptions { + sub.closeWithError(err) + sub.waitRelay() + } + for _, snapshotStream := range snapshotSubs { + _ = snapshotStream.Close() + } + if stream != nil { + _ = stream.Close() + } + s.log.Error("sqlite cdc source faulted", zap.String("source", s.name), zap.Error(err)) +} + +func (s *Source) closeStatusLocked() { + if s.status != nil && !s.statusClosed { + close(s.status) + s.statusClosed = true + } +} + +// Stop closes the observer stream and all subscriptions. It never closes the +// DB observer itself: that belongs to the SQL resource generation. +func (s *Source) Stop(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + // Serialize cleanup attempts. A timed-out attempt leaves the source in its + // pre-stop state with stopping set, so a later caller must be able to retry + // the same cleanup without racing the first attempt or observing a false + // successful stop. + if err := s.acquireStop(ctx); err != nil { + return err + } + defer s.releaseStop() + + s.mu.Lock() + if s.state == config.SourceStateStopped && !s.stopping { + s.mu.Unlock() + return nil + } + s.stopping = true + startCancel := s.startCancel + runCancel := s.runCancel + startDone := s.startDone + runDone := s.runDone + stream := s.observer + snapshotStreams := make([]sqlapi.MutationStream, 0, len(s.snapshotSubs)) + snapshotSubscriptions := make([]*subscription, 0, len(s.snapshotSubs)) + snapshotCancels := s.snapshotAcquisitionCancelsLocked() + for sub, snapshotStream := range s.snapshotSubs { + snapshotSubscriptions = append(snapshotSubscriptions, sub) + snapshotStreams = append(snapshotStreams, snapshotStream) + delete(s.snapshotSubs, sub) + } + s.mu.Unlock() + + if startCancel != nil { + startCancel() + } + if runCancel != nil { + runCancel() + } + if stream != nil { + _ = stream.Close() + } + for _, cancel := range snapshotCancels { + cancel() + } + for _, sub := range snapshotSubscriptions { + sub.closeWithError(nil) + sub.waitRelay() + } + for _, snapshotStream := range snapshotStreams { + _ = snapshotStream.Close() + } + + s.mu.Lock() + snapshotDone := s.snapshotWait + if snapshotDone == nil { + snapshotDone = make(chan struct{}) + s.snapshotWait = snapshotDone + go func() { + s.snapshotWG.Wait() + close(snapshotDone) + }() + } + s.mu.Unlock() + + stopTimeout := cleanupTimeout + if s.lifecycle.StopTimeout > 0 { + stopTimeout = s.lifecycle.StopTimeout + } + waitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), stopTimeout) + defer cancel() + if err := waitDone(waitCtx, startDone); err != nil { + return s.stopFailed(err) + } + if err := waitDone(waitCtx, runDone); err != nil { + return s.stopFailed(err) + } + if err := waitDone(waitCtx, snapshotDone); err != nil { + return s.stopFailed(err) + } + + s.subs.closeAll() + s.mu.Lock() + s.state = config.SourceStateStopped + s.stopping = false + s.observer = nil + s.observerSource = nil + s.startCancel = nil + s.runCancel = nil + s.snapshotWait = nil + s.closeStatusLocked() + s.mu.Unlock() + return nil +} + +func (s *Source) acquireStop(ctx context.Context) error { + if s.stopGate == nil { + return nil + } + select { + case <-s.stopGate: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *Source) releaseStop() { + if s.stopGate != nil { + s.stopGate <- struct{}{} + } +} + +func (s *Source) stopFailed(err error) error { + s.subs.closeWithError(err) + s.mu.Lock() + if s.state != config.SourceStateStopped { + s.state = config.SourceStateFaulted + s.sourceErr = err + } + s.mu.Unlock() + return err +} + +func waitDone(ctx context.Context, done <-chan struct{}) error { + if done == nil { + return nil + } + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *Source) beginSnapshotAcquisition(ctx context.Context, observer sqlapi.CommittedMutationSource) (context.Context, uint64, error) { + acquisitionCtx, cancel := context.WithCancel(ctx) + s.mu.Lock() + if s.state != config.SourceStateRunning || s.stopping || s.observerSource != observer { + s.mu.Unlock() + cancel() + return nil, 0, config.ErrSourceNotReady + } + s.nextSnapshotID++ + id := s.nextSnapshotID + s.snapshotAcq[id] = &snapshotAcquisition{cancel: cancel} + s.snapshotWG.Add(1) + s.mu.Unlock() + return acquisitionCtx, id, nil +} + +func (s *Source) finishSnapshotAcquisition(id uint64) { + s.mu.Lock() + acquisition, ok := s.snapshotAcq[id] + if ok { + delete(s.snapshotAcq, id) + } + s.mu.Unlock() + if !ok { + return + } + acquisition.cancel() + s.snapshotWG.Done() +} + +func (s *Source) snapshotAcquisitionCancelsLocked() []context.CancelFunc { + cancels := make([]context.CancelFunc, 0, len(s.snapshotAcq)) + for _, acquisition := range s.snapshotAcq { + cancels = append(cancels, acquisition.cancel) + } + return cancels +} + +// Subscribe exposes committed changes. Cursor resume remains unsupported +// because this process-local generation has no durable checkpoint; snapshots +// use the SQL-owned atomic fence and handoff stream per subscriber. +func (s *Source) Subscribe(ctx context.Context, opts config.StreamOptions) (config.Stream, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + if err := opts.Validate(); err != nil { + return nil, err + } + if opts.After != "" { + return nil, config.ErrUnsupported + } + + s.mu.RLock() + if s.state != config.SourceStateRunning || s.stopping { + s.mu.RUnlock() + return nil, config.ErrSourceNotReady + } + if opts.Snapshot || s.snapshot { + observer := s.observerSource + s.mu.RUnlock() + return s.subscribeSnapshot(ctx, observer, opts) + } + // Hold the source read lock while registering the subscription. Stop takes + // the write lock before closing the current subscriber set, so a stream + // cannot be inserted after lifecycle cleanup has already passed. + sub := s.subs.subscribe(s.name, opts) + s.mu.RUnlock() + return sub, nil +} + +func (s *Source) subscribeSnapshot(ctx context.Context, observer sqlapi.CommittedMutationSource, opts config.StreamOptions) (config.Stream, error) { + if observer == nil { + return nil, fmt.Errorf("%w: sqlite snapshot observer is unavailable", config.ErrUnsupported) + } + tables, noTables := intersectTables(s.tables, opts.Tables) + buffer := opts.Buffer + if buffer <= 0 { + buffer = defaultStreamBuffer + } + if buffer > maxStreamBuffer { + buffer = maxStreamBuffer + } + if noTables { + // An empty table intersection means no rows can match. Passing an empty + // list to the SQL observer would mean "all tables", so return an + // already-complete stream instead. + sub := newSubscription(s.name, opts, buffer) + sub.Close() + return sub, nil + } + + acquisitionCtx, acquisitionID, err := s.beginSnapshotAcquisition(ctx, observer) + if err != nil { + return nil, err + } + stream, err := observer.Snapshot(acquisitionCtx, sqlapi.SnapshotOptions{ + Tables: tables, + }) + if err != nil { + s.finishSnapshotAcquisition(acquisitionID) + return nil, err + } + if stream == nil { + s.finishSnapshotAcquisition(acquisitionID) + return nil, errors.New("sqlite snapshot observer returned a nil stream") + } + + sub := newSubscription(s.name, opts, buffer) + + s.mu.Lock() + if s.state != config.SourceStateRunning || s.stopping || s.observerSource != observer { + s.mu.Unlock() + _ = stream.Close() + s.finishSnapshotAcquisition(acquisitionID) + return nil, config.ErrSourceNotReady + } + if _, ok := s.snapshotAcq[acquisitionID]; !ok { + s.mu.Unlock() + _ = stream.Close() + s.finishSnapshotAcquisition(acquisitionID) + return nil, config.ErrSourceNotReady + } + s.snapshotSubs[sub] = stream + s.snapshotWG.Add(1) + s.mu.Unlock() + + go s.runSnapshot(acquisitionCtx, stream, sub, acquisitionID) + return sub, nil +} + +func (s *Source) runSnapshot(ctx context.Context, stream sqlapi.SnapshotStream, sub *subscription, acquisitionID uint64) { + defer s.snapshotWG.Done() + defer sub.waitRelay() + defer func() { + s.mu.Lock() + delete(s.snapshotSubs, sub) + s.mu.Unlock() + }() + defer s.finishSnapshotAcquisition(acquisitionID) + // The SQL observer owns the snapshot read transaction and scan worker. A + // subscriber can end for any reason (upstream error, downstream overflow, + // cancellation, or normal close), so every return path must release that + // upstream stream. Close is idempotent for the SQL observer stream. + defer func() { _ = stream.Close() }() + + watermark := stream.Watermark() + for { + select { + case <-ctx.Done(): + sub.closeWithError(ctx.Err()) + return + case <-sub.done: + return + case batch, ok := <-stream.Changes(): + if !ok { + err := stream.Err() + sub.closeWithError(err) + return + } + for i, mutation := range batch.Changes { + change, err := s.changeFromMutation(batch, i, mutation) + if err != nil { + sub.closeWithError(err) + return + } + change.Cursor = snapshotCursor(watermark, batch.Transaction, i) + if batch.Snapshot { + change.Op = "snapshot" + } + if batch.Snapshot { + if !sub.matchesSnapshot(change) { + continue + } + } else if !sub.matches(change) { + continue + } + sub.send(change, config.EstimateChangeBytes(change)) + if sub.isClosed() { + return + } + } + } + } +} + +func snapshotCursor(watermark, transaction string, index int) string { + parts := make([]string, 0, 3) + if watermark != "" { + parts = append(parts, watermark) + } + if transaction != "" { + parts = append(parts, transaction) + } + parts = append(parts, strconv.Itoa(index)) + return strings.Join(parts, "/") +} + +func intersectTables(source, requested []string) ([]string, bool) { + if len(source) == 0 { + return append([]string(nil), requested...), false + } + if len(requested) == 0 { + return append([]string(nil), source...), false + } + matched := make([]string, 0, len(requested)) + for _, want := range requested { + for _, allowed := range source { + if tableNamesEqual(want, allowed) { + matched = append(matched, want) + break + } + } + } + return matched, len(matched) == 0 +} + +func tableNamesEqual(left, right string) bool { + left = strings.ToLower(strings.TrimSpace(left)) + right = strings.ToLower(strings.TrimSpace(right)) + if left == right { + return true + } + left = tableNameOnly(left) + right = tableNameOnly(right) + return left != "" && left == right +} + +func tableNameOnly(value string) string { + if index := strings.LastIndexByte(value, '.'); index >= 0 { + return value[index+1:] + } + return value +} + +var _ config.Source = (*Source)(nil) +var _ interface { + Start(context.Context) (<-chan any, error) + Stop(context.Context) error +} = (*Source)(nil) diff --git a/service/cdc/sqlite/source_stub.go b/service/cdc/sqlite/source_stub.go new file mode 100644 index 000000000..a49e9b428 --- /dev/null +++ b/service/cdc/sqlite/source_stub.go @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build !sqlite_preupdate_hook + +package sqlite + +func buildSource(_ sourceOptions) (managedSource, error) { + return nil, ErrPreupdateTagRequired +} diff --git a/service/cdc/sqlite/source_stub_test.go b/service/cdc/sqlite/source_stub_test.go new file mode 100644 index 000000000..086d7859c --- /dev/null +++ b/service/cdc/sqlite/source_stub_test.go @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build !sqlite_preupdate_hook + +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildSourceRequiresTag(t *testing.T) { + src, err := buildSource(sourceOptions{name: "x"}) + assert.Nil(t, src) + assert.ErrorIs(t, err, ErrPreupdateTagRequired) +} diff --git a/service/cdc/sqlite/source_tagged_test.go b/service/cdc/sqlite/source_tagged_test.go new file mode 100644 index 000000000..24b8d2810 --- /dev/null +++ b/service/cdc/sqlite/source_tagged_test.go @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/wippyai/runtime/api/registry" +) + +func TestBuildSourceWithTag(t *testing.T) { + h, err := buildSource(sourceOptions{name: "x", dbResource: registry.NewID("app", "db")}) + require.NoError(t, err) + require.NotNil(t, h) + _, ok := h.(*Source) + assert.True(t, ok) +} + +func TestBuildSourceRejectsBadInterval(t *testing.T) { + _, err := buildSource(sourceOptions{name: "x", statusInterval: "nope"}) + assert.Error(t, err) +} + +func TestBuildSourceRetainsSnapshotPolicyForPerSubscriberHandoff(t *testing.T) { + h, err := buildSource(sourceOptions{name: "x", snapshot: true}) + require.NoError(t, err) + assert.True(t, h.(*Source).snapshot) +} diff --git a/service/cdc/sqlite/source_test.go b/service/cdc/sqlite/source_test.go new file mode 100644 index 000000000..5fc4dd3dd --- /dev/null +++ b/service/cdc/sqlite/source_test.go @@ -0,0 +1,746 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/resource" + cdcapi "github.com/wippyai/runtime/api/service/cdc" + sqlapi "github.com/wippyai/runtime/api/service/sql" + "github.com/wippyai/runtime/api/supervisor" + sqlservice "github.com/wippyai/runtime/service/sql" +) + +type testResourceRegistry struct { + observer sqlapi.CommittedMutationSource + releases atomic.Int32 +} + +func (r *testResourceRegistry) Acquire(context.Context, registry.ID, resource.AccessMode) (resource.Resource[any], error) { + return &testDBResource{owner: r, value: sqlservice.DBResource{ + Type: sqlapi.SQLite, + Observer: r.observer, + }}, nil +} + +func (*testResourceRegistry) List() ([]registry.ID, error) { return nil, nil } +func (*testResourceRegistry) Exists(registry.ID) bool { return true } + +type testDBResource struct { + owner *testResourceRegistry + value sqlservice.DBResource + once sync.Once +} + +func (r *testDBResource) Get() (any, error) { return r.value, nil } +func (r *testDBResource) Release() { + r.once.Do(func() { r.owner.releases.Add(1) }) +} + +type testObserver struct { + stream *testMutationStream + snapshot *testSnapshotStream + snapshotStarted chan struct{} + subOpts sqlapi.MutationOptions + snapshotCancelDelay time.Duration + mu sync.Mutex + closeN atomic.Int32 + closed bool +} + +func (o *testObserver) Subscribe(ctx context.Context, opts sqlapi.MutationOptions) (sqlapi.MutationStream, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + o.mu.Lock() + defer o.mu.Unlock() + if o.closed { + return nil, errors.New("test observer closed") + } + o.subOpts = opts + o.stream = newTestMutationStream() + return o.stream, nil +} + +func (o *testObserver) Snapshot(ctx context.Context, _ sqlapi.SnapshotOptions) (sqlapi.SnapshotStream, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if o.snapshotStarted != nil { + select { + case o.snapshotStarted <- struct{}{}: + default: + } + <-ctx.Done() + if o.snapshotCancelDelay > 0 { + time.Sleep(o.snapshotCancelDelay) + } + return nil, ctx.Err() + } + o.mu.Lock() + defer o.mu.Unlock() + if o.closed { + return nil, errors.New("test observer closed") + } + o.snapshot = &testSnapshotStream{ + testMutationStream: newTestMutationStream(), + watermark: "watermark-1", + } + return o.snapshot, nil +} + +func (o *testObserver) Close() error { + o.mu.Lock() + if o.closed { + o.mu.Unlock() + return nil + } + o.closed = true + stream := o.stream + snapshot := o.snapshot + o.mu.Unlock() + o.closeN.Add(1) + if stream != nil { + stream.closeWithError(errors.New("test SQL generation closed")) + } + if snapshot != nil { + snapshot.closeWithError(errors.New("test SQL generation closed")) + } + return nil +} + +func (o *testObserver) currentStream(t *testing.T) *testMutationStream { + t.Helper() + o.mu.Lock() + stream := o.stream + o.mu.Unlock() + require.NotNil(t, stream) + return stream +} + +func (o *testObserver) currentSnapshot(t *testing.T) *testSnapshotStream { + t.Helper() + o.mu.Lock() + stream := o.snapshot + o.mu.Unlock() + require.NotNil(t, stream) + return stream +} + +type testMutationStream struct { + err error + changes chan sqlapi.MutationBatch + mu sync.Mutex + closeN atomic.Int32 + closed bool +} + +type testSnapshotStream struct { + *testMutationStream + watermark string +} + +func (s *testSnapshotStream) Watermark() string { return s.watermark } + +func newTestMutationStream() *testMutationStream { + return &testMutationStream{changes: make(chan sqlapi.MutationBatch, 16)} +} + +func (s *testMutationStream) Changes() <-chan sqlapi.MutationBatch { return s.changes } + +func (s *testMutationStream) Err() error { + s.mu.Lock() + err := s.err + s.mu.Unlock() + return err +} + +func (s *testMutationStream) Close() error { + s.closeWithError(nil) + return nil +} + +func (s *testMutationStream) closeWithError(err error) { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + s.closed = true + s.err = err + close(s.changes) + s.mu.Unlock() + s.closeN.Add(1) +} + +func (s *testMutationStream) push(batch sqlapi.MutationBatch) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.closed { + s.changes <- batch + } +} + +func newTestSource(t *testing.T, observer *testObserver, opts sourceOptions) *Source { + t.Helper() + resources := &testResourceRegistry{observer: observer} + if opts.res == nil { + opts.res = resources + } + if opts.id.Name == "" { + opts.id = registry.NewID("app", "cdc") + } + if opts.name == "" { + opts.name = opts.id.String() + } + source, err := buildSource(opts) + require.NoError(t, err) + return source.(*Source) +} + +func receiveChange(t *testing.T, stream cdcapi.Stream) cdcapi.Change { + t.Helper() + select { + case change, ok := <-stream.Changes(): + if !ok { + require.Failf(t, "snapshot/live stream closed", "stream error: %v", stream.Err()) + } + return change + case <-time.After(time.Second): + t.Fatal("timed out waiting for SQLite CDC change") + return cdcapi.Change{} + } +} + +func requireNoChangeSource(t *testing.T, stream cdcapi.Stream) { + t.Helper() + select { + case change, ok := <-stream.Changes(): + if !ok { + t.Fatalf("stream closed while expecting no change") + } + t.Fatalf("unexpected change: %#v", change) + case <-time.After(50 * time.Millisecond): + } +} + +func waitStreamClosed(t *testing.T, stream cdcapi.Stream) error { + t.Helper() + deadline := time.After(time.Second) + for { + select { + case _, ok := <-stream.Changes(): + if !ok { + return stream.Err() + } + case <-deadline: + t.Fatal("timed out waiting for SQLite CDC stream close") + return nil + } + } +} + +func TestSourceForwardsCommittedMutationWithStableShape(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{tables: []string{"users"}}) + status, err := source.Start(context.Background()) + require.NoError(t, err) + require.Equal(t, "sqlite cdc source started", <-status) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{Ops: []string{"update"}}) + require.NoError(t, err) + defer stream.Close() + + assert.Equal(t, []string{"users"}, observer.subOpts.Tables) + observer.currentStream(t).push(sqlapi.MutationBatch{ + Transaction: "17", + Changes: []sqlapi.Mutation{{ + Schema: "main", + Table: "users", + Columns: []string{"id", "name"}, + Before: []any{int64(1), []byte("old")}, + After: []any{int64(1), []byte("new")}, + Op: "update", + }}, + }) + + change := receiveChange(t, stream) + assert.Equal(t, source.name, change.Source) + assert.Equal(t, source.id, change.SourceID) + assert.Equal(t, "update", change.Op) + assert.Equal(t, "main", change.Schema) + assert.Equal(t, "users", change.Relation) + assert.Equal(t, "17/0", change.Cursor) + assert.Equal(t, "17", change.Transaction) + assert.Equal(t, int64(1), change.After["id"]) + assert.Equal(t, []byte("old"), change.Before["name"]) + assert.Equal(t, []byte("new"), change.After["name"]) + + info := source.Info() + assert.True(t, info.Capabilities.Snapshot, "a tagged SQL observer exposes the snapshot handoff capability") + assert.False(t, info.Capabilities.Durable) + assert.False(t, info.Capabilities.Replayable) + assert.False(t, info.Capabilities.CapturesExternalWrites) + assert.True(t, info.Capabilities.BeforeImages) +} + +func TestSourceRejectsResumeButSupportsSnapshotHandoff(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + _, err = source.Subscribe(context.Background(), cdcapi.StreamOptions{After: "17/0"}) + assert.ErrorIs(t, err, cdcapi.ErrUnsupported) + snapshot, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + require.NoError(t, err) + snapshot.Close() +} + +func TestSourceSnapshotSubscriberOwnsAtomicHandoff(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + live, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + defer live.Close() + snapshot, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{ + Snapshot: true, + }) + require.NoError(t, err) + defer snapshot.Close() + + observer.currentSnapshot(t).push(sqlapi.MutationBatch{ + Transaction: "snapshot-1", + Snapshot: true, + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "users", Columns: []string{"id", "name"}, + After: []any{int64(1), "existing"}, Op: "insert", + }}, + }) + observer.currentSnapshot(t).push(sqlapi.MutationBatch{ + Transaction: "live-1", + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "users", Columns: []string{"id", "name"}, + After: []any{int64(2), "new"}, Op: "insert", + }}, + }) + + snapshotChange := receiveChange(t, snapshot) + assert.Equal(t, "snapshot", snapshotChange.Op) + assert.Equal(t, "watermark-1/snapshot-1/0", snapshotChange.Cursor) + assert.Equal(t, "existing", snapshotChange.After["name"]) + liveChange := receiveChange(t, snapshot) + assert.Equal(t, "insert", liveChange.Op) + assert.Equal(t, "new", liveChange.After["name"]) + requireNoChangeSource(t, live) +} + +func TestSourceDefaultSnapshotAppliesPerSubscriber(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{snapshot: true}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + observer.currentSnapshot(t).push(sqlapi.MutationBatch{ + Transaction: "snapshot-default", + Snapshot: true, + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "users", Columns: []string{"id"}, After: []any{int64(7)}, Op: "insert", + }}, + }) + assert.Equal(t, "snapshot", receiveChange(t, stream).Op) +} + +func TestSourceSnapshotClosesUpstreamWhenItEnds(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + require.NoError(t, err) + upstream := observer.currentSnapshot(t) + expected := errors.New("snapshot worker failed") + upstream.closeWithError(expected) + + assert.ErrorIs(t, waitStreamClosed(t, stream), expected) + assert.Eventually(t, func() bool { return upstream.closeN.Load() == 1 }, time.Second, time.Millisecond) +} + +func TestSourceSnapshotOverflowClosesUpstream(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{ + Snapshot: true, + Buffer: 1, + }) + require.NoError(t, err) + upstream := observer.currentSnapshot(t) + change := func(id int64) sqlapi.MutationBatch { + return sqlapi.MutationBatch{ + Transaction: "snapshot", + Snapshot: true, + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "users", Columns: []string{"id"}, + After: []any{id}, Op: "insert", + }}, + } + } + upstream.push(change(1)) + subscriber := stream.(*subscription) + assert.Eventually(t, func() bool { + subscriber.mu.Lock() + defer subscriber.mu.Unlock() + return len(subscriber.queue) == 1 + }, time.Second, time.Millisecond) + upstream.push(change(2)) + + assert.Eventually(t, func() bool { return upstream.closeN.Load() == 1 }, time.Second, time.Millisecond) + assert.ErrorIs(t, subscriber.Err(), errSubscriberOverflow) + stream.Close() +} + +func TestSourceStopWaitsForBlockedSnapshotAcquisition(t *testing.T) { + observer := &testObserver{ + snapshotStarted: make(chan struct{}, 1), + snapshotCancelDelay: 50 * time.Millisecond, + } + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + + subscribeDone := make(chan error, 1) + go func() { + _, subscribeErr := source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + subscribeDone <- subscribeErr + }() + select { + case <-observer.snapshotStarted: + case <-time.After(time.Second): + t.Fatal("snapshot acquisition did not start") + } + + stopDone := make(chan error, 1) + go func() { stopDone <- source.Stop(context.Background()) }() + select { + case err := <-stopDone: + t.Fatalf("Stop returned before the blocked snapshot acquisition ended: %v", err) + case <-time.After(20 * time.Millisecond): + } + require.NoError(t, <-stopDone) + assert.Error(t, <-subscribeDone) + + source.mu.RLock() + assert.Empty(t, source.snapshotAcq) + assert.Empty(t, source.snapshotSubs) + assert.Equal(t, cdcapi.SourceStateStopped, source.state) + source.mu.RUnlock() +} + +func TestSourceStopTimeoutIsRetryableAndRestartable(t *testing.T) { + observer := &testObserver{ + snapshotStarted: make(chan struct{}, 1), + snapshotCancelDelay: 50 * time.Millisecond, + } + source := newTestSource(t, observer, sourceOptions{ + lifecycle: supervisor.LifecycleConfig{StopTimeout: 10 * time.Millisecond}, + }) + _, err := source.Start(context.Background()) + require.NoError(t, err) + + subscribeDone := make(chan error, 1) + go func() { + _, subscribeErr := source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + subscribeDone <- subscribeErr + }() + select { + case <-observer.snapshotStarted: + case <-time.After(time.Second): + t.Fatal("snapshot acquisition did not start") + } + + err = source.Stop(context.Background()) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.ErrorIs(t, func() error { + _, startErr := source.Start(context.Background()) + return startErr + }(), ErrSourceClosed) + source.mu.RLock() + assert.Equal(t, cdcapi.SourceStateFaulted, source.state, "a timed-out stop reports failure without publishing Stopped") + assert.True(t, source.stopping, "a timed-out stop remains retryable") + source.mu.RUnlock() + + assert.Error(t, <-subscribeDone) + require.NoError(t, source.Stop(context.Background())) + assert.Equal(t, cdcapi.SourceStateStopped, source.Info().State) + + _, err = source.Start(context.Background()) + require.NoError(t, err) + require.NoError(t, source.Stop(context.Background())) +} + +func TestSourceFaultCancelsBlockedSnapshotAcquisition(t *testing.T) { + observer := &testObserver{snapshotStarted: make(chan struct{}, 1)} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + + subscribeDone := make(chan error, 1) + go func() { + _, subscribeErr := source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + subscribeDone <- subscribeErr + }() + select { + case <-observer.snapshotStarted: + case <-time.After(time.Second): + t.Fatal("snapshot acquisition did not start") + } + + observer.currentStream(t).closeWithError(errors.New("observer generation failed")) + assert.Error(t, <-subscribeDone) + assert.Eventually(t, func() bool { + return source.Info().State == cdcapi.SourceStateFaulted + }, time.Second, time.Millisecond) + require.NoError(t, source.Stop(context.Background())) +} + +func TestSourceLifecycleIsolationAcrossResources(t *testing.T) { + firstObserver := &testObserver{ + snapshotStarted: make(chan struct{}, 1), + snapshotCancelDelay: 50 * time.Millisecond, + } + first := newTestSource(t, firstObserver, sourceOptions{ + id: registry.NewID("app", "cdc-first"), + res: &testResourceRegistry{observer: firstObserver}, + lifecycle: supervisor.LifecycleConfig{StopTimeout: 10 * time.Millisecond}, + }) + secondObserver := &testObserver{} + second := newTestSource(t, secondObserver, sourceOptions{ + id: registry.NewID("app", "cdc-second"), + res: &testResourceRegistry{observer: secondObserver}, + }) + _, err := first.Start(context.Background()) + require.NoError(t, err) + _, err = second.Start(context.Background()) + require.NoError(t, err) + defer func() { + _ = first.Stop(context.Background()) + _ = second.Stop(context.Background()) + }() + + _, err = first.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + secondLive, err := second.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + + snapshotDone := make(chan error, 1) + go func() { + _, snapshotErr := first.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + snapshotDone <- snapshotErr + }() + select { + case <-firstObserver.snapshotStarted: + case <-time.After(time.Second): + t.Fatal("first snapshot acquisition did not start") + } + + assert.ErrorIs(t, first.Stop(context.Background()), context.DeadlineExceeded) + secondObserver.currentStream(t).push(sqlapi.MutationBatch{ + Transaction: "second-while-first-stopping", + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "t", Columns: []string{"id"}, + After: []any{int64(2)}, Op: "insert", + }}, + }) + assert.Equal(t, int64(2), receiveChange(t, secondLive).After["id"]) + assert.Equal(t, cdcapi.SourceStateRunning, second.Info().State) + + assert.Error(t, <-snapshotDone) + require.NoError(t, first.Stop(context.Background())) + assert.Equal(t, cdcapi.SourceStateStopped, first.Info().State) + assert.Equal(t, cdcapi.SourceStateRunning, second.Info().State) + assert.Equal(t, int32(0), secondObserver.closeN.Load(), "stopping one source must not close another SQL observer") + + _, err = first.Start(context.Background()) + require.NoError(t, err) + firstLive, err := first.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + firstObserver.currentStream(t).push(sqlapi.MutationBatch{ + Transaction: "first-after-restart", + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "t", Columns: []string{"id"}, + After: []any{int64(1)}, Op: "insert", + }}, + }) + assert.Equal(t, int64(1), receiveChange(t, firstLive).After["id"]) + assert.Equal(t, cdcapi.SourceStateRunning, second.Info().State) +} + +func TestSourceStopsWithoutClosingSQLGeneration(t *testing.T) { + observer := &testObserver{} + resources := &testResourceRegistry{observer: observer} + source := newTestSource(t, observer, sourceOptions{res: resources}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + assert.Equal(t, int32(1), resources.releases.Load(), "the ordinary resource borrow ends after Subscribe") + + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + snapshot, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + require.NoError(t, err) + require.NoError(t, source.Stop(context.Background())) + assert.Equal(t, int32(0), observer.closeN.Load(), "CDC does not own the SQL observer") + assert.NoError(t, waitStreamClosed(t, stream)) + assert.NoError(t, waitStreamClosed(t, snapshot)) +} + +func TestSourceCanRestartAfterStop(t *testing.T) { + observer := &testObserver{} + resources := &testResourceRegistry{observer: observer} + source := newTestSource(t, observer, sourceOptions{res: resources}) + + _, err := source.Start(context.Background()) + require.NoError(t, err) + first, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + require.NoError(t, source.Stop(context.Background())) + assert.Equal(t, cdcapi.SourceStateStopped, source.Info().State) + assert.NoError(t, waitStreamClosed(t, first)) + + _, err = source.Start(context.Background()) + require.NoError(t, err, "a supervisor stop is restartable; only resource-generation disposal is terminal") + second, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + observer.currentStream(t).push(sqlapi.MutationBatch{ + Transaction: "2", + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "t", Columns: []string{"id"}, After: []any{int64(2)}, Op: "insert", + }}, + }) + assert.Equal(t, int64(2), receiveChange(t, second).After["id"]) + require.NoError(t, source.Stop(context.Background())) + assert.Equal(t, int32(2), resources.releases.Load()) +} + +func TestSourceFaultsWhenSQLGenerationCloses(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + + require.NoError(t, observer.Close()) + assert.Error(t, waitStreamClosed(t, stream)) + assert.Equal(t, cdcapi.SourceStateFaulted, source.Info().State) + assert.NotEmpty(t, source.Info().Error) + assert.NoError(t, source.Stop(context.Background())) +} + +func TestSourceContextCancellationClosesSubscribers(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + ctx, cancel := context.WithCancel(context.Background()) + _, err := source.Start(ctx) + require.NoError(t, err) + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + + cancel() + assert.ErrorIs(t, waitStreamClosed(t, stream), context.Canceled) + assert.Equal(t, cdcapi.SourceStateFaulted, source.Info().State) + require.NoError(t, source.Stop(context.Background())) +} + +func TestSourceCanRestartAgainstReplacementSQLGeneration(t *testing.T) { + firstObserver := &testObserver{} + resources := &testResourceRegistry{observer: firstObserver} + source := newTestSource(t, firstObserver, sourceOptions{res: resources}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + first, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + require.NoError(t, firstObserver.Close()) + assert.Error(t, waitStreamClosed(t, first)) + require.NoError(t, source.Stop(context.Background())) + + secondObserver := &testObserver{} + resources.observer = secondObserver + _, err = source.Start(context.Background()) + require.NoError(t, err) + second, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + secondObserver.currentStream(t).push(sqlapi.MutationBatch{ + Transaction: "replacement-1", + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "t", Columns: []string{"id"}, After: []any{int64(9)}, Op: "insert", + }}, + }) + assert.Equal(t, int64(9), receiveChange(t, second).After["id"]) + require.NoError(t, source.Stop(context.Background())) +} + +func TestSourceClosesOnlyOverflowedSubscriber(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + laggard, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{Buffer: 1}) + require.NoError(t, err) + reader, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{Buffer: 8}) + require.NoError(t, err) + + batch := sqlapi.MutationBatch{Transaction: "1", Changes: []sqlapi.Mutation{ + {Schema: "main", Table: "t", Columns: []string{"id"}, After: []any{int64(1)}, Op: "insert"}, + {Schema: "main", Table: "t", Columns: []string{"id"}, After: []any{int64(2)}, Op: "insert"}, + }} + observer.currentStream(t).push(batch) + + assert.Equal(t, "insert", receiveChange(t, reader).Op) + assert.ErrorIs(t, waitStreamClosed(t, laggard), errSubscriberOverflow) + assert.Equal(t, "insert", receiveChange(t, reader).Op) +} + +func TestSourceMalformedMutationFaultsClosedStream(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + + observer.currentStream(t).push(sqlapi.MutationBatch{Transaction: "1", Changes: []sqlapi.Mutation{{ + Table: "t", Columns: nil, After: []any{int64(1)}, Op: "insert", + }}}) + assert.Error(t, waitStreamClosed(t, stream)) + assert.Equal(t, cdcapi.SourceStateFaulted, source.Info().State) + assert.NoError(t, source.Stop(context.Background())) +} diff --git a/service/cdc/sqlite/subscribers.go b/service/cdc/sqlite/subscribers.go new file mode 100644 index 000000000..aa85b6780 --- /dev/null +++ b/service/cdc/sqlite/subscribers.go @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "errors" + "strings" + "sync" + + config "github.com/wippyai/runtime/api/service/cdc" +) + +const ( + defaultStreamBuffer = 128 + maxStreamBuffer = 65536 +) + +var errSubscriberOverflow = errors.New("sqlite cdc subscriber backlog overflow") + +type subscribers struct { + m map[uint64]*subscription + next uint64 + mu sync.RWMutex +} + +func newSubscribers() *subscribers { + return &subscribers{m: make(map[uint64]*subscription)} +} + +func (s *subscribers) subscribe(sourceName string, opts config.StreamOptions) *subscription { + buffer := opts.Buffer + if buffer <= 0 { + buffer = defaultStreamBuffer + } + if buffer > maxStreamBuffer { + buffer = maxStreamBuffer + } + + s.mu.Lock() + s.next++ + sub := newSubscription(sourceName, opts, buffer) + sub.parent = s + sub.id = s.next + s.m[sub.id] = sub + s.mu.Unlock() + return sub +} + +func newSubscription(sourceName string, opts config.StreamOptions, buffer int) *subscription { + sub := &subscription{ + sourceName: sourceName, + // queue is the sole driver-owned backlog. changes is an unbuffered + // delivery handoff, so bytes leave the budget only after a receive. + changes: make(chan config.Change), + done: make(chan struct{}), + notify: make(chan struct{}, 1), + maxChanges: buffer, + maxBytes: opts.EffectiveMaxBytes(), + tables: filterSet(opts.Tables), + ops: filterSet(opts.Ops), + relayDone: make(chan struct{}), + } + go sub.run() + return sub +} + +func (s *subscribers) publish(change config.Change) { + s.mu.RLock() + matched := make([]*subscription, 0, len(s.m)) + for _, sub := range s.m { + if sub.matches(change) { + matched = append(matched, sub) + } + } + s.mu.RUnlock() + if len(matched) == 0 { + return + } + // Estimate the retained size once for this source event. Fan-out must not + // repeat a recursive walk for every subscriber. + bytes := config.EstimateChangeBytes(change) + for _, sub := range matched { + sub.send(change, bytes) + } +} + +func (s *subscribers) remove(id uint64) { + s.mu.Lock() + delete(s.m, id) + s.mu.Unlock() +} + +func (s *subscribers) closeAll() { + s.closeWithError(nil) +} + +func (s *subscribers) closeWithError(err error) { + s.mu.Lock() + subs := make([]*subscription, 0, len(s.m)) + for id, sub := range s.m { + subs = append(subs, sub) + delete(s.m, id) + } + s.mu.Unlock() + for _, sub := range subs { + sub.closeWithError(err) + } + for _, sub := range subs { + sub.waitRelay() + } +} + +type subscription struct { + err error + ops map[string]struct{} + parent *subscribers + changes chan config.Change + done chan struct{} + notify chan struct{} + relayDone chan struct{} + tables map[string]struct{} + sourceName string + queue []queuedChange + maxBytes int64 + maxChanges int + id uint64 + queuedBytes int64 + mu sync.Mutex + closed bool +} + +type queuedChange struct { + change config.Change + bytes int64 +} + +func (s *subscription) Changes() <-chan config.Change { return s.changes } + +func (s *subscription) Close() { + s.closeWithError(nil) + s.waitRelay() +} + +func (s *subscription) Err() error { + s.mu.Lock() + err := s.err + s.mu.Unlock() + return err +} + +func (s *subscription) send(change config.Change, bytes int64) { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + if len(s.queue) >= s.maxChanges || bytes > s.maxBytes-s.queuedBytes { + parent, id := s.closeLocked(errSubscriberOverflow) + s.mu.Unlock() + if parent != nil { + parent.remove(id) + } + return + } + s.queue = append(s.queue, queuedChange{change: change, bytes: bytes}) + s.queuedBytes += bytes + s.mu.Unlock() + select { + case s.notify <- struct{}{}: + default: + } +} + +func (s *subscription) closeWithError(err error) { + s.mu.Lock() + parent, id := s.closeLocked(err) + s.mu.Unlock() + if parent != nil { + parent.remove(id) + } +} + +func (s *subscription) waitRelay() { + <-s.relayDone +} + +func (s *subscription) closeLocked(err error) (*subscribers, uint64) { + if s.closed { + return nil, 0 + } + s.closed = true + s.err = err + close(s.done) + s.queue = nil + s.queuedBytes = 0 + return s.parent, s.id +} + +func (s *subscription) run() { + defer close(s.relayDone) + defer close(s.changes) + for { + s.mu.Lock() + if len(s.queue) == 0 { + if s.closed { + s.mu.Unlock() + return + } + notify := s.notify + done := s.done + s.mu.Unlock() + select { + case <-notify: + case <-done: + } + continue + } + item := s.queue[0] + done := s.done + s.mu.Unlock() + + select { + case <-done: + return + case s.changes <- item.change: + s.mu.Lock() + if len(s.queue) > 0 { + s.queuedBytes -= s.queue[0].bytes + s.queue[0] = queuedChange{} + s.queue = s.queue[1:] + } + s.mu.Unlock() + } + } +} + +func (s *subscription) matches(change config.Change) bool { + if len(s.ops) > 0 { + if _, ok := s.ops[strings.ToLower(change.Op)]; !ok { + return false + } + } + if len(s.tables) > 0 { + if _, ok := s.tables[strings.ToLower(change.Relation)]; ok { + return true + } + if _, ok := s.tables[strings.ToLower(change.Table)]; ok { + return true + } + return false + } + return true +} + +func (s *subscription) matchesSnapshot(change config.Change) bool { + if len(s.tables) == 0 { + return true + } + if _, ok := s.tables[strings.ToLower(change.Relation)]; ok { + return true + } + _, ok := s.tables[strings.ToLower(change.Table)] + return ok +} + +func (s *subscription) isClosed() bool { + s.mu.Lock() + closed := s.closed + s.mu.Unlock() + return closed +} + +func filterSet(values []string) map[string]struct{} { + if len(values) == 0 { + return nil + } + out := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.ToLower(strings.TrimSpace(value)) + if value != "" { + out[value] = struct{}{} + } + } + if len(out) == 0 { + return nil + } + return out +} + +var _ config.Stream = (*subscription)(nil) +var _ config.ErrStream = (*subscription)(nil) diff --git a/service/cdc/sqlite/subscribers_test.go b/service/cdc/sqlite/subscribers_test.go new file mode 100644 index 000000000..79c6a7e82 --- /dev/null +++ b/service/cdc/sqlite/subscribers_test.go @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + config "github.com/wippyai/runtime/api/service/cdc" +) + +func TestFilterSet(t *testing.T) { + assert.Nil(t, filterSet(nil)) + assert.Nil(t, filterSet([]string{" ", ""})) + + got := filterSet([]string{"Users", " orders ", "users"}) + assert.Contains(t, got, "users") + assert.Contains(t, got, "orders") + assert.Len(t, got, 2) +} + +func TestSubscriptionMatches(t *testing.T) { + all := &subscription{} + assert.True(t, all.matches(config.Change{Op: "insert", Table: "users"})) + + byOp := &subscription{ops: map[string]struct{}{"insert": {}}} + assert.True(t, byOp.matches(config.Change{Op: "insert"})) + assert.False(t, byOp.matches(config.Change{Op: "delete"})) + + byTable := &subscription{tables: map[string]struct{}{"users": {}}} + assert.True(t, byTable.matches(config.Change{Op: "insert", Table: "users"})) + assert.True(t, byTable.matches(config.Change{Op: "insert", Relation: "users"})) + assert.False(t, byTable.matches(config.Change{Op: "insert", Table: "orders"})) +} + +func TestSubscriptionSnapshotMatchesOnlyTables(t *testing.T) { + sub := &subscription{ + ops: map[string]struct{}{"insert": {}}, + tables: map[string]struct{}{"users": {}}, + } + + assert.False(t, sub.matchesSnapshot(config.Change{Op: "snapshot", Table: "orders"}), "table filter still applies to snapshot rows") + assert.True(t, sub.matchesSnapshot(config.Change{Op: "snapshot", Table: "users"})) + assert.False(t, sub.matches(config.Change{Op: "delete", Table: "users"}), "op filter still applies to normal changes") + assert.False(t, sub.matches(config.Change{Op: "insert", Table: "orders"}), "table filter still applies to normal changes") +} + +func TestSubscribersPublishAndClose(t *testing.T) { + subs := newSubscribers() + stream := subs.subscribe("s", config.StreamOptions{}) + + subs.publish(config.Change{Op: "insert", Table: "users", Source: "s"}) + + select { + case change := <-stream.Changes(): + assert.Equal(t, "insert", change.Op) + assert.Equal(t, "users", change.Table) + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for change") + } + + subs.closeAll() + select { + case _, ok := <-stream.Changes(): + assert.False(t, ok, "channel should be closed after closeAll") + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for close") + } +} + +func TestSubscribeBufferClamp(t *testing.T) { + subs := newSubscribers() + + def := subs.subscribe("s", config.StreamOptions{Buffer: 0}) + assert.Equal(t, defaultStreamBuffer, def.maxChanges) + + neg := subs.subscribe("s", config.StreamOptions{Buffer: -5}) + assert.Equal(t, defaultStreamBuffer, neg.maxChanges) + + exact := subs.subscribe("s", config.StreamOptions{Buffer: 7}) + assert.Equal(t, 7, exact.maxChanges) + + huge := subs.subscribe("s", config.StreamOptions{Buffer: maxStreamBuffer + 100}) + assert.Equal(t, maxStreamBuffer, huge.maxChanges) + assert.Zero(t, cap(def.changes), "the common adapter must not add a second queue") +} + +func TestSubscribeAssignsUniqueIncreasingIDs(t *testing.T) { + subs := newSubscribers() + a := subs.subscribe("s", config.StreamOptions{}) + b := subs.subscribe("s", config.StreamOptions{}) + assert.Equal(t, uint64(1), a.id) + assert.Equal(t, uint64(2), b.id) +} + +func TestPublishNeverBlocksAndClosesLaggard(t *testing.T) { + subs := newSubscribers() + stream := subs.subscribe("s", config.StreamOptions{Buffer: 1}) + + done := make(chan struct{}) + go func() { + for i := 0; i < 1000; i++ { + subs.publish(config.Change{Op: "insert", Table: "t"}) + } + close(done) + }() + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("publish blocked on a non-reading subscriber") + } + + for { + select { + case _, ok := <-stream.Changes(): + if !ok { + return + } + case <-time.After(2 * time.Second): + t.Fatal("laggard subscription was not closed on overflow") + } + } +} + +func TestOverflowedSubscriberDetachesImmediately(t *testing.T) { + subs := newSubscribers() + stream := subs.subscribe("s", config.StreamOptions{Buffer: 1}) + change := config.Change{Op: "insert", Table: "users"} + + bytes := config.EstimateChangeBytes(change) + stream.send(change, bytes) + stream.send(change, bytes) + + assert.ErrorIs(t, stream.Err(), errSubscriberOverflow) + subs.mu.RLock() + remaining := len(subs.m) + subs.mu.RUnlock() + assert.Zero(t, remaining) +} + +func TestOverflowedSubscriberChurnDoesNotRetainParentEntries(t *testing.T) { + subs := newSubscribers() + change := config.Change{Op: "insert", Table: "users"} + for i := 0; i < 1000; i++ { + stream := subs.subscribe("s", config.StreamOptions{Buffer: 1}) + bytes := config.EstimateChangeBytes(change) + stream.send(change, bytes) + stream.send(change, bytes) + } + + subs.mu.RLock() + remaining := len(subs.m) + subs.mu.RUnlock() + assert.Zero(t, remaining) +} + +func TestSubscriptionMaxBytesReleasesOnlyAfterDelivery(t *testing.T) { + change := config.Change{ + Op: "insert", + Table: "users", + After: map[string]any{"payload": []byte("payload")}, + } + changeBytes := config.EstimateChangeBytes(change) + sub := newSubscription("s", config.StreamOptions{Buffer: 2, MaxBytes: changeBytes + 1}, 2) + defer sub.Close() + + sub.send(change, config.EstimateChangeBytes(change)) + assert.Eventually(t, func() bool { + sub.mu.Lock() + defer sub.mu.Unlock() + return len(sub.queue) == 1 && sub.queuedBytes == changeBytes + }, time.Second, time.Millisecond) + + select { + case got := <-sub.Changes(): + assert.Equal(t, change.Table, got.Table) + case <-time.After(time.Second): + t.Fatal("timed out receiving queued change") + } + assert.Eventually(t, func() bool { + sub.mu.Lock() + defer sub.mu.Unlock() + return len(sub.queue) == 0 && sub.queuedBytes == 0 + }, time.Second, time.Millisecond) + + sub.send(change, config.EstimateChangeBytes(change)) + assert.NotErrorIs(t, sub.Err(), errSubscriberOverflow) + select { + case <-sub.Changes(): + case <-time.After(time.Second): + t.Fatal("released byte budget did not accept the next change") + } +} + +func TestSubscriptionMaxBytesOverflowIsLocal(t *testing.T) { + change := config.Change{ + Op: "insert", + Table: "users", + After: map[string]any{"payload": []byte("payload")}, + } + limit := config.EstimateChangeBytes(change) - 1 + subs := newSubscribers() + laggard := subs.subscribe("s", config.StreamOptions{MaxBytes: limit}) + reader := subs.subscribe("s", config.StreamOptions{MaxBytes: limit + 1}) + + subs.publish(change) + assert.ErrorIs(t, laggard.Err(), errSubscriberOverflow) + assert.NotErrorIs(t, reader.Err(), errSubscriberOverflow) + select { + case got := <-reader.Changes(): + assert.Equal(t, change.Table, got.Table) + case <-time.After(time.Second): + t.Fatal("unrelated subscriber did not receive change") + } + reader.Close() +} + +func TestSubscribersFilterByOp(t *testing.T) { + subs := newSubscribers() + stream := subs.subscribe("s", config.StreamOptions{Ops: []string{"delete"}}) + defer stream.Close() + + subs.publish(config.Change{Op: "insert", Table: "users"}) + subs.publish(config.Change{Op: "delete", Table: "users"}) + + select { + case change := <-stream.Changes(): + require.Equal(t, "delete", change.Op) + case <-time.After(2 * time.Second): + t.Fatal("timed out") + } +} diff --git a/service/cdc/stream.go b/service/cdc/stream.go new file mode 100644 index 000000000..af6246daf --- /dev/null +++ b/service/cdc/stream.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "sync" + + "github.com/wippyai/runtime/api/registry" + api "github.com/wippyai/runtime/api/service/cdc" +) + +// stampedStream is the boundary between a driver stream and the common CDC +// API. Drivers own transport-specific change decoding; the stable source slot +// owns the process identity and generation that consumers use for routing and +// resume diagnostics. +type stampedStream struct { + upstream api.Stream + out chan api.Change + done chan struct{} + sourceID registry.ID + generation string + once sync.Once +} + +func newStampedStream(id registry.ID, generation uint64, _ int, upstream api.Stream) *stampedStream { + stream := &stampedStream{ + upstream: upstream, + sourceID: registry.ParseID(id.String()), + generation: generationString(generation), + // The driver owns the only bounded subscriber queue. Keep this common + // identity adapter unbuffered so it cannot double retained events. + out: make(chan api.Change), + done: make(chan struct{}), + } + go stream.run() + return stream +} + +func (s *stampedStream) Changes() <-chan api.Change { return s.out } + +func (s *stampedStream) Close() { + s.once.Do(func() { + close(s.done) + s.upstream.Close() + }) +} + +func (s *stampedStream) Err() error { + return s.upstream.Err() +} + +func (s *stampedStream) run() { + defer close(s.out) + changes := s.upstream.Changes() + for { + select { + case <-s.done: + return + case change, ok := <-changes: + if !ok { + return + } + change.SourceID = s.sourceID + change.Source = s.sourceID.String() + change.Generation = s.generation + select { + case <-s.done: + return + case s.out <- change: + } + } + } +} + +var _ api.Stream = (*stampedStream)(nil) +var _ api.ErrStream = (*stampedStream)(nil) diff --git a/service/cdc/stream_test.go b/service/cdc/stream_test.go new file mode 100644 index 000000000..48a7b15c3 --- /dev/null +++ b/service/cdc/stream_test.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/registry" + api "github.com/wippyai/runtime/api/service/cdc" +) + +type stampedTestStream struct { + changes chan api.Change + once sync.Once +} + +func (s *stampedTestStream) Changes() <-chan api.Change { return s.changes } + +func (s *stampedTestStream) Close() { + s.once.Do(func() { close(s.changes) }) +} + +func (*stampedTestStream) Err() error { return nil } + +func TestStampedStreamUsesOnlyTheDriverQueue(t *testing.T) { + upstream := &stampedTestStream{changes: make(chan api.Change, 2)} + stream := newStampedStream(registry.NewID("app", "cdc"), 7, 65536, upstream) + require.Zero(t, cap(stream.Changes()), "the common adapter must not add a second event queue") + + upstream.changes <- api.Change{Op: "insert", Table: "users"} + select { + case change := <-stream.Changes(): + require.Equal(t, "insert", change.Op) + require.Equal(t, "users", change.Table) + require.Equal(t, "app:cdc", change.Source) + require.Equal(t, registry.NewID("app", "cdc"), change.SourceID) + require.Equal(t, "7", change.Generation) + case <-time.After(time.Second): + t.Fatal("timed out waiting for stamped change") + } + + stream.Close() + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("stamped stream did not close after upstream close") + } +} diff --git a/service/host/host.go b/service/host/host.go index 004922459..b0a4729df 100644 --- a/service/host/host.go +++ b/service/host/host.go @@ -172,6 +172,7 @@ func processName(start *process.Start) string { func (h *Host) sendMessages(target pid.PID, messages []*relay.Message) { pkg := relay.NewMessagePackage(pid.PID{}, target, messages...) if err := h.scheduler.Send(pkg); err != nil { + relay.ReleasePackage(pkg) h.log.Warn("failed to send messages", zap.String("target", target.String()), zap.Error(err)) @@ -186,10 +187,23 @@ func (h *Host) Terminate(_ context.Context, processID pid.PID) error { // Send implements relay.Receiver. func (h *Host) Send(pkg *relay.Package) error { + return h.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender. The actor scheduler admits +// messages without a blocking goroutine, so cancellation can stop a relay +// directly at the host boundary. +func (h *Host) SendContext(ctx context.Context, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } if h.shutdown.Load() { return ErrHostShuttingDown } - return h.scheduler.Send(pkg) + return h.scheduler.SendContext(ctx, pkg) } // Start implements supervisor.Service. diff --git a/service/host/host_test.go b/service/host/host_test.go index 4b11cad93..5a930ce92 100644 --- a/service/host/host_test.go +++ b/service/host/host_test.go @@ -24,6 +24,7 @@ import ( hostapi "github.com/wippyai/runtime/api/service/host" "github.com/wippyai/runtime/api/topology" "github.com/wippyai/runtime/internal/uniqid" + relaysys "github.com/wippyai/runtime/system/relay" "github.com/wippyai/runtime/system/scheduler/actor" securitysys "github.com/wippyai/runtime/system/security" "go.uber.org/zap" @@ -692,6 +693,81 @@ func TestHost_SendMessagesEmpty(t *testing.T) { th.host.sendMessages(target, nil) } +type contextMessageProcess struct { + ready chan struct{} + received chan struct{} + readyOnce sync.Once + recvOnce sync.Once +} + +func (p *contextMessageProcess) Init(context.Context, string, payload.Payloads) error { return nil } + +func (p *contextMessageProcess) Step(events []process.Event, out *process.StepOutput) error { + p.readyOnce.Do(func() { close(p.ready) }) + for _, event := range events { + if event.Type != process.EventMessage { + continue + } + pkg, ok := event.Data.(*relay.Package) + if !ok { + continue + } + relay.ReleasePackage(pkg) + p.recvOnce.Do(func() { close(p.received) }) + out.Done(nil) + return nil + } + out.Idle() + return nil +} + +func (p *contextMessageProcess) Close() {} + +func TestHostSendContextThroughLocalNode(t *testing.T) { + th := newTestHost() + th.start(t) + defer th.stop() + + processID := pid.PID{Node: "test-node", Host: "test:host", UniqID: "context-send"} + processID = processID.Precomputed() + proc := &contextMessageProcess{ready: make(chan struct{}), received: make(chan struct{})} + _, err := th.scheduler.Submit(context.Background(), processID, proc, "", nil) + require.NoError(t, err) + select { + case <-proc.ready: + case <-time.After(time.Second): + t.Fatal("process did not reach idle state") + } + + node := relaysys.NewNode("test-node") + require.NoError(t, node.RegisterHost("test:host", th.host)) + pkg := relay.NewPackage(pid.PID{}, processID, "context", payload.New("value")) + require.NoError(t, node.SendContext(context.Background(), pkg)) + select { + case <-proc.received: + case <-time.After(time.Second): + t.Fatal("local host did not receive cancellable delivery") + } +} + +func TestHostSendContextRejectsUnknownAndCanceledDelivery(t *testing.T) { + h := newTestHost() + h.start(t) + defer h.stop() + + unknown := relay.NewPackage(pid.PID{}, pid.PID{Host: "test:host", UniqID: "missing"}, "context", payload.New("value")) + err := h.host.SendContext(context.Background(), unknown) + require.ErrorIs(t, err, process.ErrProcessNotFound) + relay.ReleasePackage(unknown) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + canceled := relay.NewPackage(pid.PID{}, pid.PID{Host: "test:host", UniqID: "missing"}, "context", payload.New("value")) + err = h.host.SendContext(ctx, canceled) + require.ErrorIs(t, err, context.Canceled) + relay.ReleasePackage(canceled) +} + // --- Concurrent Operation Tests --- func TestHost_ConcurrentRun(t *testing.T) { diff --git a/service/sql/conn.go b/service/sql/conn.go index 54039c1fe..34c3a39f9 100644 --- a/service/sql/conn.go +++ b/service/sql/conn.go @@ -5,48 +5,52 @@ package sql import ( "context" "database/sql" - "errors" - "fmt" - "net/url" - "sort" - "strconv" - "strings" "sync" "sync/atomic" - config "github.com/wippyai/runtime/api/service/sql" - "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/resource" + sqlapi "github.com/wippyai/runtime/api/service/sql" ) // ConnPool represents a database connection pool that acts both as a service // and a resource provider type ConnPool struct { - db *sql.DB - current *dbGeneration - status chan any - config atomic.Pointer[any] - kind registry.Kind - mu sync.RWMutex - wg sync.WaitGroup - closed atomic.Bool + driver Driver + stopErr error + stopDone chan struct{} + current *dbGeneration + status chan any + config atomic.Pointer[any] + db *sql.DB + kind registry.Kind + wg sync.WaitGroup + mu sync.RWMutex + stopMu sync.Mutex + closed atomic.Bool + stopStarted bool } type dbGeneration struct { + closeErr error + observer sqlapi.CommittedMutationSource db *sql.DB closed chan struct{} - closeErr error - closeMu sync.Mutex once sync.Once + closeMu sync.Mutex refs atomic.Int32 closing atomic.Bool } -func newDBGeneration(db *sql.DB) *dbGeneration { +func newDBGeneration(db *sql.DB, observers ...sqlapi.CommittedMutationSource) *dbGeneration { + var observer sqlapi.CommittedMutationSource + if len(observers) > 0 { + observer = observers[0] + } return &dbGeneration{ - db: db, - closed: make(chan struct{}), + db: db, + closed: make(chan struct{}), + observer: observer, } } @@ -83,8 +87,13 @@ func (g *dbGeneration) closeWhenIdle() { func (g *dbGeneration) closeNow() { g.once.Do(func() { + if g.observer != nil { + _ = g.observer.Close() + } g.closeMu.Lock() - g.closeErr = g.db.Close() + if g.db != nil { + g.closeErr = g.db.Close() + } g.closeMu.Unlock() close(g.closed) }) @@ -148,100 +157,115 @@ func (p *ConnPool) Start(ctx context.Context) (<-chan any, error) { // Stop implements supervisor.Service func (p *ConnPool) Stop(ctx context.Context) error { - // Try to set closed state - if already closed, return immediately - if !p.closed.CompareAndSwap(false, true) { - return nil - } - - // Wait for all resources to be released - done := make(chan struct{}) - go func() { - p.wg.Wait() - close(done) - }() - + if ctx == nil { + ctx = context.Background() + } + p.stopMu.Lock() + // Serialize the closed transition with Acquire's WaitGroup admission. A + // positive Add must not race with cleanupStop's Wait when the pool has no + // outstanding resources; holding this mutex makes the handoff explicit. + p.closed.Store(true) + if !p.stopStarted { + p.stopStarted = true + p.stopDone = make(chan struct{}) + go p.cleanupStop(p.stopDone) + } + done := p.stopDone + p.stopMu.Unlock() select { case <-ctx.Done(): return ctx.Err() case <-done: - p.mu.Lock() - if p.current == nil && p.db != nil { - p.current = newDBGeneration(p.db) - } - gen := p.current - p.current = nil - p.db = nil - p.mu.Unlock() + p.stopMu.Lock() + err := p.stopErr + p.stopMu.Unlock() + return err + } +} + +func (p *ConnPool) cleanupStop(done chan struct{}) { + p.wg.Wait() + p.mu.Lock() + if p.current == nil && p.db != nil { + p.current = newDBGeneration(p.db) + } + gen := p.current + p.current = nil + p.db = nil + p.mu.Unlock() + var err error + if gen != nil { gen.closeWhenIdle() - return gen.waitClosed(ctx) + err = gen.waitClosed(context.Background()) } + p.stopMu.Lock() + p.stopErr = err + p.stopMu.Unlock() + close(done) } -// UpdateConfig updates the pool configuration +// UpdateConfig updates the pool configuration. It delegates engine-specific +// validation and tuning to the engine registered for the pool's kind. func (p *ConnPool) UpdateConfig(cfg any) error { if p.closed.Load() { return ErrPoolClosed } - switch c := cfg.(type) { - case *config.DBConfig: - if p.kind == config.SQLite { - return NewInvalidConfigTypeError("DBConfig", config.SQLite) - } - - if err := c.Validate(); err != nil { - return NewInvalidConfigError(err) - } + ec, ok := cfg.(sqlapi.EngineConfig) + if !ok { + return NewUnsupportedConfigTypeError(p.kind) + } - newDB, err := openStandardDB(p.kind, c) - if err != nil { - return err - } + if p.driver == nil { + return NewUnsupportedConfigTypeError(p.kind) + } - newGen := newDBGeneration(newDB) - p.mu.Lock() - if p.closed.Load() { - p.mu.Unlock() - _ = newDB.Close() - return ErrPoolClosed - } - oldGen := p.current - if oldGen == nil && p.db != nil { - oldGen = newDBGeneration(p.db) - } - p.current = newGen - p.db = newDB - p.mu.Unlock() + return p.updateConfig(context.Background(), p.driver, ec) +} - if oldGen != nil { - oldGen.closeWhenIdle() - } +func (p *ConnPool) updateConfig(ctx context.Context, driver Driver, ec sqlapi.EngineConfig) error { + if p.closed.Load() { + return ErrPoolClosed + } - var cfg any = c - p.config.Store(&cfg) + if err := driver.ValidateConfigType(ec); err != nil { + return err + } - case *config.SQLiteConfig: - if p.kind != config.SQLite { - return NewInvalidConfigTypeError("SQLiteConfig", p.kind) - } + if err := ec.Validate(); err != nil { + return NewInvalidConfigError(err) + } - if err := c.Validate(); err != nil { - return NewInvalidConfigError(err) - } + opened, err := openDriverDB(ctx, driver, ec) + if err != nil { + return err + } + newGen := newDBGeneration(opened.DB, opened.Observer) - gen := p.currentGeneration() - if gen == nil { - return ErrPoolClosed + p.mu.Lock() + if p.closed.Load() { + p.mu.Unlock() + if opened.Observer != nil { + _ = opened.Observer.Close() } - gen.db.SetConnMaxLifetime(c.Pool.MaxLifetime) - - var cfg any = c - p.config.Store(&cfg) + _ = opened.DB.Close() + return ErrPoolClosed + } + oldGen := p.current + if oldGen == nil && p.db != nil { + oldGen = newDBGeneration(p.db) + } + p.current = newGen + p.db = opened.DB + p.mu.Unlock() - default: - return NewUnsupportedConfigTypeError(p.kind) + if oldGen != nil { + oldGen.closeWhenIdle() } + var stored any = ec + p.config.Store(&stored) + return nil } @@ -256,8 +280,15 @@ func (p *ConnPool) Acquire( return nil, NewUnsupportedAccessModeError(string(mode)) } - // Track resource usage before checking closed state to avoid race with Stop() + // Admission is serialized with Stop. This prevents a positive WaitGroup Add + // from racing with cleanupStop's Wait after the counter reaches zero. + p.stopMu.Lock() + if p.closed.Load() { + p.stopMu.Unlock() + return nil, ErrPoolClosed + } p.wg.Add(1) + p.stopMu.Unlock() if p.closed.Load() { p.wg.Done() @@ -280,160 +311,6 @@ func (p *ConnPool) Acquire( } } -func openStandardDB(kind registry.Kind, cfg *config.DBConfig) (*sql.DB, error) { - dsn, err := buildDSN(kind, cfg) - if err != nil { - return nil, NewInvalidDSNError(err) - } - - db, err := sql.Open(getDriver(kind), dsn) - if err != nil { - return nil, NewConnectionPoolCreationError(err) - } - - db.SetMaxOpenConns(cfg.Pool.MaxOpen) - db.SetMaxIdleConns(cfg.Pool.MaxIdle) - db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) - return db, nil -} - -// Helper to build DSN string for different database types -func buildDSN(kind registry.Kind, cfg *config.DBConfig) (string, error) { - switch kind { - case config.Postgres: - if err := validateDSNFields(cfg); err != nil { - return "", err - } - opts := buildPostgresOptionsString(cfg.Options) - var b strings.Builder - b.Grow(128) - b.WriteString("host=") - b.WriteString(quotePostgresValue(cfg.Host)) - b.WriteString(" port=") - b.WriteString(strconv.Itoa(cfg.Port)) - b.WriteString(" user=") - b.WriteString(quotePostgresValue(cfg.Username)) - b.WriteString(" password=") - b.WriteString(quotePostgresValue(cfg.Password)) - b.WriteString(" dbname=") - b.WriteString(quotePostgresValue(cfg.Database)) - if opts != "" { - b.WriteString(" ") - b.WriteString(opts) - } - return b.String(), nil - - case config.MySQL: - if err := validateDSNFields(cfg); err != nil { - return "", err - } - opts := buildMySQLOptionsString(cfg.Options) - var b strings.Builder - b.Grow(128) - b.WriteString(cfg.Username) - b.WriteString(":") - b.WriteString(cfg.Password) - b.WriteString("@tcp(") - b.WriteString(cfg.Host) - b.WriteString(":") - b.WriteString(strconv.Itoa(cfg.Port)) - b.WriteString(")/") - b.WriteString(cfg.Database) - if opts != "" { - b.WriteString("?") - b.WriteString(opts) - } - return b.String(), nil - - default: - return "", NewUnsupportedDatabaseTypeError(kind) - } -} - -func validateDSNFields(cfg *config.DBConfig) error { - switch { - case cfg.Host == "": - return NewInvalidDSNError(errors.New("host is empty")) - case cfg.Port <= 0: - return NewInvalidDSNError(fmt.Errorf("port is invalid: %d", cfg.Port)) - case cfg.Username == "": - return NewInvalidDSNError(errors.New("username is empty")) - case cfg.Database == "": - return NewInvalidDSNError(errors.New("database is empty")) - } - return nil -} - -func quotePostgresValue(value string) string { - var b strings.Builder - b.Grow(len(value) + 2) - b.WriteByte('\'') - for i := 0; i < len(value); i++ { - c := value[i] - if c == '\\' || c == '\'' { - b.WriteByte('\\') - } - b.WriteByte(c) - } - b.WriteByte('\'') - return b.String() -} - -func getDriver(kind registry.Kind) string { - switch kind { - case config.Postgres: - return "postgres" - case config.MySQL: - return "mysql" - default: - return kind - } -} - -// buildPostgresOptionsString renders lib/pq keyword/value options. -func buildPostgresOptionsString(options map[string]string) string { - if len(options) == 0 { - return "" - } - - keys := make([]string, 0, len(options)) - for k := range options { - keys = append(keys, k) - } - sort.Strings(keys) - - var b strings.Builder - b.Grow(len(options) * 20) - for i, k := range keys { - if i > 0 { - b.WriteString(" ") - } - b.WriteString(k) - b.WriteString("=") - b.WriteString(quotePostgresValue(options[k])) - } - - return b.String() -} - -func buildMySQLOptionsString(options map[string]string) string { - if len(options) == 0 { - return "" - } - - values := url.Values{} - for k, v := range options { - values.Set(k, v) - } - return values.Encode() -} - -// Helper kept for older internal tests and benchmarks. PostgreSQL was the only -// historical caller that used this space-separated keyword/value form. -func buildOptionsString(options map[string]string) string { - return buildPostgresOptionsString(options) -} - // DBConn represents a database connection resource type DBConn struct { pool *ConnPool @@ -444,8 +321,9 @@ type DBConn struct { // DBResource contains both the database connection and its type type DBResource struct { - DB *sql.DB // The database connection - Type registry.Kind // The database type (postgres, mysql, sqlite, etc.) + Observer sqlapi.CommittedMutationSource + DB *sql.DB // The database connection + Type registry.Kind // The database type (postgres, mysql, sqlite, etc.) } // newDBConn creates a new database resource @@ -465,8 +343,9 @@ func (r *DBConn) Get() (any, error) { // Return both the DB and its type return DBResource{ - DB: r.gen.db, - Type: r.dbType, + DB: r.gen.db, + Type: r.dbType, + Observer: r.gen.observer, }, nil } diff --git a/service/sql/conn_test.go b/service/sql/conn_test.go index 0da2e344f..e0e98f5d8 100644 --- a/service/sql/conn_test.go +++ b/service/sql/conn_test.go @@ -28,6 +28,7 @@ func newTestPool(t *testing.T) *ConnPool { pool := &ConnPool{ kind: apiconfig.SQLite, db: db, + driver: func() Driver { d, _ := testDriverFor(apiconfig.SQLite); return d }(), status: make(chan any, 1), } @@ -179,6 +180,23 @@ func TestConnPool_StopTimeout(t *testing.T) { assert.Equal(t, context.DeadlineExceeded, err) } +func TestConnPool_StopTimeoutStillCleansUp(t *testing.T) { + pool := newTestPool(t) + ctx := context.Background() + _, err := pool.Start(ctx) + require.NoError(t, err) + res, err := pool.Acquire(ctx, testID, resource.ModeNormal) + require.NoError(t, err) + stopCtx, cancel := context.WithTimeout(ctx, 25*time.Millisecond) + err = pool.Stop(stopCtx) + cancel() + require.ErrorIs(t, err, context.DeadlineExceeded) + res.Release() + require.NoError(t, pool.Stop(ctx)) + _, err = pool.Start(ctx) + assert.ErrorIs(t, err, ErrPoolClosed) +} + func TestDBConn_DoubleRelease(t *testing.T) { pool := newTestPool(t) ctx := context.Background() @@ -363,133 +381,6 @@ func TestConnPool_UpdateConfigSwapsStandardDBAndRetiresOldAfterRelease(t *testin require.NoError(t, pool.Stop(ctx)) } -func TestBuildDSN(t *testing.T) { - tests := []struct { - cfg *apiconfig.DBConfig - name string - kind string - wantErr bool - }{ - { - name: "postgres", - kind: apiconfig.Postgres, - cfg: &apiconfig.DBConfig{ - Host: "localhost", Port: 5432, Database: "db", - Username: "user", Password: "pass", - }, - wantErr: false, - }, - { - name: "mysql", - kind: apiconfig.MySQL, - cfg: &apiconfig.DBConfig{ - Host: "localhost", Port: 3306, Database: "db", - Username: "user", Password: "pass", - }, - wantErr: false, - }, - { - name: "unsupported", - kind: "db.unknown", - cfg: &apiconfig.DBConfig{}, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := buildDSN(tt.kind, tt.cfg) - if tt.wantErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - }) - } -} - -func TestBuildOptionsString(t *testing.T) { - t.Run("empty options", func(t *testing.T) { - result := buildOptionsString(nil) - assert.Empty(t, result) - }) - - t.Run("single option", func(t *testing.T) { - result := buildOptionsString(map[string]string{"sslmode": "disable"}) - assert.Equal(t, "sslmode='disable'", result) - }) - - t.Run("postgres options are stable and space separated", func(t *testing.T) { - result := buildPostgresOptionsString(map[string]string{ - "sslmode": "disable", - "connect_timeout": "10", - "application_name": "test", - }) - assert.Equal(t, "application_name='test' connect_timeout='10' sslmode='disable'", result) - }) - - t.Run("mysql options are stable query parameters", func(t *testing.T) { - result := buildMySQLOptionsString(map[string]string{ - "charset": "utf8mb4", - "parseTime": "true", - "timeout": "2s", - }) - assert.Equal(t, "charset=utf8mb4&parseTime=true&timeout=2s", result) - }) -} - -func TestQuotePostgresValue(t *testing.T) { - assert.Equal(t, "'alice'", quotePostgresValue("alice")) - assert.Equal(t, "''", quotePostgresValue("")) - assert.Equal(t, "'se cret'", quotePostgresValue("se cret")) - assert.Equal(t, `'O\'Brien'`, quotePostgresValue("O'Brien")) - assert.Equal(t, `'a\\b'`, quotePostgresValue(`a\b`)) -} - -func TestValidateDSNFields(t *testing.T) { - base := func() *apiconfig.DBConfig { - return &apiconfig.DBConfig{Host: "h", Port: 5432, Username: "u", Database: "d"} - } - - require.NoError(t, validateDSNFields(base())) - - c := base() - c.Host = "" - assert.ErrorContains(t, validateDSNFields(c), "host is empty") - - c = base() - c.Port = 0 - assert.ErrorContains(t, validateDSNFields(c), "port is invalid") - - c = base() - c.Username = "" - assert.ErrorContains(t, validateDSNFields(c), "username is empty") - - c = base() - c.Database = "" - assert.ErrorContains(t, validateDSNFields(c), "database is empty") -} - -func TestBuildDSN_EmptyUsernameDoesNotAbsorbNextToken(t *testing.T) { - cfg := &apiconfig.DBConfig{ - Host: "h", - Port: 5432, - Database: "d", - Username: "", - Password: "secret", - } - - _, err := buildDSN(apiconfig.Postgres, cfg) - require.Error(t, err) - assert.Contains(t, err.Error(), "username is empty") -} - -func TestGetDriver(t *testing.T) { - assert.Equal(t, "postgres", getDriver(apiconfig.Postgres)) - assert.Equal(t, "mysql", getDriver(apiconfig.MySQL)) - assert.Equal(t, "unknown", getDriver("unknown")) -} - // Benchmarks func newBenchPool(b *testing.B) *ConnPool { @@ -559,31 +450,3 @@ func BenchmarkConnPool_ConcurrentAcquire(b *testing.B) { } }) } - -func BenchmarkBuildDSN_Postgres(b *testing.B) { - cfg := &apiconfig.DBConfig{ - Host: "localhost", Port: 5432, Database: "db", - Username: "user", Password: "pass", - Options: map[string]string{"sslmode": "disable"}, - } - - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - _, _ = buildDSN(apiconfig.Postgres, cfg) - } -} - -func BenchmarkBuildOptionsString(b *testing.B) { - opts := map[string]string{ - "sslmode": "disable", - "connect_timeout": "10", - "application_name": "test", - } - - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - _ = buildOptionsString(opts) - } -} diff --git a/service/sql/driver.go b/service/sql/driver.go new file mode 100644 index 000000000..2a7214e65 --- /dev/null +++ b/service/sql/driver.go @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sql diff --git a/service/sql/driver_test.go b/service/sql/driver_test.go new file mode 100644 index 000000000..deb8c27b4 --- /dev/null +++ b/service/sql/driver_test.go @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sql + +import ( + "context" + "database/sql" + "errors" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + "go.uber.org/zap" +) + +// fixedConfigEngine returns a preset SQLite config regardless of the entry, so driver +// tests can drive createPool to sql.Open without a transcoder. Prepare can be forced +// to fail to exercise the create lifecycle's close-on-error branch. +type fixedConfigEngine struct { + prepareErr error + kind registry.Kind + driver string +} + +func (e fixedConfigEngine) Kind() registry.Kind { return e.kind } + +func (e fixedConfigEngine) DriverName() string { return e.driver } + +func (fixedConfigEngine) DecodeConfig(context.Context, payload.Transcoder, registry.Entry) (config.EngineConfig, error) { + return &config.SQLiteConfig{File: ":memory:", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, nil +} + +func (fixedConfigEngine) ResolveEnv(context.Context, EngineDeps, config.EngineConfig) error { + return nil +} + +func (fixedConfigEngine) BuildDSN(config.EngineConfig) (string, error) { + return ":memory:", nil +} + +func (e fixedConfigEngine) Prepare(context.Context, *sql.DB, config.EngineConfig) error { + return e.prepareErr +} + +func (fixedConfigEngine) Tune(*sql.DB, config.EngineConfig) {} + +func (fixedConfigEngine) ValidateConfigType(config.EngineConfig) error { + return nil +} + +func TestDriverSelectionIsExplicit(t *testing.T) { + const kind = registry.Kind("db.sql.drivernametest") + driver := fixedConfigEngine{kind: kind, driver: "sqlite3"} + deps := EngineDeps{Log: zap.NewNop()} + entry := registry.Entry{ID: registry.NewID("test", "ov"), Kind: kind, Data: payload.New("x")} + + _, _, err := NewDefaultPoolFactory().CreatePool(context.Background(), deps, entry) + require.Error(t, err, "an unconfigured factory must not discover a driver globally") + + factory := NewDefaultPoolFactory(driver) + pool, _, err := factory.CreatePool(context.Background(), deps, entry) + require.NoError(t, err) + require.NotNil(t, pool) + require.NoError(t, pool.Stop(context.Background())) +} + +func TestCreatePoolClosesOnPrepareError(t *testing.T) { + const kind = registry.Kind("db.sql.preparefailtest") + prepErr := errors.New("prepare boom") + driver := fixedConfigEngine{kind: kind, driver: "sqlite3", prepareErr: prepErr} + + entry := registry.Entry{ID: registry.NewID("test", "pf"), Kind: kind, Data: payload.New("x")} + pool, _, err := NewDefaultPoolFactory(driver).CreatePool(context.Background(), EngineDeps{Log: zap.NewNop()}, entry) + + require.Error(t, err) + assert.Nil(t, pool) + assert.ErrorIs(t, err, prepErr) +} diff --git a/service/sql/engine.go b/service/sql/engine.go new file mode 100644 index 000000000..46de55cf1 --- /dev/null +++ b/service/sql/engine.go @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sql + +import ( + "context" + "database/sql" + "fmt" + + envapi "github.com/wippyai/runtime/api/env" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + "go.uber.org/zap" +) + +// EngineDeps carries the shared collaborators an engine needs to turn a registry +// entry into a configured pool. +type EngineDeps struct { + Transcoder payload.Transcoder + Env envapi.Registry + Log *zap.Logger +} + +// Engine is a self-contained SQL dialect. Engines are supplied to a Manager (or +// Factory) explicitly, so the SQL service has no process-global engine registry. +// The original contract intentionally remains small: custom engines may keep +// using BuildDSN plus database/sql, while engines that own a physical connector +// can additionally implement DBOpener. +type Engine interface { + Kind() registry.Kind + DriverName() string + DecodeConfig(ctx context.Context, dtt payload.Transcoder, entry registry.Entry) (config.EngineConfig, error) + ResolveEnv(ctx context.Context, deps EngineDeps, cfg config.EngineConfig) error + BuildDSN(cfg config.EngineConfig) (string, error) + Prepare(ctx context.Context, db *sql.DB, cfg config.EngineConfig) error + Tune(db *sql.DB, cfg config.EngineConfig) + ValidateConfigType(cfg config.EngineConfig) error +} + +// Driver is the explicit-injection name for an Engine. It is an alias so +// existing extensions implementing the original Engine contract remain valid. +type Driver = Engine + +// DBOpener is the optional physical-handle seam. A driver that implements it +// owns the database connector and any capabilities attached to that physical +// handle (for example SQLite mutation observation). Engines that do not need +// that ownership use the Engine.BuildDSN fallback in openDriverDB. +type DBOpener interface { + Open(ctx context.Context, cfg config.EngineConfig) (OpenedDB, error) +} + +// OpenedDB is the physical database handle created by a Driver. Observer is an +// optional engine capability and is deliberately kept beside the handle so it +// cannot be accidentally shared between unrelated pool generations. +type OpenedDB struct { + DB *sql.DB + Observer config.CommittedMutationSource +} + +// createPool runs the generic create lifecycle for a known engine. +func createPool(ctx context.Context, deps EngineDeps, driver Driver, entry registry.Entry) (*ConnPool, config.EngineConfig, error) { + cfg, err := driver.DecodeConfig(ctx, deps.Transcoder, entry) + if err != nil { + return nil, nil, NewInvalidConfigError(err) + } + + if err := driver.ResolveEnv(ctx, deps, cfg); err != nil { + return nil, nil, err + } + + if err := cfg.Validate(); err != nil { + return nil, nil, NewInvalidConfigError(err) + } + + opened, err := openDriverDB(ctx, driver, cfg) + if err != nil { + return nil, nil, err + } + + pool := &ConnPool{ + kind: driver.Kind(), + driver: driver, + db: opened.DB, + current: newDBGeneration(opened.DB, opened.Observer), + status: make(chan any, 1), + } + + var cfgAny any = cfg + pool.config.Store(&cfgAny) + + return pool, cfg, nil +} + +// updatePool runs the generic update lifecycle for a known engine. +func updatePool(ctx context.Context, deps EngineDeps, driver Driver, pool *ConnPool, entry registry.Entry) (config.EngineConfig, error) { + cfg, err := driver.DecodeConfig(ctx, deps.Transcoder, entry) + if err != nil { + return nil, NewInvalidConfigError(err) + } + + if err := driver.ResolveEnv(ctx, deps, cfg); err != nil { + return nil, err + } + + if err := pool.updateConfig(ctx, driver, cfg); err != nil { + return nil, NewPoolUpdateError(err) + } + + return cfg, nil +} + +func openDriverDB(ctx context.Context, driver Driver, cfg config.EngineConfig) (OpenedDB, error) { + var ( + opened OpenedDB + err error + ) + if opener, ok := driver.(DBOpener); ok { + opened, err = opener.Open(ctx, cfg) + } else { + var dsn string + dsn, err = driver.BuildDSN(cfg) + if err != nil { + return OpenedDB{}, NewInvalidDSNError(err) + } + opened.DB, err = sql.Open(driver.DriverName(), dsn) + if err != nil { + return OpenedDB{}, NewConnectionPoolCreationError(err) + } + } + if err != nil { + return OpenedDB{}, err + } + if opened.DB == nil { + if opened.Observer != nil { + _ = opened.Observer.Close() + } + return OpenedDB{}, NewConnectionPoolCreationError( + fmt.Errorf("driver %q returned a nil database", driver.Kind()), + ) + } + + if err := driver.Prepare(ctx, opened.DB, cfg); err != nil { + _ = opened.DB.Close() + if opened.Observer != nil { + _ = opened.Observer.Close() + } + return OpenedDB{}, err + } + + driver.Tune(opened.DB, cfg) + return opened, nil +} diff --git a/service/sql/engine/all/all.go b/service/sql/engine/all/all.go new file mode 100644 index 000000000..45ce33bad --- /dev/null +++ b/service/sql/engine/all/all.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package all provides the built-in SQL drivers for composition roots that want +// the standard Wippy database set. It does not register anything globally. +package all + +import ( + sqlservice "github.com/wippyai/runtime/service/sql" + "github.com/wippyai/runtime/service/sql/engine/sqlite" + "github.com/wippyai/runtime/service/sql/engine/standard" +) + +// Drivers returns the built-in SQL drivers in a deterministic order. +func Drivers() []sqlservice.Driver { + return []sqlservice.Driver{ + standard.NewPostgresDriver(), + standard.NewMySQLDriver(), + sqlite.NewDriver(), + } +} diff --git a/service/sql/engine/sqlite/observer.go b/service/sql/engine/sqlite/observer.go new file mode 100644 index 000000000..9ffb31d9d --- /dev/null +++ b/service/sql/engine/sqlite/observer.go @@ -0,0 +1,2076 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "bytes" + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "io" + "reflect" + "strconv" + "strings" + "sync" + "sync/atomic" + + "github.com/mattn/go-sqlite3" + + sqlapi "github.com/wippyai/runtime/api/service/sql" +) + +var ( + errObserverClosed = errors.New("sqlite mutation observer is closed") + errObserverOverflow = errors.New("sqlite mutation observer backlog overflow") + errObserverAmbiguous = errors.New("sqlite mutation observer cannot determine statement outcome") +) + +const ( + defaultSnapshotBatchSize = 512 + maxSnapshotBatchSize = 4096 + mutationStructuralBytes = 128 + valueStructuralBytes = 24 +) + +// sqliteConnector keeps the SQLite driver and its connection state owned by +// one SQL pool. It intentionally uses sql.OpenDB instead of sql.Register, so +// no process-global driver name or file-path registry is involved. +type sqliteConnector struct { + driver *sqlite3.SQLiteDriver + backend *sqliteBackend + dsn string +} + +func (c *sqliteConnector) Connect(context.Context) (driver.Conn, error) { + raw, err := c.driver.Open(c.dsn) + if err != nil { + return nil, err + } + sqliteConn, ok := raw.(*sqlite3.SQLiteConn) + if !ok { + _ = raw.Close() + return nil, fmt.Errorf("sqlite driver returned %T", raw) + } + + conn := &observedConn{ + raw: raw, + sqlite: sqliteConn, + backend: c.backend, + state: &sqliteConnectionState{ + backend: c.backend, sqlite: sqliteConn, + maxChanges: c.backend.maxChanges, maxBytes: c.backend.maxBytes, + maxCommitEnds: c.backend.maxChanges, + }, + } + // Install hooks for every physical connection when it is created. This + // avoids trying to mutate a connection that may be in use when a stream is + // subscribed; the backend decides whether candidates are retained. + conn.bindIfActive() + return conn, nil +} + +func (c *sqliteConnector) Driver() driver.Driver { return c.driver } + +func openSQLite(_ context.Context, dsn string, limits ...int) (*sql.DB, sqlapi.CommittedMutationSource, error) { + maxChanges, maxBytes := observerLimits(limits) + backend := newSQLiteBackend(maxChanges, maxBytes) + connector := &sqliteConnector{ + dsn: dsn, + driver: &sqlite3.SQLiteDriver{}, + backend: backend, + } + db := sql.OpenDB(connector) + backend.db = db + return db, backend, nil +} + +type sqliteBackend struct { + relayWake chan struct{} + streams map[*mutationStream]struct{} + fence chan struct{} + db *sql.DB + relayDone chan struct{} + relayQueue []*backendBatch + maxChanges int + sequence atomic.Uint64 + maxBytes int + relayChanges int + relayBytes int + mu sync.Mutex + closed bool +} + +type backendBatch struct { + streams []*mutationStream + batch sqlapi.MutationBatch + bytes int +} + +func newSQLiteBackend(maxChanges, maxBytes int) *sqliteBackend { + fence := make(chan struct{}, 1) + fence <- struct{}{} + backend := &sqliteBackend{ + streams: make(map[*mutationStream]struct{}), + fence: fence, + maxChanges: maxChanges, + maxBytes: maxBytes, + relayWake: make(chan struct{}, 1), + relayDone: make(chan struct{}), + } + go func() { + defer close(backend.relayDone) + backend.relay() + }() + return backend +} + +func observerLimits(limits []int) (int, int) { + maxChanges, maxBytes := sqlapi.DefaultMaxMutationChanges, sqlapi.DefaultMaxMutationBytes + if len(limits) > 0 && limits[0] > 0 { + maxChanges = limits[0] + } + if len(limits) > 1 && limits[1] > 0 { + maxBytes = limits[1] + } + return maxChanges, maxBytes +} + +func (b *sqliteBackend) acquireFence(ctx context.Context) error { + select { + case <-b.fence: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (b *sqliteBackend) acquireCommitFence() { + <-b.fence +} + +func (b *sqliteBackend) releaseFence() { + b.fence <- struct{}{} +} + +func (b *sqliteBackend) hasObservers() bool { + b.mu.Lock() + active := !b.closed && len(b.streams) > 0 + b.mu.Unlock() + return active +} + +func (b *sqliteBackend) Subscribe(ctx context.Context, opts sqlapi.MutationOptions) (sqlapi.MutationStream, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if opts.MaxChanges <= 0 { + opts.MaxChanges = b.maxChanges + } + if opts.MaxBytes <= 0 { + opts.MaxBytes = b.maxBytes + } + if err := b.validateTables(ctx, opts.Tables); err != nil { + return nil, err + } + b.mu.Lock() + if b.closed { + b.mu.Unlock() + return nil, errObserverClosed + } + stream := newMutationStream(ctx, b, opts) + b.streams[stream] = struct{}{} + b.mu.Unlock() + stream.start() + return stream, nil +} + +func (b *sqliteBackend) validateTables(ctx context.Context, requested []string) error { + b.mu.Lock() + db := b.db + closed := b.closed + b.mu.Unlock() + if closed { + return errObserverClosed + } + if db == nil { + return errors.New("sqlite observer has no database") + } + conn, err := db.Conn(ctx) + if err != nil { + return fmt.Errorf("acquire sqlite observer connection: %w", err) + } + defer conn.Close() + return conn.Raw(func(raw any) error { + observed, ok := raw.(*observedConn) + if !ok { + return fmt.Errorf("sqlite observer received %T", raw) + } + state := &sqliteConnectionState{backend: b, sqlite: observed.sqlite} + tables, err := tablesForValidation(observed.sqlite, requested) + if err != nil { + return err + } + for _, table := range tables { + if err := state.validateTable(table.schema, table.name); err != nil { + return err + } + } + return nil + }) +} + +func tablesForValidation(conn *sqlite3.SQLiteConn, requested []string) ([]snapshotTable, error) { + if len(requested) == 0 { + rows, err := conn.Query(`SELECT name FROM main.sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name`, nil) + if err != nil { + return nil, err + } + defer rows.Close() + var tables []snapshotTable + values := make([]driver.Value, len(rows.Columns())) + for { + err := rows.Next(values) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, err + } + table, ok := values[0].(string) + if !ok { + return nil, fmt.Errorf("sqlite table name has type %T", values[0]) + } + tables = append(tables, snapshotTable{schema: "main", name: table}) + } + return tables, nil + } + tables := make([]snapshotTable, 0, len(requested)) + for _, name := range requested { + parts := strings.SplitN(name, ".", 2) + if len(parts) == 1 { + tables = append(tables, snapshotTable{schema: "main", name: parts[0]}) + } else { + tables = append(tables, snapshotTable{schema: parts[0], name: parts[1]}) + } + } + return tables, nil +} + +func (b *sqliteBackend) Snapshot(ctx context.Context, opts sqlapi.SnapshotOptions) (sqlapi.SnapshotStream, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + b.mu.Lock() + closed := b.closed + db := b.db + b.mu.Unlock() + if closed { + return nil, errObserverClosed + } + if db == nil { + return nil, errors.New("sqlite snapshot has no database") + } + // Reserve the physical connection before taking the fence. SQL pools may + // have a single connection; taking the fence first would let a writer hold + // that connection while waiting for the snapshot and deadlock both paths. + conn, err := db.Conn(ctx) + if err != nil { + return nil, fmt.Errorf("acquire sqlite snapshot connection: %w", err) + } + if err := b.acquireFence(ctx); err != nil { + _ = conn.Close() + return nil, err + } + release := true + defer func() { + if release { + b.releaseFence() + } + }() + tx, err := conn.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + _ = conn.Close() + return nil, fmt.Errorf("begin sqlite snapshot: %w", err) + } + // database/sql may defer BEGIN until the first operation. Force a read + // while the fence is held so the transaction's SQLite read view is fixed + // before later commits can be buffered as live changes. + var schemaVersion int64 + if err := tx.QueryRowContext(ctx, "PRAGMA schema_version").Scan(&schemaVersion); err != nil { + _ = tx.Rollback() + _ = conn.Close() + return nil, fmt.Errorf("establish sqlite snapshot view: %w", err) + } + if err := validateSnapshotTablesTx(ctx, tx, opts.Tables); err != nil { + _ = tx.Rollback() + _ = conn.Close() + return nil, err + } + watermark := strconv.FormatUint(b.sequence.Load(), 10) + scanCtx, cancel := context.WithCancel(ctx) + if opts.MaxChanges <= 0 { + opts.MaxChanges = b.maxChanges + } + if opts.MaxBytes <= 0 { + opts.MaxBytes = b.maxBytes + } + b.mu.Lock() + if b.closed { + b.mu.Unlock() + cancel() + _ = tx.Rollback() + _ = conn.Close() + return nil, errObserverClosed + } + stream := newSnapshotStream(scanCtx, b, opts, watermark, cancel) + b.streams[stream] = struct{}{} + b.mu.Unlock() + // The fence remains held until the stream is registered and its read view + // has been established. New commits therefore receive a sequence greater + // than watermark and are buffered by this stream. + release = false + b.releaseFence() + stream.start() + go b.scanSnapshot(scanCtx, conn, tx, stream, opts) + return stream, nil +} + +func (b *sqliteBackend) scanSnapshot(ctx context.Context, conn *sql.Conn, tx *sql.Tx, stream *mutationStream, opts sqlapi.SnapshotOptions) { + defer conn.Close() + defer func() { + if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { + stream.finishSnapshot(err) + } + }() + tables, err := snapshotTables(ctx, tx, opts.Tables) + if err == nil { + batchSize := normalizeSnapshotBatchSize(opts.BatchSize) + for _, table := range tables { + if err = scanSnapshotTable(ctx, tx, stream, table.schema, table.name, batchSize); err != nil { + break + } + } + } + if err != nil { + _ = tx.Rollback() + b.remove(stream, err) + return + } + if err = tx.Commit(); err != nil { + b.remove(stream, fmt.Errorf("commit sqlite snapshot: %w", err)) + return + } + stream.finishSnapshot(nil) +} + +func normalizeSnapshotBatchSize(value int) int { + if value <= 0 { + return defaultSnapshotBatchSize + } + if value > maxSnapshotBatchSize { + return maxSnapshotBatchSize + } + return value +} + +type snapshotTable struct { + schema string + name string +} + +func snapshotTables(ctx context.Context, tx *sql.Tx, requested []string) ([]snapshotTable, error) { + if len(requested) == 0 { + rows, err := tx.QueryContext(ctx, `SELECT name FROM main.sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + var tables []snapshotTable + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + tables = append(tables, snapshotTable{schema: "main", name: name}) + } + return tables, rows.Err() + } + tables := make([]snapshotTable, 0, len(requested)) + for _, name := range requested { + parts := strings.SplitN(name, ".", 2) + if len(parts) == 1 { + tables = append(tables, snapshotTable{schema: "main", name: parts[0]}) + } else { + tables = append(tables, snapshotTable{schema: parts[0], name: parts[1]}) + } + } + return tables, nil +} + +func validateSnapshotTablesTx(ctx context.Context, tx *sql.Tx, requested []string) error { + tables, err := snapshotTables(ctx, tx, requested) + if err != nil { + return err + } + for _, table := range tables { + if strings.EqualFold(table.schema, "temp") { + return errors.New("sqlite snapshot does not support TEMP tables") + } + var definition sql.NullString + query := sqliteMasterQuery(table.schema) + if err := tx.QueryRowContext(ctx, query, table.name).Scan(&definition); err != nil { + return fmt.Errorf("inspect sqlite snapshot table %s.%s: %w", table.schema, table.name, err) + } + upper := strings.ToUpper(definition.String) + if strings.Contains(upper, "WITHOUT ROWID") { + return fmt.Errorf("sqlite snapshot does not support WITHOUT ROWID table %s.%s", table.schema, table.name) + } + if strings.HasPrefix(strings.TrimSpace(upper), "CREATE VIRTUAL TABLE") { + return fmt.Errorf("sqlite snapshot does not support virtual table %s.%s", table.schema, table.name) + } + } + return nil +} + +func scanSnapshotTable(ctx context.Context, tx *sql.Tx, stream *mutationStream, schema, table string, batchSize int) error { + rows, err := tx.QueryContext(ctx, fmt.Sprintf("SELECT rowid, * FROM %s.%s", quoteIdentifier(schema), quoteIdentifier(table))) + if err != nil { + return fmt.Errorf("scan sqlite snapshot %s.%s: %w", schema, table, err) + } + defer rows.Close() + columns, err := rows.Columns() + if err != nil { + return err + } + if len(columns) == 0 { + return nil + } + columns = append([]string(nil), columns[1:]...) + batcher := newSnapshotBatcher(stream.watermark, batchSize, stream.maxChanges, stream.maxBytes) + for rows.Next() { + values := make([]any, len(columns)+1) + dest := make([]any, len(values)) + for i := range values { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + return err + } + rowID, ok := values[0].(int64) + if !ok || rowID == 0 { + return fmt.Errorf("sqlite snapshot %s.%s returned invalid rowid %v", schema, table, values[0]) + } + after := append([]any(nil), values[1:]...) + change := sqlapi.Mutation{ + Schema: schema, Table: table, Columns: columns, + RowID: rowID, After: after, Op: "snapshot", + } + if err := batcher.add(change, stream.pushSnapshot); err != nil { + return err + } + } + if err := rows.Err(); err != nil { + return err + } + return batcher.flush(stream.pushSnapshot) +} + +type snapshotBatcher struct { + transaction string + changes []sqlapi.Mutation + batchBytes int + batchSize int + maxChanges int + maxBytes int +} + +func newSnapshotBatcher(transaction string, batchSize, maxChanges, maxBytes int) *snapshotBatcher { + return &snapshotBatcher{ + transaction: transaction, + batchBytes: mutationBatchBytes(sqlapi.MutationBatch{Transaction: transaction}), + batchSize: batchSize, + maxChanges: maxChanges, + maxBytes: maxBytes, + } +} + +func (b *snapshotBatcher) add(change sqlapi.Mutation, emit func(sqlapi.MutationBatch) error) error { + changeBytes := mutationSize(change) + if len(b.changes) > 0 { + bytesExceed := b.maxBytes > 0 && (b.batchBytes > b.maxBytes || changeBytes > b.maxBytes-b.batchBytes) + changesExceed := b.maxChanges > 0 && len(b.changes) >= b.maxChanges + if bytesExceed || changesExceed { + if err := b.flush(emit); err != nil { + return err + } + } + } + b.changes = append(b.changes, change) + b.batchBytes = saturatingAdd(b.batchBytes, changeBytes) + if len(b.changes) >= b.batchSize || + (b.maxBytes > 0 && b.batchBytes >= b.maxBytes) || + (b.maxChanges > 0 && len(b.changes) >= b.maxChanges) { + return b.flush(emit) + } + return nil +} + +func (b *snapshotBatcher) flush(emit func(sqlapi.MutationBatch) error) error { + if len(b.changes) == 0 { + return nil + } + batch := sqlapi.MutationBatch{ + Transaction: b.transaction, + Snapshot: true, + Changes: append([]sqlapi.Mutation(nil), b.changes...), + } + b.changes = b.changes[:0] + b.batchBytes = mutationBatchBytes(sqlapi.MutationBatch{Transaction: b.transaction}) + return emit(batch) +} + +func (b *sqliteBackend) remove(stream *mutationStream, err error) { + b.mu.Lock() + delete(b.streams, stream) + b.mu.Unlock() + + stream.closeWithError(err) +} + +func (b *sqliteBackend) publish(changes []sqlapi.Mutation) { + if len(changes) == 0 { + return + } + b.mu.Lock() + if b.closed { + b.mu.Unlock() + return + } + if len(b.streams) == 0 { + b.mu.Unlock() + return + } + streams := make([]*mutationStream, 0, len(b.streams)) + for stream := range b.streams { + streams = append(streams, stream) + } + changes = append([]sqlapi.Mutation(nil), changes...) + sequence := b.sequence.Add(1) + batch := sqlapi.MutationBatch{ + Transaction: strconv.FormatUint(sequence, 10), + Changes: changes, + } + batchBytes := mutationBatchBytes(batch) + if (b.maxChanges > 0 && (len(changes) > b.maxChanges || b.relayChanges > b.maxChanges-len(changes))) || + (b.maxBytes > 0 && (batchBytes > b.maxBytes || b.relayBytes > b.maxBytes-batchBytes)) { + b.closed = true + b.streams = make(map[*mutationStream]struct{}) + b.relayQueue = nil + b.relayChanges = 0 + b.relayBytes = 0 + b.mu.Unlock() + b.closeStreams(streams, errObserverOverflow) + b.signalRelay() + return + } + b.relayQueue = append(b.relayQueue, &backendBatch{batch: batch, streams: streams, bytes: batchBytes}) + b.relayChanges = saturatingAdd(b.relayChanges, len(changes)) + b.relayBytes = saturatingAdd(b.relayBytes, batchBytes) + b.mu.Unlock() + b.signalRelay() +} + +func (b *sqliteBackend) fail(err error) { + b.mu.Lock() + if b.closed { + b.mu.Unlock() + return + } + b.closed = true + streams := make([]*mutationStream, 0, len(b.streams)) + for stream := range b.streams { + streams = append(streams, stream) + } + b.streams = make(map[*mutationStream]struct{}) + b.relayQueue = nil + b.relayChanges = 0 + b.relayBytes = 0 + b.mu.Unlock() + + b.closeStreams(streams, err) + b.signalRelay() +} + +func (b *sqliteBackend) Close() error { + b.mu.Lock() + if b.closed { + b.mu.Unlock() + <-b.relayDone + return nil + } + b.closed = true + streams := make([]*mutationStream, 0, len(b.streams)) + for stream := range b.streams { + streams = append(streams, stream) + } + b.streams = make(map[*mutationStream]struct{}) + b.relayQueue = nil + b.relayChanges = 0 + b.relayBytes = 0 + b.mu.Unlock() + + b.closeStreams(streams, errObserverClosed) + b.signalRelay() + <-b.relayDone + return nil +} + +func (b *sqliteBackend) closeStreams(streams []*mutationStream, err error) { + for _, stream := range streams { + stream.closeWithError(err) + } +} + +func (b *sqliteBackend) signalRelay() { + select { + case b.relayWake <- struct{}{}: + default: + } +} + +func (b *sqliteBackend) relay() { + for { + b.mu.Lock() + if len(b.relayQueue) == 0 { + closed := b.closed + b.mu.Unlock() + if closed { + return + } + <-b.relayWake + continue + } + item := b.relayQueue[0] + b.mu.Unlock() + + for _, stream := range item.streams { + stream.push(item.batch) + } + + b.mu.Lock() + if len(b.relayQueue) > 0 && b.relayQueue[0] == item { + b.relayQueue = b.relayQueue[1:] + b.relayChanges -= len(item.batch.Changes) + b.relayBytes -= item.bytes + } + b.mu.Unlock() + } +} + +// sqliteConnectionState is attached to one physical SQLite connection. The +// hooks only collect a candidate transaction. Publication happens from the +// driver wrappers after Exec/Commit/Rows completion, when statement rollback +// and savepoint effects are known. +type sqliteConnectionState struct { + unsupported error + failed error + sqlite *sqlite3.SQLiteConn + backend *sqliteBackend + statementSavepointVerb string + statementSavepointName string + savepoints []savepoint + pending []sqlapi.Mutation + commitEnds []int + prepareMeta statementMeta + statementMark int + maxBytes int + maxChanges int + pendingBytes int + confirmedEnds int + maxCommitEnds int + rollbackSeen bool + ddlInTxn bool + dmlInTxn bool + fenceHeld bool + statementDDL bool + commitPending bool + rollbackUnconfirmed bool +} + +// statementMeta is collected by SQLite's authorizer while a statement is +// prepared. It avoids interpreting comments, literals, or quoted identifiers +// as executable SQL control words. A prepared statement carries this metadata +// to its later execution; direct Exec/Query paths merge it after the driver's +// native prepare loop returns. +type statementMeta struct { + unsupported error + savepointVerb string + savepointName string + savepointCount int + ddl bool +} + +type savepoint struct { + name string + index int +} + +func (s *sqliteConnectionState) bind(conn *sqlite3.SQLiteConn) { + conn.RegisterPreUpdateHook(s.preUpdate) + conn.RegisterCommitHook(s.commit) + conn.RegisterRollbackHook(s.rollback) + conn.RegisterAuthorizer(s.authorizer) + s.sqlite = conn +} + +func (s *sqliteConnectionState) authorizer(action int, arg1, arg2, _ string) int { + // Reaching authorizer for another prepared statement proves that any + // earlier commit-hook boundary belongs to a completed statement. This is + // the native boundary signal needed when a later statement in one Exec + // fails before its own pre-update hook runs. + if len(s.commitEnds) > s.confirmedEnds { + s.confirmedEnds = len(s.commitEnds) + } + switch action { + case sqlite3.SQLITE_CREATE_INDEX, + sqlite3.SQLITE_CREATE_TABLE, + sqlite3.SQLITE_CREATE_TEMP_INDEX, + sqlite3.SQLITE_CREATE_TEMP_TABLE, + sqlite3.SQLITE_CREATE_TEMP_TRIGGER, + sqlite3.SQLITE_CREATE_TEMP_VIEW, + sqlite3.SQLITE_CREATE_TRIGGER, + sqlite3.SQLITE_CREATE_VIEW, + sqlite3.SQLITE_DROP_INDEX, + sqlite3.SQLITE_DROP_TABLE, + sqlite3.SQLITE_DROP_TEMP_INDEX, + sqlite3.SQLITE_DROP_TEMP_TABLE, + sqlite3.SQLITE_DROP_TEMP_TRIGGER, + sqlite3.SQLITE_DROP_TEMP_VIEW, + sqlite3.SQLITE_DROP_TRIGGER, + sqlite3.SQLITE_DROP_VIEW, + sqlite3.SQLITE_ALTER_TABLE, + sqlite3.SQLITE_ATTACH, + sqlite3.SQLITE_DETACH: + s.prepareMeta.ddl = true + case sqlite3.SQLITE_CREATE_VTABLE: + s.prepareMeta.ddl = true + s.prepareMeta.unsupported = fmt.Errorf("sqlite mutation observer cannot observe virtual table %s", arg1) + case sqlite3.SQLITE_DROP_VTABLE: + s.prepareMeta.ddl = true + s.prepareMeta.unsupported = fmt.Errorf("sqlite mutation observer cannot observe dropped virtual table %s", arg1) + case sqlite3.SQLITE_SAVEPOINT: + verb, name := authorizerSavepoint(arg1, arg2) + if verb != "" { + s.prepareMeta.savepointCount++ + s.prepareMeta.savepointVerb = verb + s.prepareMeta.savepointName = name + } + } + return sqlite3.SQLITE_OK +} + +func authorizerSavepoint(operation, name string) (string, string) { + switch strings.ToLower(operation) { + case "begin": + return "savepoint", normalizeSavepointName(name) + case "rollback": + return "rollback to", normalizeSavepointName(name) + case "release": + return "release", normalizeSavepointName(name) + default: + return "", "" + } +} + +func (s *sqliteConnectionState) preUpdate(data sqlite3.SQLitePreUpdateData) { + if s.failed != nil || s.statementDDL || strings.HasPrefix(strings.ToLower(data.TableName), "sqlite_") { + return + } + // A later statement can only reach pre-update after the preceding + // autocommit callback returned successfully. Confirm that preceding fence + // boundary before collecting the new candidate. + if len(s.commitEnds) > s.confirmedEnds { + s.confirmedEnds = len(s.commitEnds) + } + count := data.Count() + var before, after []any + var err error + switch data.Op { + case sqlite3.SQLITE_INSERT: + after, err = scanSQLiteRow(&data, count, true) + case sqlite3.SQLITE_UPDATE: + before, err = scanSQLiteRow(&data, count, false) + if err == nil { + after, err = scanSQLiteRow(&data, count, true) + } + case sqlite3.SQLITE_DELETE: + before, err = scanSQLiteRow(&data, count, false) + } + if err != nil { + s.failed = err + return + } + + op := "unknown" + switch data.Op { + case sqlite3.SQLITE_INSERT: + op = "insert" + case sqlite3.SQLITE_UPDATE: + op = "update" + case sqlite3.SQLITE_DELETE: + op = "delete" + } + s.pending = append(s.pending, sqlapi.Mutation{ + Schema: data.DatabaseName, + Table: data.TableName, + OldRowID: data.OldRowID, + RowID: data.NewRowID, + Before: before, + After: after, + Op: op, + }) + s.pendingBytes = saturatingAdd(s.pendingBytes, mutationSize(s.pending[len(s.pending)-1])) + if (s.maxChanges > 0 && len(s.pending) > s.maxChanges) || (s.maxBytes > 0 && s.pendingBytes > s.maxBytes) { + s.failed = errObserverOverflow + return + } + s.dmlInTxn = true +} + +func (s *sqliteConnectionState) commit() int { + // A single go-sqlite3 ExecContext may execute several semicolon-separated + // statements inside one C call. Reuse the fence when SQLite invokes the + // commit hook more than once before the wrapper gets control back; the + // wrapper publishes the combined candidate and releases it exactly once. + if !s.fenceHeld { + s.backend.acquireCommitFence() + s.fenceHeld = true + } + s.commitPending = true + if s.maxCommitEnds > 0 && len(s.commitEnds) >= s.maxCommitEnds { + s.failed = errObserverOverflow + return 0 + } + s.commitEnds = append(s.commitEnds, len(s.pending)) + return 0 +} + +func (s *sqliteConnectionState) rollback() { + s.rollbackSeen = true + // A failed ROLLBACK TO/RELEASE SAVEPOINT can invoke SQLite's rollback + // hook even though the surrounding transaction remains active. The + // authorizer metadata identifies that control statement; defer bookkeeping + // until the wrapper sees its actual error instead of discarding the outer + // transaction candidate here. + if s.prepareMeta.savepointVerb != "" || s.statementSavepointVerb != "" { + return + } + if len(s.commitEnds) > s.confirmedEnds { + s.rollbackUnconfirmed = true + } + if len(s.commitEnds) > 0 { + s.commitPending = true + s.failed = nil + return + } + s.pending = nil + s.pendingBytes = 0 + s.savepoints = nil + s.statementMark = 0 + s.statementSavepointVerb = "" + s.statementSavepointName = "" + s.prepareMeta = statementMeta{} + s.commitPending = false + s.commitEnds = nil + s.confirmedEnds = 0 + if s.fenceHeld { + s.fenceHeld = false + s.backend.releaseFence() + } + s.ddlInTxn = false + s.dmlInTxn = false + s.failed = nil + s.statementDDL = false + s.statementSavepointVerb = "" + s.statementSavepointName = "" + s.prepareMeta = statementMeta{} + s.unsupported = nil +} + +func (s *sqliteConnectionState) statementBegin(_ string) { + s.statementBeginWithMeta(statementMeta{}) +} + +func (s *sqliteConnectionState) statementBeginWithMeta(meta statementMeta) { + s.statementMark = len(s.pending) + s.statementDDL = meta.ddl + s.statementSavepointVerb = meta.savepointVerb + s.statementSavepointName = meta.savepointName + s.rollbackSeen = false + s.rollbackUnconfirmed = false + s.prepareMeta = statementMeta{} +} + +func (s *sqliteConnectionState) statementEnd(_ string, err error) { + s.statementEndWithMeta(err, statementMeta{}) +} + +func (s *sqliteConnectionState) statementEndWithMeta(err error, meta statementMeta) { + s.applyStatementMeta(meta) + if s.rollbackUnconfirmed { + s.resolveRollbackOutcome() + } + if err == nil { + s.confirmedEnds = len(s.commitEnds) + } else if !s.rollbackSeen { + // A commit hook boundary is authoritative once the driver call has + // returned without a rollback callback. Keep confirmed prefixes even + // when a later native statement in the same call reports an error. + s.confirmedEnds = len(s.commitEnds) + } + lastBoundary := 0 + if len(s.commitEnds) > 0 { + lastBoundary = s.commitEnds[len(s.commitEnds)-1] + } + residual := len(s.pending) > s.statementMark + if len(s.commitEnds) > 0 { + residual = len(s.pending) > lastBoundary + } + if err != nil && !s.rollbackSeen && residual { + // SQLite's pre-update hook does not expose whether a failed DML + // statement used ABORT or FAIL. Do not infer the conflict mode from + // caller SQL; closing the observer is safer than publishing a partial + // candidate whose commit status is unknown. + s.failAmbiguous() + truncate := s.statementMark + if lastBoundary > truncate { + truncate = lastBoundary + } + if truncate <= len(s.pending) { + s.pending = s.pending[:truncate] + } + s.recomputePendingBytes() + } + s.statementMark = 0 + s.rollbackSeen = false + s.rollbackUnconfirmed = false + if err == nil { + if s.statementDDL { + s.ddlInTxn = true + } + s.applySavepoint() + } + s.statementDDL = false + s.statementSavepointVerb = "" + s.statementSavepointName = "" + if s.unsupported != nil && s.backend.hasObservers() { + s.backend.fail(s.unsupported) + } + s.unsupported = nil + if s.failed != nil { + if s.commitPending { + s.finalize() + } + return + } + if s.commitPending { + s.finalize() + } +} + +// resolveRollbackOutcome runs after SQLite has returned from the physical +// operation. A rollback hook can follow a committed autocommit prefix when a +// later native statement in the same Exec fails, but it can also follow a +// failed physical commit. The hook alone cannot distinguish those cases. The +// wrapper therefore verifies the unconfirmed net rows on the same connection; +// if they are not provably committed, the observer fails closed. +func (s *sqliteConnectionState) resolveRollbackOutcome() { + if !s.rollbackUnconfirmed { + return + } + if s.sqlite == nil || !s.sqlite.AutoCommit() { + s.failAmbiguous() + return + } + base := 0 + if s.confirmedEnds > 0 { + base = s.commitEnds[s.confirmedEnds-1] + } + if base > len(s.pending) { + s.failAmbiguous() + return + } + changes, err := s.netChangesFor(s.pending[base:]) + if err != nil { + s.failAmbiguous() + return + } + committed, err := s.finalRowsMatch(changes) + if err != nil { + s.failAmbiguous() + return + } + if !committed { + if s.confirmedEnds > 0 { + s.pending = s.pending[:base] + s.recomputePendingBytes() + s.commitEnds = s.commitEnds[:s.confirmedEnds] + s.rollbackUnconfirmed = false + return + } + s.failAmbiguous() + return + } + s.confirmedEnds = len(s.commitEnds) + s.rollbackUnconfirmed = false +} + +func (s *sqliteConnectionState) finalRowsMatch(changes []sqlapi.Mutation) (bool, error) { + if len(changes) == 0 { + return true, nil + } + idsByTable := make(map[string][]int64) + seen := make(map[string]map[int64]struct{}) + for _, change := range changes { + rowID := change.RowID + if change.Op == "delete" { + rowID = change.OldRowID + } + if rowID == 0 { + return false, fmt.Errorf("sqlite mutation observer cannot verify %s.%s row", change.Schema, change.Table) + } + tableKey := change.Schema + "\x00" + change.Table + if seen[tableKey] == nil { + seen[tableKey] = make(map[int64]struct{}) + } + if _, ok := seen[tableKey][rowID]; !ok { + seen[tableKey][rowID] = struct{}{} + idsByTable[tableKey] = append(idsByTable[tableKey], rowID) + } + } + + rowsByKey := make(map[mutationKey][]any) + for tableKey, rowIDs := range idsByTable { + parts := strings.SplitN(tableKey, "\x00", 2) + if len(parts) != 2 { + return false, errors.New("sqlite mutation observer table key is invalid") + } + for start := 0; start < len(rowIDs); start += 500 { + end := start + 500 + if end > len(rowIDs) { + end = len(rowIDs) + } + placeholders := make([]string, end-start) + args := make([]driver.Value, end-start) + for i, rowID := range rowIDs[start:end] { + placeholders[i] = "?" + args[i] = rowID + } + query := fmt.Sprintf("SELECT rowid, * FROM %s.%s WHERE rowid IN (%s)", quoteIdentifier(parts[0]), quoteIdentifier(parts[1]), strings.Join(placeholders, ",")) + rawRows, err := s.sqlite.Query(query, args) + if err != nil { + return false, err + } + values := make([]driver.Value, len(rawRows.Columns())) + for { + err = rawRows.Next(values) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + _ = rawRows.Close() + return false, err + } + rowID, ok := values[0].(int64) + if !ok { + _ = rawRows.Close() + return false, fmt.Errorf("sqlite mutation observer returned rowid type %T", values[0]) + } + after := make([]any, len(values)-1) + for i := range after { + after[i] = values[i+1] + } + rowsByKey[mutationKey{schema: parts[0], table: parts[1], rowID: rowID}] = after + } + _ = rawRows.Close() + } + } + + for _, change := range changes { + rowID := change.RowID + if change.Op == "delete" { + rowID = change.OldRowID + } + row, exists := rowsByKey[mutationKey{schema: change.Schema, table: change.Table, rowID: rowID}] + switch change.Op { + case "delete": + if exists { + return false, nil + } + case "insert", "update": + if !exists || !mutationValuesEqual(row, change.After) { + return false, nil + } + default: + return false, fmt.Errorf("sqlite mutation observer cannot verify operation %q", change.Op) + } + } + return true, nil +} + +func mutationValuesEqual(left, right []any) bool { + if len(left) != len(right) { + return false + } + for i := range left { + if !mutationValueEqual(left[i], right[i]) { + return false + } + } + return true +} + +func mutationValueEqual(left, right any) bool { + if leftBytes, ok := left.([]byte); ok { + switch rightValue := right.(type) { + case []byte: + return bytes.Equal(leftBytes, rightValue) + case string: + return string(leftBytes) == rightValue + } + } + if rightBytes, ok := right.([]byte); ok { + if leftString, ok := left.(string); ok { + return leftString == string(rightBytes) + } + } + return reflect.DeepEqual(left, right) +} + +func (s *sqliteConnectionState) applyStatementMeta(meta statementMeta) { + if s.prepareMeta.savepointCount > 1 || meta.savepointCount > 1 { + s.failAmbiguous() + } + if s.prepareMeta.ddl || meta.ddl { + s.statementDDL = true + } + if s.prepareMeta.unsupported != nil { + s.unsupported = s.prepareMeta.unsupported + } + if meta.unsupported != nil { + s.unsupported = meta.unsupported + } + if s.prepareMeta.savepointVerb != "" { + s.statementSavepointVerb = s.prepareMeta.savepointVerb + s.statementSavepointName = s.prepareMeta.savepointName + } + if meta.savepointVerb != "" { + s.statementSavepointVerb = meta.savepointVerb + s.statementSavepointName = meta.savepointName + } + s.prepareMeta = statementMeta{} +} + +func (s *sqliteConnectionState) failAmbiguous() { + s.failed = errObserverAmbiguous + if s.backend.hasObservers() { + s.backend.fail(errObserverAmbiguous) + } +} + +func (s *sqliteConnectionState) finalizeAfterError(_ string, err error) { + s.statementEndWithMeta(err, statementMeta{}) +} + +// finalize runs only after SQLite has returned from the operation that caused +// the commit hook. The hook is therefore a candidate marker; it never emits +// data itself. Net reduction uses the hook images collected on this physical +// connection, after statement/savepoint outcomes are known. +func (s *sqliteConnectionState) finalize() { + if !s.commitPending { + return + } + s.commitPending = false + defer func() { + if s.fenceHeld { + s.fenceHeld = false + s.backend.releaseFence() + } + }() + if !s.backend.hasObservers() { + s.resetTransaction() + return + } + if s.failed != nil { + s.backend.fail(s.failed) + s.resetTransaction() + return + } + if s.ddlInTxn && s.dmlInTxn { + s.backend.fail(errors.New("sqlite mutation observer cannot represent DDL and DML in one transaction")) + s.resetTransaction() + return + } + ends := s.commitEnds + if len(ends) == 0 { + ends = []int{len(s.pending)} + } + start := 0 + for _, end := range ends { + if end < start || end > len(s.pending) { + s.backend.fail(errors.New("sqlite mutation observer commit boundary is invalid")) + s.resetTransaction() + return + } + changes, err := s.netChangesFor(s.pending[start:end]) + if err != nil { + s.backend.fail(err) + s.resetTransaction() + return + } + s.backend.publish(changes) + start = end + } + s.resetTransaction() +} + +func (s *sqliteConnectionState) resetTransaction() { + s.pending = nil + s.pendingBytes = 0 + s.savepoints = nil + s.statementMark = 0 + s.statementSavepointVerb = "" + s.statementSavepointName = "" + s.prepareMeta = statementMeta{} + s.commitPending = false + s.commitEnds = nil + s.confirmedEnds = 0 + s.ddlInTxn = false + s.dmlInTxn = false + s.statementDDL = false + s.unsupported = nil + s.rollbackSeen = false + s.rollbackUnconfirmed = false + s.prepareMeta = statementMeta{} +} + +// Savepoint state is applied only after SQLite reports success. A failed +// ROLLBACK TO/RELEASE must not change the candidate transaction in memory. +func (s *sqliteConnectionState) applySavepoint() { + verb, name := s.statementSavepointVerb, s.statementSavepointName + switch verb { + case "savepoint": + s.savepoints = append(s.savepoints, savepoint{name: name, index: len(s.pending)}) + case "rollback to": + if index, ok := s.findSavepoint(name); ok { + s.pending = s.pending[:index] + s.recomputePendingBytes() + for i := len(s.savepoints) - 1; i >= 0; i-- { + if s.savepoints[i].name == name { + s.savepoints = s.savepoints[:i+1] + break + } + } + } + case "release": + if index, ok := s.findSavepoint(name); ok { + for i := len(s.savepoints) - 1; i >= 0; i-- { + if s.savepoints[i].index == index { + s.savepoints = s.savepoints[:i] + break + } + } + } + } +} + +func (s *sqliteConnectionState) recomputePendingBytes() { + s.pendingBytes = 0 + for _, change := range s.pending { + s.pendingBytes += mutationSize(change) + } +} + +type mutationKey struct { + schema string + table string + rowID int64 +} + +type netMutation struct { + first string + last string + mutation sqlapi.Mutation +} + +// netChanges retains the earliest before-image for each row and the latest +// pre-update after-image. SQLite invokes the hook for every trigger-generated +// row change too, so the latest image is the committed row state without an +// O(N) SELECT round trip during the commit fence. Table metadata is resolved +// once per touched table. +func (s *sqliteConnectionState) netChangesFor(pending []sqlapi.Mutation) ([]sqlapi.Mutation, error) { + if len(pending) == 0 { + return nil, nil + } + if s.sqlite == nil { + return nil, errors.New("sqlite mutation observer has no physical connection") + } + nets := make(map[mutationKey]*netMutation, len(pending)) + order := make([]mutationKey, 0, len(pending)) + aliases := make(map[mutationKey]mutationKey) + columnsByTable := make(map[string][]string) + for _, change := range pending { + if change.OldRowID == 0 && change.RowID == 0 { + return nil, fmt.Errorf("sqlite mutation observer cannot identify %s.%s row", change.Schema, change.Table) + } + tableKey := change.Schema + "\x00" + change.Table + columns, ok := columnsByTable[tableKey] + if !ok { + if err := s.validateTable(change.Schema, change.Table); err != nil { + return nil, err + } + var err error + columns, err = s.tableColumns(change.Schema, change.Table) + if err != nil { + return nil, err + } + columnsByTable[tableKey] = columns + } + key := mutationKey{schema: change.Schema, table: change.Table, rowID: change.OldRowID} + if key.rowID == 0 { + key.rowID = change.RowID + } + if alias, ok := aliases[key]; ok { + key = alias + } + current, ok := nets[key] + if !ok { + current = &netMutation{mutation: change, first: change.Op, last: change.Op} + current.mutation.Columns = columns + nets[key] = current + order = append(order, key) + } else { + current.last = change.Op + current.mutation.Columns = columns + if change.Op == "delete" { + current.mutation.After = nil + } else if change.After != nil { + current.mutation.After = change.After + } + if current.mutation.RowID != change.RowID && change.RowID != 0 { + aliases[mutationKey{schema: change.Schema, table: change.Table, rowID: change.RowID}] = key + current.mutation.RowID = change.RowID + } + if current.mutation.OldRowID == 0 { + current.mutation.OldRowID = change.OldRowID + } + } + if change.RowID != 0 { + aliases[mutationKey{schema: change.Schema, table: change.Table, rowID: change.RowID}] = key + } + } + + result := make([]sqlapi.Mutation, 0, len(order)) + for _, key := range order { + current := nets[key] + if current.first == "insert" && current.last == "delete" { + continue + } + change := current.mutation + switch { + case current.last == "delete": + change.Op = "delete" + change.RowID = 0 + case current.first == "insert": + change.Op = "insert" + default: + change.Op = "update" + } + result = append(result, change) + } + return result, nil +} + +func (s *sqliteConnectionState) findSavepoint(name string) (int, bool) { + for i := len(s.savepoints) - 1; i >= 0; i-- { + if s.savepoints[i].name == name { + return s.savepoints[i].index, true + } + } + return 0, false +} + +func (s *sqliteConnectionState) validateTable(schema, table string) error { + if strings.EqualFold(schema, "temp") { + return errors.New("sqlite mutation observer does not support TEMP tables") + } + query := sqliteMasterQuery(schema) + rows, err := s.sqlite.Query(query, []driver.Value{table}) + if err != nil { + return fmt.Errorf("inspect sqlite table %s.%s: %w", schema, table, err) + } + defer rows.Close() + values := make([]driver.Value, len(rows.Columns())) + if err := rows.Next(values); err != nil { + if errors.Is(err, io.EOF) { + return fmt.Errorf("sqlite table %s.%s disappeared", schema, table) + } + return err + } + definition := "" + switch value := values[0].(type) { + case string: + definition = value + case []byte: + definition = string(value) + } + upper := strings.ToUpper(definition) + if strings.Contains(upper, "WITHOUT ROWID") { + return fmt.Errorf("sqlite mutation observer does not support WITHOUT ROWID table %s.%s", schema, table) + } + if strings.HasPrefix(strings.TrimSpace(upper), "CREATE VIRTUAL TABLE") { + return fmt.Errorf("sqlite mutation observer does not support virtual table %s.%s", schema, table) + } + return nil +} + +func (s *sqliteConnectionState) tableColumns(schema, table string) ([]string, error) { + query := fmt.Sprintf("SELECT * FROM %s.%s LIMIT 0", quoteIdentifier(schema), quoteIdentifier(table)) + rows, err := s.sqlite.Query(query, nil) + if err != nil { + return nil, fmt.Errorf("read sqlite columns %s.%s: %w", schema, table, err) + } + defer rows.Close() + return append([]string(nil), rows.Columns()...), nil +} + +func quoteIdentifier(value string) string { + return `"` + strings.ReplaceAll(value, `"`, `""`) + `"` +} + +func sqliteMasterQuery(schema string) string { + return "SELECT sql FROM " + quoteIdentifier(schema) + ".sqlite_master WHERE type = 'table' AND name = ?" +} + +func scanSQLiteRow(data *sqlite3.SQLitePreUpdateData, count int, isNew bool) ([]any, error) { + if count <= 0 { + return nil, nil + } + values := make([]any, count) + var err error + if isNew { + err = data.New(values...) + } else { + err = data.Old(values...) + } + return values, err +} + +func normalizeSavepointName(name string) string { + name = strings.TrimSpace(strings.TrimSuffix(name, ";")) + name = strings.Trim(name, "`\"[]") + return strings.ToLower(name) +} + +// observedConn delegates all normal SQL behavior while ensuring every physical +// connection operation gets a post-operation finalization point. +type observedConn struct { + raw driver.Conn + sqlite *sqlite3.SQLiteConn + backend *sqliteBackend + state *sqliteConnectionState +} + +func (c *observedConn) bindIfActive() { + c.state.bind(c.sqlite) +} + +func (c *observedConn) Prepare(query string) (driver.Stmt, error) { + c.state.prepareMeta = statementMeta{} + stmt, err := c.raw.Prepare(query) + meta := c.state.prepareMeta + c.state.prepareMeta = statementMeta{} + if err != nil { + return nil, err + } + return &observedStmt{raw: stmt, conn: c, query: query, meta: meta}, nil +} + +func (c *observedConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + preparer, ok := c.raw.(driver.ConnPrepareContext) + if !ok { + return nil, driver.ErrSkip + } + c.state.prepareMeta = statementMeta{} + stmt, err := preparer.PrepareContext(ctx, query) + meta := c.state.prepareMeta + c.state.prepareMeta = statementMeta{} + if err != nil { + return nil, err + } + return &observedStmt{raw: stmt, conn: c, query: query, meta: meta}, nil +} + +func (c *observedConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + execer, ok := c.raw.(driver.ExecerContext) + if !ok { + return nil, driver.ErrSkip + } + c.state.statementBegin(query) + result, err := execer.ExecContext(ctx, query, args) + c.state.statementEnd(query, err) + return result, err +} + +func (c *observedConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + queryer, ok := c.raw.(driver.QueryerContext) + if !ok { + return nil, driver.ErrSkip + } + c.state.statementBegin(query) + rows, err := queryer.QueryContext(ctx, query, args) + if err != nil { + c.state.statementEnd(query, err) + return nil, err + } + return &observedRows{raw: rows, conn: c, query: query}, nil +} + +func (c *observedConn) CheckNamedValue(value *driver.NamedValue) error { + checker, ok := c.raw.(driver.NamedValueChecker) + if !ok { + return driver.ErrSkip + } + return checker.CheckNamedValue(value) +} + +func (c *observedConn) Ping(ctx context.Context) error { + pinger, ok := c.raw.(driver.Pinger) + if !ok { + return nil + } + return pinger.Ping(ctx) +} + +func (c *observedConn) Close() error { + c.state.rollback() + return c.raw.Close() +} + +func (c *observedConn) Begin() (driver.Tx, error) { + //nolint:staticcheck // driver.Conn requires Begin for legacy driver compatibility. + tx, err := c.raw.Begin() + if err != nil { + return nil, err + } + return &observedTx{raw: tx, conn: c}, nil +} + +func (c *observedConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + begin, ok := c.raw.(driver.ConnBeginTx) + if !ok { + return c.Begin() + } + tx, err := begin.BeginTx(ctx, opts) + if err != nil { + return nil, err + } + return &observedTx{raw: tx, conn: c}, nil +} + +type observedTx struct { + raw driver.Tx + conn *observedConn +} + +func (t *observedTx) Commit() error { + err := t.raw.Commit() + if t.conn.state.commitPending { + t.conn.state.finalizeAfterError("COMMIT", err) + } + return err +} + +func (t *observedTx) Rollback() error { + err := t.raw.Rollback() + t.conn.state.rollback() + return err +} + +type observedStmt struct { + raw driver.Stmt + conn *observedConn + query string + meta statementMeta +} + +func (s *observedStmt) Close() error { return s.raw.Close() } +func (s *observedStmt) NumInput() int { return s.raw.NumInput() } + +func (s *observedStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + execer, ok := s.raw.(driver.StmtExecContext) + if !ok { + return nil, driver.ErrSkip + } + s.conn.state.statementBeginWithMeta(s.meta) + result, err := execer.ExecContext(ctx, args) + s.conn.state.statementEndWithMeta(err, s.meta) + return result, err +} + +func (s *observedStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + queryer, ok := s.raw.(driver.StmtQueryContext) + if !ok { + return nil, driver.ErrSkip + } + s.conn.state.statementBeginWithMeta(s.meta) + rows, err := queryer.QueryContext(ctx, args) + if err != nil { + s.conn.state.statementEndWithMeta(err, s.meta) + return nil, err + } + return &observedRows{raw: rows, conn: s.conn, query: s.query, meta: s.meta}, nil +} + +func (s *observedStmt) ColumnConverter(index int) driver.ValueConverter { + //nolint:staticcheck // Preserve the optional legacy converter exposed by the wrapped driver. + if converter, ok := s.raw.(driver.ColumnConverter); ok { + return converter.ColumnConverter(index) + } + return driver.DefaultParameterConverter +} + +func (s *observedStmt) Exec(args []driver.Value) (driver.Result, error) { + s.conn.state.statementBeginWithMeta(s.meta) + //nolint:staticcheck // driver.Stmt requires Exec for legacy driver compatibility. + result, err := s.raw.Exec(args) + s.conn.state.statementEndWithMeta(err, s.meta) + return result, err +} + +func (s *observedStmt) Query(args []driver.Value) (driver.Rows, error) { + s.conn.state.statementBeginWithMeta(s.meta) + //nolint:staticcheck // driver.Stmt requires Query for legacy driver compatibility. + rows, err := s.raw.Query(args) + if err != nil { + s.conn.state.statementEndWithMeta(err, s.meta) + return nil, err + } + return &observedRows{raw: rows, conn: s.conn, query: s.query, meta: s.meta}, nil +} + +type observedRows struct { + raw driver.Rows + conn *observedConn + query string + meta statementMeta + closed bool + mu sync.Mutex +} + +func (r *observedRows) Columns() []string { return r.raw.Columns() } + +func (r *observedRows) Next(dest []driver.Value) error { + err := r.raw.Next(dest) + if errors.Is(err, io.EOF) || err != nil { + r.finish(err) + } + return err +} + +func (r *observedRows) Close() error { + err := r.raw.Close() + r.finish(err) + return err +} + +func (r *observedRows) ColumnTypeDatabaseTypeName(index int) string { + if rows, ok := r.raw.(driver.RowsColumnTypeDatabaseTypeName); ok { + return rows.ColumnTypeDatabaseTypeName(index) + } + return "" +} + +func (r *observedRows) ColumnTypeLength(index int) (int64, bool) { + if rows, ok := r.raw.(driver.RowsColumnTypeLength); ok { + return rows.ColumnTypeLength(index) + } + return 0, false +} + +func (r *observedRows) ColumnTypeNullable(index int) (bool, bool) { + if rows, ok := r.raw.(driver.RowsColumnTypeNullable); ok { + return rows.ColumnTypeNullable(index) + } + return false, false +} + +func (r *observedRows) ColumnTypePrecisionScale(index int) (int64, int64, bool) { + if rows, ok := r.raw.(driver.RowsColumnTypePrecisionScale); ok { + return rows.ColumnTypePrecisionScale(index) + } + return 0, 0, false +} + +func (r *observedRows) ColumnTypeScanType(index int) reflect.Type { + if rows, ok := r.raw.(driver.RowsColumnTypeScanType); ok { + return rows.ColumnTypeScanType(index) + } + return reflect.TypeOf((*any)(nil)).Elem() +} + +func (r *observedRows) finish(err error) { + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return + } + r.closed = true + r.mu.Unlock() + r.conn.state.statementEndWithMeta(err, r.meta) +} + +type mutationStream struct { + err error + ctx context.Context + changes chan sqlapi.MutationBatch + notify chan struct{} + done chan struct{} + cancel context.CancelFunc + backend *sqliteBackend + watermark string + queue []sqlapi.MutationBatch + pending []sqlapi.MutationBatch + opts sqlapi.MutationOptions + queuedChanges int + queuedBytes int + maxChanges int + maxBytes int + mu sync.Mutex + snapshotting bool + closed bool +} + +func newMutationStream(ctx context.Context, backend *sqliteBackend, opts sqlapi.MutationOptions) *mutationStream { + if ctx == nil { + ctx = context.Background() + } + stream := &mutationStream{ + ctx: ctx, + backend: backend, + opts: opts, + changes: make(chan sqlapi.MutationBatch), + notify: make(chan struct{}, 1), + done: make(chan struct{}), + maxChanges: opts.MaxChanges, + maxBytes: opts.MaxBytes, + } + return stream +} + +func (s *mutationStream) start() { + go s.relay() +} + +func newSnapshotStream(ctx context.Context, backend *sqliteBackend, opts sqlapi.SnapshotOptions, watermark string, cancel context.CancelFunc) *mutationStream { + stream := newMutationStream(ctx, backend, sqlapi.MutationOptions{ + Tables: opts.Tables, MaxChanges: opts.MaxChanges, MaxBytes: opts.MaxBytes, + }) + stream.snapshotting = true + stream.watermark = watermark + stream.cancel = cancel + return stream +} + +func (s *mutationStream) Changes() <-chan sqlapi.MutationBatch { return s.changes } + +func (s *mutationStream) Err() error { + s.mu.Lock() + err := s.err + s.mu.Unlock() + return err +} + +func (s *mutationStream) Close() error { + s.mu.Lock() + cancel := s.cancel + s.mu.Unlock() + if cancel != nil { + cancel() + } + s.backend.remove(s, nil) + return nil +} + +func (s *mutationStream) push(batch sqlapi.MutationBatch) { + batch = filterBatch(batch, s.opts) + if len(batch.Changes) == 0 { + return + } + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + overflow := false + if s.snapshotting { + if !s.enqueuePendingLocked(batch) { + s.closeLocked(errObserverOverflow) + overflow = true + } + } else if !s.enqueueLocked(batch) { + s.closeLocked(errObserverOverflow) + overflow = true + } + s.mu.Unlock() + if overflow { + s.backend.remove(s, errObserverOverflow) + } +} + +func filterBatch(batch sqlapi.MutationBatch, opts sqlapi.MutationOptions) sqlapi.MutationBatch { + if len(opts.Tables) == 0 && len(opts.Operations) == 0 { + return batch + } + filtered := make([]sqlapi.Mutation, 0, len(batch.Changes)) + for _, change := range batch.Changes { + if len(opts.Tables) > 0 && !matchesTable(change.Schema, change.Table, opts.Tables) { + continue + } + if len(opts.Operations) > 0 && !matchesValue(change.Op, opts.Operations) { + continue + } + filtered = append(filtered, change) + } + batch.Changes = filtered + return batch +} + +func matchesTable(schema, table string, filters []string) bool { + for _, filter := range filters { + if filter == table || filter == schema+"."+table { + return true + } + } + return false +} + +func matchesValue(value string, filters []string) bool { + for _, filter := range filters { + if strings.EqualFold(value, filter) { + return true + } + } + return false +} + +func (s *mutationStream) pushSnapshot(batch sqlapi.MutationBatch) error { + s.mu.Lock() + if s.closed { + err := s.err + s.mu.Unlock() + if err != nil { + return err + } + return errObserverClosed + } + if !s.enqueueLocked(batch) { + s.closeLocked(errObserverOverflow) + s.mu.Unlock() + s.backend.remove(s, errObserverOverflow) + return errObserverOverflow + } + s.mu.Unlock() + return nil +} + +func (s *mutationStream) finishSnapshot(err error) { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + if err != nil { + s.closeLocked(err) + s.mu.Unlock() + s.backend.remove(s, err) + return + } + s.snapshotting = false + s.queue = append(s.queue, s.pending...) + s.pending = nil + s.signalLocked() + s.mu.Unlock() +} + +func (s *mutationStream) Watermark() string { return s.watermark } + +func (s *mutationStream) enqueueLocked(batch sqlapi.MutationBatch) bool { + changes := len(batch.Changes) + bytes := mutationBatchBytes(batch) + if s.maxChanges > 0 && (changes > s.maxChanges || s.queuedChanges > s.maxChanges-changes) { + return false + } + if s.maxBytes > 0 && (bytes > s.maxBytes || s.queuedBytes > s.maxBytes-bytes) { + return false + } + s.queue = append(s.queue, batch) + s.queuedChanges = saturatingAdd(s.queuedChanges, changes) + s.queuedBytes = saturatingAdd(s.queuedBytes, bytes) + s.signalLocked() + return true +} + +func (s *mutationStream) enqueuePendingLocked(batch sqlapi.MutationBatch) bool { + changes := len(batch.Changes) + bytes := mutationBatchBytes(batch) + if s.maxChanges > 0 && (changes > s.maxChanges || s.queuedChanges > s.maxChanges-changes) { + return false + } + if s.maxBytes > 0 && (bytes > s.maxBytes || s.queuedBytes > s.maxBytes-bytes) { + return false + } + s.pending = append(s.pending, batch) + s.queuedChanges = saturatingAdd(s.queuedChanges, changes) + s.queuedBytes = saturatingAdd(s.queuedBytes, bytes) + s.signalLocked() + return true +} + +func (s *mutationStream) signalLocked() { + select { + case s.notify <- struct{}{}: + default: + } +} + +func (s *mutationStream) relay() { + for { + s.mu.Lock() + if len(s.queue) == 0 { + closed := s.closed + s.mu.Unlock() + if closed { + close(s.changes) + return + } + select { + case <-s.notify: + case <-s.ctx.Done(): + s.backend.remove(s, s.ctx.Err()) + close(s.changes) + return + } + continue + } + batch := s.queue[0] + s.mu.Unlock() + + select { + case s.changes <- batch: + case <-s.done: + close(s.changes) + return + case <-s.ctx.Done(): + s.backend.remove(s, s.ctx.Err()) + close(s.changes) + return + } + + s.mu.Lock() + if len(s.queue) > 0 { + s.queue = s.queue[1:] + s.queuedChanges -= len(batch.Changes) + s.queuedBytes -= mutationBatchBytes(batch) + } + s.mu.Unlock() + } +} + +func mutationBatchBytes(batch sqlapi.MutationBatch) int { + bytes := saturatingAdd(mutationStructuralBytes, len(batch.Transaction)) + for _, change := range batch.Changes { + bytes = saturatingAdd(bytes, mutationSize(change)) + } + return bytes +} + +func mutationSize(change sqlapi.Mutation) int { + bytes := mutationStructuralBytes + bytes = saturatingAdd(bytes, len(change.Schema)) + bytes = saturatingAdd(bytes, len(change.Table)) + bytes = saturatingAdd(bytes, len(change.Op)) + for _, column := range change.Columns { + bytes = saturatingAdd(bytes, len(column)) + } + bytes = saturatingAdd(bytes, mutationValuesBytes(change.Before)) + return saturatingAdd(bytes, mutationValuesBytes(change.After)) +} + +func mutationValuesBytes(values []any) int { + bytes := 0 + for _, value := range values { + bytes = saturatingAdd(bytes, valueStructuralBytes) + switch value := value.(type) { + case nil: + bytes = saturatingAdd(bytes, 1) + case []byte: + bytes = saturatingAdd(bytes, len(value)) + case string: + bytes = saturatingAdd(bytes, len(value)) + default: + bytes = saturatingAdd(bytes, 16) + } + } + return bytes +} + +func saturatingAdd(left, right int) int { + if left < 0 || right < 0 { + return int(^uint(0) >> 1) + } + maxInt := int(^uint(0) >> 1) + if left > maxInt-right { + return maxInt + } + return left + right +} + +func (s *mutationStream) closeWithError(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.closeLocked(err) +} + +func (s *mutationStream) closeLocked(err error) { + if s.closed { + return + } + s.closed = true + s.err = err + if s.cancel != nil { + s.cancel() + s.cancel = nil + } + close(s.done) + s.queue = nil + s.pending = nil + s.queuedChanges = 0 + s.queuedBytes = 0 + s.signalLocked() +} + +var _ driver.Connector = (*sqliteConnector)(nil) +var _ driver.Conn = (*observedConn)(nil) +var _ driver.Tx = (*observedTx)(nil) +var _ driver.Stmt = (*observedStmt)(nil) +var _ driver.Rows = (*observedRows)(nil) +var _ sqlapi.CommittedMutationSource = (*sqliteBackend)(nil) +var _ sqlapi.MutationStream = (*mutationStream)(nil) diff --git a/service/sql/engine/sqlite/observer_stub.go b/service/sql/engine/sqlite/observer_stub.go new file mode 100644 index 000000000..83141cfd4 --- /dev/null +++ b/service/sql/engine/sqlite/observer_stub.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build !sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "database/sql" + "errors" + + sqlapi "github.com/wippyai/runtime/api/service/sql" +) + +var errObserverUnavailable = errors.New("sqlite committed-mutation observation is unavailable in this build") + +// openSQLite uses the normal registered SQLite driver in builds without the +// optional pre-update hook. The SQL resource remains fully usable; CDC callers +// receive an explicit unsupported capability instead of a partially working +// capture source. +func openSQLite(_ context.Context, dsn string, _ ...int) (*sql.DB, sqlapi.CommittedMutationSource, error) { + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return nil, nil, err + } + return db, nil, nil +} diff --git a/service/sql/engine/sqlite/observer_test.go b/service/sql/engine/sqlite/observer_test.go new file mode 100644 index 000000000..86cfe68a4 --- /dev/null +++ b/service/sql/engine/sqlite/observer_test.go @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + config "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" +) + +func openObservedDB(t *testing.T, file string) (*openedDBForTest, error) { + t.Helper() + opened, err := (engine{}).Open(context.Background(), &config.SQLiteConfig{File: file}) + if err != nil { + return nil, err + } + if err := (engine{}).Prepare(context.Background(), opened.DB, &config.SQLiteConfig{File: file}); err != nil { + _ = opened.DB.Close() + return nil, err + } + (engine{}).Tune(opened.DB, &config.SQLiteConfig{File: file, Pool: config.PoolConfig{MaxLifetime: time.Hour}}) + return &openedDBForTest{opened: opened}, nil +} + +type openedDBForTest struct { + opened sqlservice.OpenedDB +} + +func (o *openedDBForTest) Close() { + if o.opened.Observer != nil { + _ = o.opened.Observer.Close() + } + _ = o.opened.DB.Close() +} + +func TestPerPoolObserverCapturesOwnDatabase(t *testing.T) { + first, err := openObservedDB(t, filepath.Join(t.TempDir(), "first.db")) + require.NoError(t, err) + defer first.Close() + second, err := openObservedDB(t, filepath.Join(t.TempDir(), "second.db")) + require.NoError(t, err) + defer second.Close() + + for _, db := range []*openedDBForTest{first, second} { + _, err := db.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + } + + firstStream, err := first.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = firstStream.Close() }() + secondStream, err := second.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = secondStream.Close() }() + + _, err = first.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'first')`) + require.NoError(t, err) + _, err = second.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'second')`) + require.NoError(t, err) + + firstBatch := receiveBatch(t, firstStream) + secondBatch := receiveBatch(t, secondStream) + require.Len(t, firstBatch.Changes, 1) + require.Len(t, secondBatch.Changes, 1) + assert.Equal(t, []byte("first"), firstBatch.Changes[0].After[1]) + assert.Equal(t, []byte("second"), secondBatch.Changes[0].After[1]) + + select { + case batch := <-firstStream.Changes(): + t.Fatalf("first pool received unrelated batch: %#v", batch) + case <-time.After(50 * time.Millisecond): + } +} + +func TestPerPoolSnapshotLiveIsolation(t *testing.T) { + first, err := openObservedDB(t, filepath.Join(t.TempDir(), "snapshot-first.db")) + require.NoError(t, err) + defer first.Close() + second, err := openObservedDB(t, filepath.Join(t.TempDir(), "snapshot-second.db")) + require.NoError(t, err) + defer second.Close() + + for _, db := range []*openedDBForTest{first, second} { + _, err = db.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + _, err = db.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'initial')`) + require.NoError(t, err) + } + + type snapshotResult struct { + stream config.SnapshotStream + err error + } + firstSnapshotCh := make(chan snapshotResult, 1) + secondSnapshotCh := make(chan snapshotResult, 1) + go func() { + stream, snapshotErr := first.opened.Observer.Snapshot(context.Background(), config.SnapshotOptions{ + Tables: []string{"items"}, BatchSize: 4096, + }) + firstSnapshotCh <- snapshotResult{stream: stream, err: snapshotErr} + }() + go func() { + stream, snapshotErr := second.opened.Observer.Snapshot(context.Background(), config.SnapshotOptions{ + Tables: []string{"items"}, BatchSize: 4096, + }) + secondSnapshotCh <- snapshotResult{stream: stream, err: snapshotErr} + }() + firstSnapshot := (<-firstSnapshotCh) + secondSnapshot := (<-secondSnapshotCh) + require.NoError(t, firstSnapshot.err) + require.NoError(t, secondSnapshot.err) + defer func() { _ = firstSnapshot.stream.Close() }() + defer func() { _ = secondSnapshot.stream.Close() }() + + firstBatch := receiveBatch(t, firstSnapshot.stream) + secondBatch := receiveBatch(t, secondSnapshot.stream) + require.True(t, firstBatch.Snapshot) + require.True(t, secondBatch.Snapshot) + assert.Equal(t, "0", firstBatch.Transaction) + assert.Equal(t, "0", secondBatch.Transaction) + + _, err = first.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'first-live')`) + require.NoError(t, err) + _, err = second.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'second-live')`) + require.NoError(t, err) + firstLive := receiveBatch(t, firstSnapshot.stream) + secondLive := receiveBatch(t, secondSnapshot.stream) + require.False(t, firstLive.Snapshot) + require.False(t, secondLive.Snapshot) + assert.Equal(t, "1", firstLive.Transaction) + assert.Equal(t, "1", secondLive.Transaction) + assert.Equal(t, []byte("first-live"), firstLive.Changes[0].After[1]) + assert.Equal(t, []byte("second-live"), secondLive.Changes[0].After[1]) + require.NoError(t, firstSnapshot.stream.Close()) + require.NoError(t, secondSnapshot.stream.Close()) + + firstBackpressured, err := first.opened.Observer.Subscribe(context.Background(), config.MutationOptions{MaxChanges: 1}) + require.NoError(t, err) + secondLiveStream, err := second.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + _, err = first.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (3, 'first-backpressure')`) + require.NoError(t, err) + _, err = second.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (3, 'second-live')`) + require.NoError(t, err) + require.Equal(t, int64(3), receiveBatch(t, secondLiveStream).Changes[0].RowID) + + _, err = first.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (4, 'first-overflow')`) + require.NoError(t, err) + select { + case _, ok := <-firstBackpressured.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("first pool backpressure stream did not close") + } + assert.ErrorIs(t, firstBackpressured.Err(), errObserverOverflow) + require.NoError(t, first.opened.Observer.Close()) + + _, err = second.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (4, 'second-after-close')`) + require.NoError(t, err) + secondAfterClose := receiveBatch(t, secondLiveStream) + assert.Equal(t, int64(4), secondAfterClose.Changes[0].RowID) + assert.Equal(t, []byte("second-after-close"), secondAfterClose.Changes[0].After[1]) + _ = secondLiveStream.Close() +} + +func TestObserverRebindsAfterConnectionExpiry(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "reconnect.db")) + require.NoError(t, err) + defer observed.Close() + + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + + observed.opened.DB.SetConnMaxLifetime(time.Nanosecond) + time.Sleep(2 * time.Millisecond) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'rebound')`) + require.NoError(t, err) + + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, []byte("rebound"), batch.Changes[0].After[1]) +} + +func TestObserverClosesWithGeneration(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "close.db")) + require.NoError(t, err) + + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + require.NoError(t, observed.opened.Observer.Close()) + + select { + case _, ok := <-stream.Changes(): + assert.False(t, ok) + case <-time.After(time.Second): + t.Fatal("observer stream did not close with its generation") + } + assert.ErrorIs(t, stream.Err(), errObserverClosed) + _ = observed.opened.DB.Close() +} + +func TestObserverCloseCancelsSnapshotRead(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "snapshot-close.db")) + require.NoError(t, err) + defer observed.opened.DB.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + for i := 1; i <= 64; i++ { + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (?, ?)`, i, "value") + require.NoError(t, err) + } + snapshot, err := observed.opened.Observer.Snapshot(context.Background(), config.SnapshotOptions{ + Tables: []string{"items"}, BatchSize: 1, + }) + require.NoError(t, err) + require.NoError(t, observed.opened.Observer.Close()) + select { + case _, ok := <-snapshot.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("snapshot stream did not close after observer shutdown") + } + assert.ErrorIs(t, snapshot.Err(), errObserverClosed) +} + +func TestObserverPublishesNetTransactionAfterCommit(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "net.db")) + require.NoError(t, err) + defer observed.Close() + + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'first')`) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `UPDATE items SET value = 'second' WHERE id = 1`) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `UPDATE items SET value = 'final' WHERE id = 1`) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + change := batch.Changes[0] + assert.Equal(t, "insert", change.Op) + assert.Nil(t, change.Before) + assert.Equal(t, []byte("final"), change.After[1]) + + tx, err = observed.opened.DB.BeginTx(context.Background(), nil) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'transient')`) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `DELETE FROM items WHERE id = 2`) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + select { + case extra := <-stream.Changes(): + t.Fatalf("insert/delete cycle was published: %#v", extra) + case <-time.After(50 * time.Millisecond): + } +} + +func TestObserverSavepointRollbackDoesNotPublishRolledBackRows(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "savepoint.db")) + require.NoError(t, err) + defer observed.Close() + + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'kept')`) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `SAVEPOINT /* comment */ nested`) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'rolled-back')`) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `ROLLBACK /* comment */ TO SAVEPOINT nested`) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `RELEASE /* comment */ SAVEPOINT nested`) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(1), batch.Changes[0].RowID) + assert.Equal(t, []byte("kept"), batch.Changes[0].After[1]) + select { + case extra := <-stream.Changes(): + t.Fatalf("rolled-back row was published: %#v", extra) + case <-time.After(50 * time.Millisecond): + } +} + +func TestObserverFailsClosedForAmbiguousPartialStatement(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "partial.db")) + require.NoError(t, err) + defer observed.Close() + + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `INSERT OR FAIL INTO items (id, value) VALUES (1, 'first'), (2, 'first')`) + require.Error(t, err) + require.NoError(t, tx.Commit()) + + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("observer did not fail closed for ambiguous partial statement") + } + assert.ErrorIs(t, stream.Err(), errObserverAmbiguous) +} + +func TestObserverSnapshotFencesLiveWrites(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "snapshot.db")) + require.NoError(t, err) + defer observed.Close() + + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + for i := 1; i <= 3; i++ { + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (?, ?)`, i, "old") + require.NoError(t, err) + } + + snapshot, err := observed.opened.Observer.Snapshot(context.Background(), config.SnapshotOptions{ + Tables: []string{"items"}, BatchSize: 1, MaxChanges: 32, MaxBytes: 1 << 20, + }) + require.NoError(t, err) + defer func() { _ = snapshot.Close() }() + + writeDone := make(chan error, 1) + go func() { + _, writeErr := observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (99, 'live')`) + writeDone <- writeErr + }() + + var snapshotRows []int64 + for len(snapshotRows) < 3 { + batch := receiveBatch(t, snapshot) + require.True(t, batch.Snapshot) + for _, change := range batch.Changes { + snapshotRows = append(snapshotRows, change.RowID) + } + } + require.NoError(t, <-writeDone) + for { + batch := receiveBatch(t, snapshot) + if batch.Snapshot { + continue + } + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(99), batch.Changes[0].RowID) + assert.Equal(t, []byte("live"), batch.Changes[0].After[1]) + break + } + assert.ElementsMatch(t, []int64{1, 2, 3}, snapshotRows) +} + +func TestObserverSnapshotHandoffIncludesInFlightWriterAsLive(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "snapshot-inflight.db")) + require.NoError(t, err) + defer observed.Close() + assert.Equal(t, 0, observed.opened.DB.Stats().MaxOpenConnections, "file-backed SQLite should retain the default unlimited pool") + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (10, 'in-flight')`) + require.NoError(t, err) + + snapshot, err := observed.opened.Observer.Snapshot(context.Background(), config.SnapshotOptions{Tables: []string{"items"}, BatchSize: 8}) + require.NoError(t, err) + defer func() { _ = snapshot.Close() }() + require.NoError(t, tx.Commit()) + + select { + case batch := <-snapshot.Changes(): + if batch.Snapshot { + t.Fatalf("uncommitted writer appeared in snapshot: %#v", batch) + } + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(10), batch.Changes[0].RowID) + case <-time.After(time.Second): + t.Fatal("in-flight writer did not arrive as live batch") + } +} + +func TestObserverSnapshotFlushesBeforeByteBudget(t *testing.T) { + value := strings.Repeat("x", 256) + const maxBytes = 800 + batcher := newSnapshotBatcher("0", 4096, config.DefaultMaxMutationChanges, maxBytes) + var batches []config.MutationBatch + emit := func(batch config.MutationBatch) error { + batches = append(batches, batch) + return nil + } + for i := 1; i <= 8; i++ { + err := batcher.add(config.Mutation{ + Schema: "main", Table: "items", Columns: []string{"id", "value"}, + RowID: int64(i), After: []any{int64(i), []byte(value)}, Op: "snapshot", + }, emit) + require.NoError(t, err) + } + require.NoError(t, batcher.flush(emit)) + require.Len(t, batches, 8) + for _, batch := range batches { + require.True(t, batch.Snapshot) + require.NotEmpty(t, batch.Changes) + assert.LessOrEqual(t, mutationBatchBytes(batch), maxBytes) + assert.Len(t, batch.Changes, 1) + } +} + +func TestObserverCancelledStreamCannotRemainRegistered(t *testing.T) { + backend := newSQLiteBackend(config.DefaultMaxMutationChanges, config.DefaultMaxMutationBytes) + defer func() { _ = backend.Close() }() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + stream := newMutationStream(ctx, backend, config.MutationOptions{ + MaxChanges: config.DefaultMaxMutationChanges, + MaxBytes: config.DefaultMaxMutationBytes, + }) + backend.mu.Lock() + backend.streams[stream] = struct{}{} + backend.mu.Unlock() + stream.start() + waitForStreamCount(t, backend, 0) + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("cancelled stream did not close") + } + assert.ErrorIs(t, stream.Err(), context.Canceled) +} + +func TestObserverRemovesCancelledAndOverflowedStreams(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "stream-churn.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + backend := observed.opened.Observer.(*sqliteBackend) + + ctx, cancel := context.WithCancel(context.Background()) + cancelled, err := observed.opened.Observer.Subscribe(ctx, config.MutationOptions{}) + require.NoError(t, err) + cancel() + waitForStreamCount(t, backend, 0) + select { + case _, ok := <-cancelled.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("cancelled stream did not close") + } + + overflowed, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{MaxChanges: 1}) + require.NoError(t, err) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'one')`) + require.NoError(t, err) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'two')`) + require.NoError(t, err) + waitForStreamCount(t, backend, 0) + select { + case _, ok := <-overflowed.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("overflowed stream did not close") + } + assert.ErrorIs(t, overflowed.Err(), errObserverOverflow) +} + +func TestObserverRelayHandlesManyStreamsWithoutBlockingCommit(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "stream-scale.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + backend := observed.opened.Observer.(*sqliteBackend) + const streamCount = 256 + for range streamCount { + _, err = observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{MaxChanges: 1}) + require.NoError(t, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, err = observed.opened.DB.ExecContext(ctx, `INSERT INTO items (id, value) VALUES (1, 'one')`) + require.NoError(t, err) + _, err = observed.opened.DB.ExecContext(ctx, `INSERT INTO items (id, value) VALUES (2, 'two')`) + require.NoError(t, err) + waitForStreamCount(t, backend, 0) +} + +func TestObserverBoundsCommitMarkers(t *testing.T) { + backend := newSQLiteBackend(2, config.DefaultMaxMutationBytes) + defer func() { _ = backend.Close() }() + state := &sqliteConnectionState{backend: backend, maxCommitEnds: 2} + + assert.Equal(t, 0, state.commit()) + assert.Equal(t, 0, state.commit()) + assert.Equal(t, 0, state.commit()) + assert.Len(t, state.commitEnds, 2) + assert.ErrorIs(t, state.failed, errObserverOverflow) + state.finalize() +} + +func TestObserverAbortedStatementDoesNotPublishPartialRows(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "abort.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'same'), (2, 'same')`) + require.Error(t, err) + require.NoError(t, tx.Commit()) + select { + case _, ok := <-stream.Changes(): + require.False(t, ok, "ambiguous statement must close the observer stream") + case <-time.After(time.Second): + t.Fatal("ambiguous statement did not close the observer stream") + } + require.ErrorIs(t, stream.Err(), errObserverAmbiguous) +} + +func TestObserverFailsClosedWithoutSQLConflictInference(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "conflict-lexing.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `INSERT /* OR FAIL */ INTO items (id, value) VALUES (1, 'literal'), (2, 'literal')`) + require.Error(t, err) + require.NoError(t, tx.Commit()) + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(100 * time.Millisecond): + t.Fatal("observer did not fail closed for ambiguous conflict text") + } + assert.ErrorIs(t, stream.Err(), errObserverAmbiguous) +} + +func TestObserverFailedSavepointCommandDoesNotCorruptCapture(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "failed-savepoint.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'kept')`) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `ROLLBACK TO SAVEPOINT missing`) + require.Error(t, err) + require.NoError(t, tx.Commit()) + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(1), batch.Changes[0].RowID) +} + +func TestObserverFailsClosedForMultipleSavepointsInOneExec(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "multi-savepoint.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + _, err = observed.opened.DB.ExecContext(context.Background(), `SAVEPOINT first; SAVEPOINT second`) + require.NoError(t, err) + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("observer did not fail closed for ambiguous multi-savepoint Exec") + } + assert.ErrorIs(t, stream.Err(), errObserverAmbiguous) +} + +func TestObserverReturningRowsFinalizeOnClose(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "returning.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + rows, err := observed.opened.DB.QueryContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'returned') RETURNING id`) + require.NoError(t, err) + var id int64 + require.True(t, rows.Next()) + require.NoError(t, rows.Scan(&id)) + require.Equal(t, int64(1), id) + require.NoError(t, rows.Close()) + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(1), batch.Changes[0].RowID) +} + +func TestObserverFiltersLiveMutations(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "filters.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT); CREATE TABLE ignored (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{Tables: []string{"items"}, Operations: []string{"insert"}}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO ignored (id, value) VALUES (1, 'no')`) + require.NoError(t, err) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'yes')`) + require.NoError(t, err) + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, "items", batch.Changes[0].Table) + select { + case extra := <-stream.Changes(): + t.Fatalf("filtered mutation was published: %#v", extra) + case <-time.After(50 * time.Millisecond): + } +} + +func TestObserverOverflowClosesWithoutBlockingCommit(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "overflow.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{MaxChanges: 1, MaxBytes: 1 << 20}) + require.NoError(t, err) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'one'), (2, 'two')`) + require.NoError(t, err) + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("overflow stream did not close") + } + assert.ErrorIs(t, stream.Err(), errObserverOverflow) +} + +func TestObserverKeepsMultiStatementCommitBoundaries(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "boundaries.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (?,'one'); INSERT INTO items (id, value) VALUES (?,'two')`, 1, 2) + require.NoError(t, err) + first := receiveBatch(t, stream) + second := receiveBatch(t, stream) + require.Len(t, first.Changes, 1) + require.Len(t, second.Changes, 1) + assert.Equal(t, int64(1), first.Changes[0].RowID) + assert.Equal(t, int64(2), second.Changes[0].RowID) +} + +func TestObserverKeepsCommittedPrefixBeforeParameterizedLaterError(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "parameterized-boundary-error.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + _, err = observed.opened.DB.ExecContext(context.Background(), + `INSERT INTO items (id, value) VALUES (?,'one'); INSERT INTO items (id, value) VALUES (?,'one')`, + 1, 2, + ) + require.Error(t, err) + select { + case batch := <-stream.Changes(): + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(1), batch.Changes[0].RowID) + case <-time.After(time.Second): + t.Fatal("committed prefix was not published") + } +} + +func TestObserverKeepsCommittedPrefixBeforeParameterizedSyntaxTail(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "parameterized-syntax-tail.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + _, err = observed.opened.DB.ExecContext(context.Background(), + `INSERT INTO items (id, value) VALUES (?, 'one'); INSER INTO items (id, value) VALUES (?, 'two')`, + 1, 2, + ) + require.Error(t, err) + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(1), batch.Changes[0].RowID) +} + +func TestObserverPublishesEarlierAutocommitBeforeLaterError(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "boundary-error.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1,'one')`) + require.NoError(t, err) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2,'one')`) + require.Error(t, err) + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(1), batch.Changes[0].RowID) + select { + case extra := <-stream.Changes(): + t.Fatalf("failed later statement was published: %#v", extra) + case <-time.After(50 * time.Millisecond): + } +} + +func TestObserverRejectsUnsupportedVirtualTable(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "virtual.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE VIRTUAL TABLE docs USING fts5(content)`) + if err != nil { + t.Skipf("sqlite build has no fts5 virtual table: %v", err) + } + _, err = observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{Tables: []string{"docs"}}) + require.Error(t, err) + assert.Contains(t, err.Error(), "virtual table") +} + +func TestObserverFailsClosedWhenVirtualTableIsCreatedDynamically(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "virtual-dynamic.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{Tables: []string{"items"}}) + require.NoError(t, err) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE VIRTUAL TABLE docs USING fts5(content)`) + if err != nil { + t.Skipf("sqlite build has no fts5 virtual table: %v", err) + } + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("observer did not fail closed for dynamic virtual table") + } + assert.Contains(t, stream.Err().Error(), "virtual table") +} + +func TestObserverDetectsDDLThroughAuthorizerWithLeadingComment(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "ddl-comment.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `/* CREATE TABLE */ CREATE TABLE other (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'value')`) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("observer did not fail closed for DDL and DML transaction") + } + assert.Error(t, stream.Err()) +} + +func receiveBatch(t *testing.T, stream interface { + Changes() <-chan config.MutationBatch +}) config.MutationBatch { + t.Helper() + select { + case batch := <-stream.Changes(): + return batch + case <-time.After(time.Second): + t.Fatal("timed out waiting for mutation batch") + return config.MutationBatch{} + } +} + +func waitForStreamCount(t *testing.T, backend *sqliteBackend, want int) { + t.Helper() + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + backend.mu.Lock() + count := len(backend.streams) + backend.mu.Unlock() + if count == want { + return + } + time.Sleep(time.Millisecond) + } + backend.mu.Lock() + count := len(backend.streams) + backend.mu.Unlock() + t.Fatalf("stream count = %d, want %d", count, want) +} diff --git a/service/sql/engine/sqlite/sqlite.go b/service/sql/engine/sqlite/sqlite.go new file mode 100644 index 000000000..754b9af31 --- /dev/null +++ b/service/sql/engine/sqlite/sqlite.go @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package sqlite implements the file-backed SQLite SQL driver. It is explicitly +// constructed by the boot graph and owns the connector/connection lifecycle for +// each pool generation. +package sqlite + +import ( + "context" + "database/sql" + "fmt" + + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" + entryutil "github.com/wippyai/runtime/system/entry" +) + +// defaultDriver is retained for diagnostics and config validation. Physical +// opens use the connector-owned driver in observer.go when the preupdate build +// tag is enabled, so no process-global driver name is replaced. +const defaultDriver = "sqlite3" + +type engine struct{} + +// NewDriver returns a SQLite SQL driver. The concrete return keeps the +// connector-owned Open capability available to composition/integration code; +// it remains assignable to service/sql.Driver wherever only the base engine +// contract is needed. +func NewDriver() engine { return engine{} } + +func (engine) Kind() registry.Kind { + return config.SQLite +} + +func (engine) DriverName() string { + return defaultDriver +} + +func (engine) DecodeConfig(ctx context.Context, dtt payload.Transcoder, entry registry.Entry) (config.EngineConfig, error) { + cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, dtt, entry) + if err != nil { + return nil, err + } + + return cfg, nil +} + +func (engine) ResolveEnv(context.Context, sqlservice.EngineDeps, config.EngineConfig) error { + return nil +} + +func (engine) Open(ctx context.Context, ec config.EngineConfig) (sqlservice.OpenedDB, error) { + dsn, err := engine{}.BuildDSN(ec) + if err != nil { + return sqlservice.OpenedDB{}, err + } + + cfg, ok := ec.(*config.SQLiteConfig) + if !ok { + return sqlservice.OpenedDB{}, sqlservice.NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), config.SQLite) + } + db, observer, err := openSQLite(ctx, dsn, cfg.MaxMutationChanges, cfg.MaxMutationBytes) + if err != nil { + return sqlservice.OpenedDB{}, sqlservice.NewConnectionPoolCreationError(err) + } + + return sqlservice.OpenedDB{DB: db, Observer: observer}, nil +} + +func (engine) BuildDSN(ec config.EngineConfig) (string, error) { + cfg, ok := ec.(*config.SQLiteConfig) + if !ok { + return "", sqlservice.NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), config.SQLite) + } + + if cfg.File == ":memory:" { + return ":memory:", nil + } + + return "file:" + cfg.File + "?mode=rwc", nil +} + +func (engine) Prepare(ctx context.Context, db *sql.DB, _ config.EngineConfig) error { + if _, err := db.ExecContext(ctx, "PRAGMA journal_mode=WAL;"); err != nil { + return sqlservice.NewWALModeError(err) + } + + return nil +} + +func (engine) Tune(db *sql.DB, ec config.EngineConfig) { + cfg, ok := ec.(*config.SQLiteConfig) + if !ok { + return + } + + // A private in-memory database is scoped to one physical connection, so + // sharing it across a pool would create multiple unrelated databases. File + // databases, however, need the configured pool width so a snapshot read + // transaction does not consume the only writer connection. + if cfg.File == ":memory:" { + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + } else { + db.SetMaxOpenConns(cfg.Pool.MaxOpen) + db.SetMaxIdleConns(cfg.Pool.MaxIdle) + } + db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) +} + +func (engine) ValidateConfigType(ec config.EngineConfig) error { + if _, ok := ec.(*config.SQLiteConfig); !ok { + return sqlservice.NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), config.SQLite) + } + + return nil +} diff --git a/service/sql/engine/sqlite/sqlite_test.go b/service/sql/engine/sqlite/sqlite_test.go new file mode 100644 index 000000000..ad7222f50 --- /dev/null +++ b/service/sql/engine/sqlite/sqlite_test.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" + "go.uber.org/zap" +) + +func TestKindDriverRegistered(t *testing.T) { + e := engine{} + assert.Equal(t, config.SQLite, e.Kind()) + assert.Equal(t, "sqlite3", e.DriverName()) + + _, _, err := sqlservice.NewDefaultPoolFactory(NewDriver()).CreatePool( + context.Background(), + sqlservice.EngineDeps{Log: zap.NewNop()}, + registry.Entry{ID: registry.NewID("t", "x"), Kind: config.SQLite, Data: nil}, + ) + require.Error(t, err) + assert.NotContains(t, err.Error(), "unsupported entry kind") +} + +func TestBuildDSN(t *testing.T) { + e := engine{} + + mem, err := e.BuildDSN(&config.SQLiteConfig{File: ":memory:"}) + require.NoError(t, err) + assert.Equal(t, ":memory:", mem) + + file, err := e.BuildDSN(&config.SQLiteConfig{File: "/tmp/app.db"}) + require.NoError(t, err) + assert.Equal(t, "file:/tmp/app.db?mode=rwc", file) + + _, err = e.BuildDSN(&config.DBConfig{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid config type") +} + +func TestPrepareEnablesWAL(t *testing.T) { + file := filepath.Join(t.TempDir(), "app.db") + db, err := sql.Open("sqlite3", "file:"+file+"?mode=rwc") + require.NoError(t, err) + defer func() { _ = db.Close() }() + + require.NoError(t, engine{}.Prepare(context.Background(), db, &config.SQLiteConfig{File: file})) + + var mode string + require.NoError(t, db.QueryRowContext(context.Background(), "PRAGMA journal_mode;").Scan(&mode)) + assert.Equal(t, "wal", mode) +} + +func TestTuneSingleWriter(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + require.NoError(t, err) + defer func() { _ = db.Close() }() + + engine{}.Tune(db, &config.SQLiteConfig{File: ":memory:", Pool: config.PoolConfig{MaxLifetime: time.Hour, MaxOpen: 4, MaxIdle: 4}}) + assert.Equal(t, 1, db.Stats().MaxOpenConnections) +} + +func TestTuneHonorsFilePoolWidth(t *testing.T) { + db, err := sql.Open("sqlite3", "file:test-tune-pool?mode=memory&cache=shared") + require.NoError(t, err) + defer func() { _ = db.Close() }() + + engine{}.Tune(db, &config.SQLiteConfig{ + File: filepath.Join(t.TempDir(), "tune.db"), + Pool: config.PoolConfig{MaxOpen: 4, MaxIdle: 3, MaxLifetime: time.Hour}, + }) + assert.Equal(t, 4, db.Stats().MaxOpenConnections) +} + +func TestValidateConfigType(t *testing.T) { + require.NoError(t, engine{}.ValidateConfigType(&config.SQLiteConfig{File: ":memory:"})) + err := engine{}.ValidateConfigType(&config.DBConfig{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid config type") +} diff --git a/service/sql/engine/standard/standard.go b/service/sql/engine/standard/standard.go new file mode 100644 index 000000000..70f6fd1ac --- /dev/null +++ b/service/sql/engine/standard/standard.go @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package standard implements the network SQL drivers (PostgreSQL and MySQL) +// that share DBConfig. Drivers are constructed explicitly by the boot graph; +// importing this package does not mutate process-global SQL state. +package standard + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/url" + "sort" + "strconv" + "strings" + + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" + entryutil "github.com/wippyai/runtime/system/entry" +) + +// engine serves a network SQL dialect. Postgres and MySQL share the same DBConfig, +// env resolution, and pool tuning, differing only in driver name and DSN format, so +// each is a separate instance carrying its own DSN builder. +type engine struct { + dsn func(*config.DBConfig) (string, error) + kind registry.Kind + driver string +} + +// NewPostgresDriver returns the PostgreSQL SQL driver. +func NewPostgresDriver() sqlservice.Driver { + return engine{kind: config.Postgres, driver: "postgres", dsn: buildPostgresDSN} +} + +// NewMySQLDriver returns the MySQL SQL driver. +func NewMySQLDriver() sqlservice.Driver { + return engine{kind: config.MySQL, driver: "mysql", dsn: buildMySQLDSN} +} + +func (e engine) Kind() registry.Kind { + return e.kind +} + +func (e engine) DriverName() string { + return e.driver +} + +func (engine) DecodeConfig(ctx context.Context, dtt payload.Transcoder, entry registry.Entry) (config.EngineConfig, error) { + cfg, err := entryutil.DecodeEntryConfig[config.DBConfig](ctx, dtt, entry) + if err != nil { + return nil, err + } + + return cfg, nil +} + +func (engine) ResolveEnv(context.Context, sqlservice.EngineDeps, config.EngineConfig) error { + return nil +} + +func (e engine) Open(_ context.Context, ec config.EngineConfig) (sqlservice.OpenedDB, error) { + dsn, err := e.BuildDSN(ec) + if err != nil { + return sqlservice.OpenedDB{}, err + } + + db, err := sql.Open(e.driver, dsn) + if err != nil { + return sqlservice.OpenedDB{}, sqlservice.NewConnectionPoolCreationError(err) + } + + return sqlservice.OpenedDB{DB: db}, nil +} + +func (e engine) BuildDSN(ec config.EngineConfig) (string, error) { + cfg, ok := ec.(*config.DBConfig) + if !ok { + return "", sqlservice.NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), e.kind) + } + + return e.dsn(cfg) +} + +func (engine) Prepare(context.Context, *sql.DB, config.EngineConfig) error { + return nil +} + +func (engine) Tune(db *sql.DB, ec config.EngineConfig) { + cfg, ok := ec.(*config.DBConfig) + if !ok { + return + } + + db.SetMaxOpenConns(cfg.Pool.MaxOpen) + db.SetMaxIdleConns(cfg.Pool.MaxIdle) + db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) +} + +func (e engine) ValidateConfigType(ec config.EngineConfig) error { + if _, ok := ec.(*config.DBConfig); !ok { + return sqlservice.NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), e.kind) + } + + return nil +} + +func buildPostgresDSN(cfg *config.DBConfig) (string, error) { + if err := validateDSNFields(cfg); err != nil { + return "", err + } + opts := buildPostgresOptionsString(cfg.Options) + var b strings.Builder + b.Grow(128) + b.WriteString("host=") + b.WriteString(quotePostgresValue(cfg.Host)) + b.WriteString(" port=") + b.WriteString(strconv.Itoa(cfg.Port)) + b.WriteString(" user=") + b.WriteString(quotePostgresValue(cfg.Username)) + b.WriteString(" password=") + b.WriteString(quotePostgresValue(cfg.Password)) + b.WriteString(" dbname=") + b.WriteString(quotePostgresValue(cfg.Database)) + if opts != "" { + b.WriteString(" ") + b.WriteString(opts) + } + + return b.String(), nil +} + +func buildMySQLDSN(cfg *config.DBConfig) (string, error) { + if err := validateDSNFields(cfg); err != nil { + return "", err + } + opts := buildMySQLOptionsString(cfg.Options) + var b strings.Builder + b.Grow(128) + b.WriteString(cfg.Username) + b.WriteString(":") + b.WriteString(cfg.Password) + b.WriteString("@tcp(") + b.WriteString(cfg.Host) + b.WriteString(":") + b.WriteString(strconv.Itoa(cfg.Port)) + b.WriteString(")/") + b.WriteString(cfg.Database) + if opts != "" { + b.WriteString("?") + b.WriteString(opts) + } + + return b.String(), nil +} + +// buildPostgresOptionsString renders lib/pq keyword/value options. +func buildPostgresOptionsString(options map[string]string) string { + if len(options) == 0 { + return "" + } + + keys := make([]string, 0, len(options)) + for k := range options { + keys = append(keys, k) + } + sort.Strings(keys) + + var b strings.Builder + b.Grow(len(options) * 20) + for i, k := range keys { + if i > 0 { + b.WriteString(" ") + } + b.WriteString(k) + b.WriteString("=") + b.WriteString(quotePostgresValue(options[k])) + } + + return b.String() +} + +func buildMySQLOptionsString(options map[string]string) string { + if len(options) == 0 { + return "" + } + + values := url.Values{} + for k, v := range options { + values.Set(k, v) + } + + return values.Encode() +} + +func validateDSNFields(cfg *config.DBConfig) error { + switch { + case cfg.Host == "": + return sqlservice.NewInvalidDSNError(errors.New("host is empty")) + case cfg.Port <= 0: + return sqlservice.NewInvalidDSNError(fmt.Errorf("port is invalid: %d", cfg.Port)) + case cfg.Username == "": + return sqlservice.NewInvalidDSNError(errors.New("username is empty")) + case cfg.Database == "": + return sqlservice.NewInvalidDSNError(errors.New("database is empty")) + } + return nil +} + +func quotePostgresValue(value string) string { + var b strings.Builder + b.Grow(len(value) + 2) + b.WriteByte('\'') + for i := 0; i < len(value); i++ { + c := value[i] + if c == '\\' || c == '\'' { + b.WriteByte('\\') + } + b.WriteByte(c) + } + b.WriteByte('\'') + return b.String() +} diff --git a/service/sql/engine/standard/standard_test.go b/service/sql/engine/standard/standard_test.go new file mode 100644 index 000000000..bb980f8df --- /dev/null +++ b/service/sql/engine/standard/standard_test.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: MPL-2.0 + +package standard + +import ( + "context" + "database/sql" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" + "go.uber.org/zap" +) + +func sqlOpenMemory(*testing.T) (*sql.DB, error) { + return sql.Open("sqlite3", ":memory:") +} + +func TestRegistered(t *testing.T) { + for _, driver := range []sqlservice.Driver{NewPostgresDriver(), NewMySQLDriver()} { + _, _, err := sqlservice.NewDefaultPoolFactory(driver).CreatePool( + context.Background(), + sqlservice.EngineDeps{Log: zap.NewNop()}, + registry.Entry{ID: registry.NewID("t", "x"), Kind: driver.Kind(), Data: nil}, + ) + require.Error(t, err) + assert.NotContains(t, err.Error(), "unsupported entry kind", "driver %s must be accepted", driver.Kind()) + } +} + +func TestBuildDSN(t *testing.T) { + tests := []struct { + name string + kind registry.Kind + cfg *config.DBConfig + expected string + }{ + { + name: "postgres", + kind: config.Postgres, + cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "testdb", Username: "user", Password: "pass", Options: map[string]string{"sslmode": "disable"}}, + expected: "host='localhost' port=5432 user='user' password='pass' dbname='testdb' sslmode='disable'", + }, + { + name: "postgres with connect timeout", + kind: config.Postgres, + cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "testdb", Username: "user", Password: "pass", Options: map[string]string{"connect_timeout": "2", "sslmode": "disable"}}, + expected: "host='localhost' port=5432 user='user' password='pass' dbname='testdb' connect_timeout='2' sslmode='disable'", + }, + { + name: "mysql", + kind: config.MySQL, + cfg: &config.DBConfig{Host: "localhost", Port: 3306, Database: "testdb", Username: "user", Password: "pass", Options: map[string]string{"charset": "utf8mb4"}}, + expected: "user:pass@tcp(localhost:3306)/testdb?charset=utf8mb4", + }, + { + name: "mysql with query options", + kind: config.MySQL, + cfg: &config.DBConfig{Host: "localhost", Port: 3306, Database: "testdb", Username: "user", Password: "pass", Options: map[string]string{"charset": "utf8mb4", "parseTime": "true", "timeout": "2s"}}, + expected: "user:pass@tcp(localhost:3306)/testdb?charset=utf8mb4&parseTime=true&timeout=2s", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := engine{kind: tt.kind} + if tt.kind == config.Postgres { + e.dsn = buildPostgresDSN + } else { + e.dsn = buildMySQLDSN + } + dsn, err := e.BuildDSN(tt.cfg) + require.NoError(t, err) + assert.Equal(t, tt.expected, dsn) + }) + } +} + +func TestBuildDSN_WrongType(t *testing.T) { + e := engine{kind: config.Postgres, dsn: buildPostgresDSN} + _, err := e.BuildDSN(&config.SQLiteConfig{File: ":memory:"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid config type") +} + +func TestDriverNameAndKind(t *testing.T) { + e := engine{kind: config.Postgres, driver: "postgres"} + assert.Equal(t, config.Postgres, e.Kind()) + assert.Equal(t, "postgres", e.DriverName()) +} + +func TestOptionsStrings(t *testing.T) { + assert.Empty(t, buildPostgresOptionsString(nil)) + assert.Equal(t, "application_name='test' connect_timeout='10' sslmode='disable'", + buildPostgresOptionsString(map[string]string{"sslmode": "disable", "connect_timeout": "10", "application_name": "test"})) + assert.Empty(t, buildMySQLOptionsString(nil)) + assert.Equal(t, "charset=utf8mb4&parseTime=true&timeout=2s", + buildMySQLOptionsString(map[string]string{"charset": "utf8mb4", "parseTime": "true", "timeout": "2s"})) +} + +func TestTuneAndValidateConfigType(t *testing.T) { + e := engine{kind: config.Postgres} + require.NoError(t, e.ValidateConfigType(&config.DBConfig{})) + require.Error(t, e.ValidateConfigType(&config.SQLiteConfig{File: ":memory:"})) + + db, err := sqlOpenMemory(t) + require.NoError(t, err) + defer func() { _ = db.Close() }() + e.Tune(db, &config.DBConfig{Pool: config.PoolConfig{MaxOpen: 7, MaxIdle: 3, MaxLifetime: time.Hour}}) + assert.Equal(t, 7, db.Stats().MaxOpenConnections) +} + +func TestQuotePostgresValue(t *testing.T) { + assert.Equal(t, "'alice'", quotePostgresValue("alice")) + assert.Equal(t, "''", quotePostgresValue("")) + assert.Equal(t, "'se cret'", quotePostgresValue("se cret")) + assert.Equal(t, `'O\'Brien'`, quotePostgresValue("O'Brien")) + assert.Equal(t, `'a\\b'`, quotePostgresValue(`a\b`)) +} + +func TestValidateDSNFields(t *testing.T) { + base := func() *config.DBConfig { + return &config.DBConfig{Host: "h", Port: 5432, Username: "u", Database: "d"} + } + + require.NoError(t, validateDSNFields(base())) + + c := base() + c.Host = "" + assert.ErrorContains(t, validateDSNFields(c), "host is empty") + + c = base() + c.Port = 0 + assert.ErrorContains(t, validateDSNFields(c), "port is invalid") + + c = base() + c.Username = "" + assert.ErrorContains(t, validateDSNFields(c), "username is empty") + + c = base() + c.Database = "" + assert.ErrorContains(t, validateDSNFields(c), "database is empty") +} diff --git a/service/sql/engines_stub_test.go b/service/sql/engines_stub_test.go new file mode 100644 index 000000000..99efc414a --- /dev/null +++ b/service/sql/engines_stub_test.go @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sql + +import ( + "context" + "database/sql" + "fmt" + + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + entryutil "github.com/wippyai/runtime/system/entry" +) + +// stubEngine is a faithful-but-minimal engine registered only for the core dispatch +// tests. The real engines live in service/sql/engine/* and cannot be imported here +// (that would be an import cycle), so these stubs exercise the registry, factory, +// manager, and ConnPool plumbing. Real DSN/env/WAL behavior is tested in the engine +// sub-packages. +type stubEngine struct { + kind registry.Kind + driver string + isSQLite bool +} + +func testDrivers() []Driver { + return []Driver{ + stubEngine{kind: config.Postgres, driver: "postgres"}, + stubEngine{kind: config.MySQL, driver: "mysql"}, + stubEngine{kind: config.SQLite, driver: "sqlite3", isSQLite: true}, + } +} + +func testDriverFor(kind registry.Kind) (Driver, bool) { + for _, driver := range testDrivers() { + if driver.Kind() == kind { + return driver, true + } + } + return nil, false +} + +func (e stubEngine) Kind() registry.Kind { + return e.kind +} + +func (e stubEngine) DriverName() string { + return e.driver +} + +func (e stubEngine) DecodeConfig(ctx context.Context, dtt payload.Transcoder, entry registry.Entry) (config.EngineConfig, error) { + if e.isSQLite { + cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, dtt, entry) + if err != nil { + return nil, err + } + return cfg, nil + } + + cfg, err := entryutil.DecodeEntryConfig[config.DBConfig](ctx, dtt, entry) + if err != nil { + return nil, err + } + return cfg, nil +} + +func (stubEngine) ResolveEnv(context.Context, EngineDeps, config.EngineConfig) error { + return nil +} + +func (e stubEngine) Open(_ context.Context, ec config.EngineConfig) (OpenedDB, error) { + dsn, err := e.BuildDSN(ec) + if err != nil { + return OpenedDB{}, err + } + db, err := sql.Open(e.driver, dsn) + if err != nil { + return OpenedDB{}, err + } + return OpenedDB{DB: db}, nil +} + +func (e stubEngine) BuildDSN(config.EngineConfig) (string, error) { + if e.isSQLite { + return ":memory:", nil + } + return "host=stub", nil +} + +func (stubEngine) Prepare(context.Context, *sql.DB, config.EngineConfig) error { + return nil +} + +func (e stubEngine) Tune(db *sql.DB, ec config.EngineConfig) { + if e.isSQLite { + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + if cfg, ok := ec.(*config.SQLiteConfig); ok { + db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) + } + return + } + + if cfg, ok := ec.(*config.DBConfig); ok { + db.SetMaxOpenConns(cfg.Pool.MaxOpen) + db.SetMaxIdleConns(cfg.Pool.MaxIdle) + db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) + } +} + +func (e stubEngine) ValidateConfigType(ec config.EngineConfig) error { + if e.isSQLite { + if _, ok := ec.(*config.SQLiteConfig); !ok { + return NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), e.kind) + } + return nil + } + + if _, ok := ec.(*config.DBConfig); !ok { + return NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), e.kind) + } + return nil +} diff --git a/service/sql/errors.go b/service/sql/errors.go index 42b00daa8..2b3e4a05a 100644 --- a/service/sql/errors.go +++ b/service/sql/errors.go @@ -52,12 +52,6 @@ func NewUnsupportedAccessModeError(mode string) apierror.Error { WithDetails(attrs.NewBagFrom(map[string]any{"mode": mode})) } -func NewUnsupportedDatabaseTypeError(kind registry.Kind) apierror.Error { - return apierror.New(apierror.Invalid, "unsupported database type"). - WithRetryable(apierror.False). - WithDetails(attrs.NewBagFrom(map[string]any{"database_type": kind})) -} - func NewConnectionPoolCreationError(err error) apierror.Error { apiErr := apierror.New(apierror.Internal, "failed to create connection pool").WithRetryable(apierror.False) if err != nil { @@ -66,14 +60,6 @@ func NewConnectionPoolCreationError(err error) apierror.Error { return apiErr } -func NewSQLiteConnectionCreationError(err error) apierror.Error { - apiErr := apierror.New(apierror.Internal, "failed to create SQLite connection").WithRetryable(apierror.False) - if err != nil { - apiErr = apiErr.WithDetails(attrs.NewBagFrom(map[string]any{"cause": err.Error()})).WithCause(err) - } - return apiErr -} - func NewWALModeError(err error) apierror.Error { apiErr := apierror.New(apierror.Internal, "failed to enable WAL mode").WithRetryable(apierror.False) if err != nil { @@ -115,11 +101,3 @@ func NewPoolUpdateError(err error) apierror.Error { } return apiErr } - -func NewSQLiteUpdateError(err error) apierror.Error { - apiErr := apierror.New(apierror.Internal, "failed to update SQLite config").WithRetryable(apierror.False) - if err != nil { - apiErr = apiErr.WithDetails(attrs.NewBagFrom(map[string]any{"cause": err.Error()})).WithCause(err) - } - return apiErr -} diff --git a/service/sql/factory.go b/service/sql/factory.go index 38e370ae8..1cc2232f1 100644 --- a/service/sql/factory.go +++ b/service/sql/factory.go @@ -4,94 +4,51 @@ package sql import ( "context" - "database/sql" "github.com/wippyai/runtime/api/registry" config "github.com/wippyai/runtime/api/service/sql" ) -// PoolFactoryAPI defines the interface for creating database connection pools -type PoolFactoryAPI interface { - // CreateStandardPool creates a connection pool for standard SQL databases (Postgres, MySQL) - CreateStandardPool(ctx context.Context, kind registry.Kind, cfg *config.DBConfig) (*ConnPool, error) - - // CreateSQLitePool creates a connection pool for SQLite databases - CreateSQLitePool(ctx context.Context, cfg *config.SQLiteConfig) (*ConnPool, error) +// Factory creates and updates connection pools. It dispatches to the engine +// registered for an entry's kind, so it never needs per-engine branches. +type Factory interface { + CreatePool(ctx context.Context, deps EngineDeps, entry registry.Entry) (*ConnPool, config.EngineConfig, error) + UpdatePool(ctx context.Context, deps EngineDeps, pool *ConnPool, entry registry.Entry) (config.EngineConfig, error) } -// DefaultPoolFactory is the default implementation of PoolFactoryAPI -type DefaultPoolFactory struct{} - -// NewDefaultPoolFactory creates a new default pool factory -func NewDefaultPoolFactory() PoolFactoryAPI { - return &DefaultPoolFactory{} +// DefaultPoolFactory dispatches entries to the drivers supplied at +// construction. It deliberately has no package-global registry. +type DefaultPoolFactory struct { + drivers map[registry.Kind]Driver } -// CreateStandardPool implements PoolFactoryAPI.CreateStandardPool -func (f *DefaultPoolFactory) CreateStandardPool(_ context.Context, kind registry.Kind, cfg *config.DBConfig) (*ConnPool, error) { - if err := cfg.Validate(); err != nil { - return nil, NewInvalidConfigError(err) +// NewDefaultPoolFactory creates a pool factory with the supplied drivers. +func NewDefaultPoolFactory(drivers ...Driver) Factory { + registered := make(map[registry.Kind]Driver, len(drivers)) + for _, driver := range drivers { + if driver != nil { + registered[driver.Kind()] = driver + } } - - db, err := openStandardDB(kind, cfg) - if err != nil { - return nil, err - } - - pool := &ConnPool{ - kind: kind, - db: db, - current: newDBGeneration(db), - status: make(chan any, 1), - } - - var cfgAny any = cfg - pool.config.Store(&cfgAny) - - return pool, nil + return &DefaultPoolFactory{drivers: registered} } -// CreateSQLitePool implements PoolFactoryAPI.CreateSQLitePool -func (f *DefaultPoolFactory) CreateSQLitePool(ctx context.Context, cfg *config.SQLiteConfig) (*ConnPool, error) { - if err := cfg.Validate(); err != nil { - return nil, NewInvalidConfigError(err) +// CreatePool implements Factory.CreatePool. +func (f *DefaultPoolFactory) CreatePool(ctx context.Context, deps EngineDeps, entry registry.Entry) (*ConnPool, config.EngineConfig, error) { + driver, ok := f.drivers[entry.Kind] + if !ok { + return nil, nil, NewUnsupportedEntryKindError(entry.Kind) } - var dsn string - - // Handle in-memory database - if cfg.File == ":memory:" { - dsn = ":memory:" - } else { - // Use the file path directly - dsn = "file:" + cfg.File + "?mode=rwc" - } - - db, err := sql.Open("sqlite3", dsn) - if err != nil { - return nil, NewSQLiteConnectionCreationError(err) - } - - // Enable WAL mode for better concurrency - if _, err := db.ExecContext(ctx, "PRAGMA journal_mode=WAL;"); err != nil { - _ = db.Close() - return nil, NewWALModeError(err) - } - - // SQLite specific settings - db.SetMaxOpenConns(1) // SQLite supports only one writer - db.SetMaxIdleConns(1) - db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) + return createPool(ctx, deps, driver, entry) +} - pool := &ConnPool{ - kind: config.SQLite, - db: db, - current: newDBGeneration(db), - status: make(chan any, 1), +// UpdatePool implements Factory.UpdatePool. +func (f *DefaultPoolFactory) UpdatePool(ctx context.Context, deps EngineDeps, pool *ConnPool, entry registry.Entry) (config.EngineConfig, error) { + driver, ok := f.drivers[entry.Kind] + if !ok { + return nil, NewUnsupportedEntryKindError(entry.Kind) } - var cfgAny any = cfg - pool.config.Store(&cfgAny) - - return pool, nil + return updatePool(ctx, deps, driver, pool, entry) } diff --git a/service/sql/factory_test.go b/service/sql/factory_test.go index 7b21637cc..92f667618 100644 --- a/service/sql/factory_test.go +++ b/service/sql/factory_test.go @@ -4,14 +4,18 @@ package sql import ( "context" + "fmt" "testing" "time" _ "github.com/mattn/go-sqlite3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/payload" "github.com/wippyai/runtime/api/registry" config "github.com/wippyai/runtime/api/service/sql" + "github.com/wippyai/runtime/api/supervisor" + "go.uber.org/zap" ) func createTestDBConfig() *config.DBConfig { @@ -32,128 +36,45 @@ func createTestDBConfig() *config.DBConfig { } } -// TestDefaultPoolFactory_BuildDSN tests DSN string building without connecting to actual databases -func TestDefaultPoolFactory_BuildDSN(t *testing.T) { - tests := []struct { - name string - kind registry.Kind - cfg *config.DBConfig - expected string - isError bool - }{ - { - name: "PostgreSQL DSN", - kind: config.Postgres, - cfg: &config.DBConfig{ - Host: "localhost", - Port: 5432, - Database: "testdb", - Username: "user", - Password: "pass", - Options: map[string]string{ - "sslmode": "disable", - }, - }, - expected: "host='localhost' port=5432 user='user' password='pass' dbname='testdb' sslmode='disable'", - isError: false, - }, - { - name: "PostgreSQL DSN with connect timeout", - kind: config.Postgres, - cfg: &config.DBConfig{ - Host: "localhost", - Port: 5432, - Database: "testdb", - Username: "user", - Password: "pass", - Options: map[string]string{ - "connect_timeout": "2", - "sslmode": "disable", - }, - }, - expected: "host='localhost' port=5432 user='user' password='pass' dbname='testdb' connect_timeout='2' sslmode='disable'", - isError: false, - }, - { - name: "MySQL DSN", - kind: config.MySQL, - cfg: &config.DBConfig{ - Host: "localhost", - Port: 3306, - Database: "testdb", - Username: "user", - Password: "pass", - Options: map[string]string{ - "charset": "utf8mb4", - }, - }, - expected: "user:pass@tcp(localhost:3306)/testdb?charset=utf8mb4", - isError: false, - }, - { - name: "MySQL DSN with query options", - kind: config.MySQL, - cfg: &config.DBConfig{ - Host: "localhost", - Port: 3306, - Database: "testdb", - Username: "user", - Password: "pass", - Options: map[string]string{ - "charset": "utf8mb4", - "parseTime": "true", - "timeout": "2s", - }, - }, - expected: "user:pass@tcp(localhost:3306)/testdb?charset=utf8mb4&parseTime=true&timeout=2s", - isError: false, - }, - { - name: "Unsupported database type", - kind: "db.unsupported", - cfg: createTestDBConfig(), - expected: "", - isError: true, - }, - { - name: "PostgreSQL DSN with empty username is rejected", - kind: config.Postgres, - cfg: &config.DBConfig{ - Host: "localhost", - Port: 5432, - Database: "testdb", - Username: "", - Password: "secret", - Options: map[string]string{ - "sslmode": "disable", - }, - }, - expected: "", - isError: true, - }, - } +// fixedTranscoder decodes registry entries into a preset configuration, letting +// factory tests exercise the engine lifecycle without a real registry payload. +type fixedTranscoder struct{ cfg any } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dsn, err := buildDSN(tt.kind, tt.cfg) +func (f fixedTranscoder) Marshal(v any) (payload.Payload, error) { + return payload.New(v), nil +} - if tt.isError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.Equal(t, tt.expected, dsn) - } - }) +func (f fixedTranscoder) Unmarshal(_ payload.Payload, v any) error { + switch target := v.(type) { + case *config.DBConfig: + if c, ok := f.cfg.(*config.DBConfig); ok { + *target = *c + return nil + } + case *config.SQLiteConfig: + if c, ok := f.cfg.(*config.SQLiteConfig); ok { + *target = *c + return nil + } } + return fmt.Errorf("unexpected decode target %T", v) } -// TestDefaultPoolFactory_CreateStandardPool tests standard pool factory methods validation -func TestDefaultPoolFactory_CreateStandardPool(t *testing.T) { - // We'll test the validation logic without actually connecting - factory := &DefaultPoolFactory{} +func (f fixedTranscoder) Transcode(p payload.Payload, format payload.Format) (payload.Payload, error) { + return payload.NewPayload(p.Data(), format), nil +} + +func depsFor(cfg any) EngineDeps { + return EngineDeps{Transcoder: fixedTranscoder{cfg: cfg}, Env: NewMockEnvRegistry(), Log: zap.NewNop()} +} + +// TestDefaultPoolFactory_CreatePoolValidation tests pool creation validation through +// the registry-backed factory. +func TestDefaultPoolFactory_CreatePoolValidation(t *testing.T) { + factory := NewDefaultPoolFactory(testDrivers()...) tests := []struct { - cfg *config.DBConfig + cfg any name string kind registry.Kind errMsg string @@ -162,50 +83,58 @@ func TestDefaultPoolFactory_CreateStandardPool(t *testing.T) { { name: "Invalid configuration - empty host", kind: config.Postgres, - cfg: &config.DBConfig{Host: "", Port: 5432, Database: "db", Username: "user", Password: "pass"}, + cfg: &config.DBConfig{Host: "", Port: 5432, Database: "db", Username: "user", Password: "pass", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, isError: true, errMsg: "invalid configuration", }, { name: "Invalid configuration - zero port", kind: config.Postgres, - cfg: &config.DBConfig{Host: "localhost", Port: 0, Database: "db", Username: "user", Password: "pass"}, + cfg: &config.DBConfig{Host: "localhost", Port: 0, Database: "db", Username: "user", Password: "pass", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, isError: true, errMsg: "invalid configuration", }, { name: "Invalid configuration - empty database", kind: config.Postgres, - cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "", Username: "user", Password: "pass"}, + cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "", Username: "user", Password: "pass", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, isError: true, errMsg: "invalid configuration", }, { name: "Invalid configuration - empty username", kind: config.Postgres, - cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "db", Username: "", Password: "pass"}, + cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "db", Username: "", Password: "pass", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, isError: true, errMsg: "invalid configuration", }, { name: "Invalid configuration - empty password", kind: config.Postgres, - cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "db", Username: "user", Password: ""}, + cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "db", Username: "user", Password: "", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, + isError: true, + errMsg: "invalid configuration", + }, + { + name: "Invalid configuration - SQLite empty file", + kind: config.SQLite, + cfg: &config.SQLiteConfig{File: "", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, isError: true, errMsg: "invalid configuration", }, { - name: "Unsupported database type", + name: "Unsupported entry kind", kind: "db.unsupported", cfg: createTestDBConfig(), isError: true, - errMsg: "invalid connection config", + errMsg: "unsupported entry kind", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - pool, err := factory.CreateStandardPool(context.Background(), tt.kind, tt.cfg) + entry := registry.Entry{ID: registry.NewID("test", "x"), Kind: tt.kind, Data: payload.New("x")} + pool, _, err := factory.CreatePool(context.Background(), depsFor(tt.cfg), entry) if tt.isError { require.Error(t, err) @@ -216,39 +145,22 @@ func TestDefaultPoolFactory_CreateStandardPool(t *testing.T) { } } -// TestDefaultPoolFactory_CreateSQLitePoolValidation tests SQLite pool validation -func TestDefaultPoolFactory_CreateSQLitePoolValidation(t *testing.T) { - factory := &DefaultPoolFactory{} - - tests := []struct { - cfg *config.SQLiteConfig - name string - errMsg string - isError bool - }{ - { - name: "Invalid configuration - empty file", - cfg: &config.SQLiteConfig{File: "", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, - isError: true, - errMsg: "invalid configuration", - }, - { - name: "Invalid configuration - zero max lifetime", - cfg: &config.SQLiteConfig{File: ":memory:", Pool: config.PoolConfig{MaxLifetime: 0}}, - isError: true, - errMsg: "invalid configuration", - }, +// TestDefaultPoolFactory_CreatePoolSQLiteSuccess exercises the full create lifecycle +// (open, WAL prepare, tune, store) for a SQLite pool. +func TestDefaultPoolFactory_CreatePoolSQLiteSuccess(t *testing.T) { + cfg := &config.SQLiteConfig{ + File: ":memory:", + Lifecycle: supervisor.LifecycleConfig{StartTimeout: time.Minute}, + Pool: config.PoolConfig{MaxLifetime: time.Hour}, } + entry := registry.Entry{ID: registry.NewID("test", "lite"), Kind: config.SQLite, Data: payload.New("x")} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pool, err := factory.CreateSQLitePool(context.Background(), tt.cfg) + pool, ec, err := NewDefaultPoolFactory(testDrivers()...).CreatePool(context.Background(), depsFor(cfg), entry) + require.NoError(t, err) + require.NotNil(t, pool) + assert.Equal(t, config.SQLite, pool.kind) + require.NotNil(t, ec) + assert.Equal(t, time.Minute, ec.LifecycleConfig().StartTimeout) - if tt.isError { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errMsg) - assert.Nil(t, pool) - } - }) - } + require.NoError(t, pool.Stop(context.Background())) } diff --git a/service/sql/manager.go b/service/sql/manager.go index 3d11f0598..fed8ca564 100644 --- a/service/sql/manager.go +++ b/service/sql/manager.go @@ -6,13 +6,12 @@ import ( "context" "sync" + envapi "github.com/wippyai/runtime/api/env" "github.com/wippyai/runtime/api/event" "github.com/wippyai/runtime/api/payload" "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/resource" - config "github.com/wippyai/runtime/api/service/sql" "github.com/wippyai/runtime/api/supervisor" - entryutil "github.com/wippyai/runtime/system/entry" "go.uber.org/zap" ) @@ -20,19 +19,48 @@ import ( type Manager struct { dtt payload.Transcoder bus event.Bus - factory PoolFactoryAPI + factory Factory + env envapi.Registry log *zap.Logger services map[registry.ID]*ConnPool mu sync.RWMutex } +// Option configures a SQL Manager. Drivers are injected at boot, matching the +// service/net composition pattern; importing a driver package has no side +// effects on other managers or pools. +type Option func(*managerOptions) + +type managerOptions struct { + drivers []Driver +} + +// WithDriver adds one or more concrete SQL drivers to the manager. +func WithDriver(drivers ...Driver) Option { + return func(opts *managerOptions) { + for _, driver := range drivers { + if driver != nil { + opts.drivers = append(opts.drivers, driver) + } + } + } +} + // NewManager creates a new SQL service manager func NewManager( dtt payload.Transcoder, bus event.Bus, log *zap.Logger, + envRegistry envapi.Registry, + opts ...Option, ) (*Manager, error) { - return NewManagerWithFactory(dtt, bus, log, NewDefaultPoolFactory()) + var options managerOptions + for _, opt := range opts { + if opt != nil { + opt(&options) + } + } + return NewManagerWithFactory(dtt, bus, log, envRegistry, NewDefaultPoolFactory(options.drivers...)) } // NewManagerWithFactory creates a new SQL service manager with the specified pool factory @@ -40,7 +68,8 @@ func NewManagerWithFactory( dtt payload.Transcoder, bus event.Bus, log *zap.Logger, - factory PoolFactoryAPI, + envRegistry envapi.Registry, + factory Factory, ) (*Manager, error) { if dtt == nil { return nil, ErrTranscoderRequired @@ -60,123 +89,57 @@ func NewManagerWithFactory( dtt: dtt, bus: bus, factory: factory, + env: envRegistry, services: make(map[registry.ID]*ConnPool), }, nil } -// Add implements registry.EntryListener -func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { - m.mu.Lock() - defer m.mu.Unlock() - - switch entry.Kind { - case config.Postgres, config.MySQL: - return m.handleStandardDBAdd(ctx, entry) - case config.SQLite: - return m.handleSQLiteAdd(ctx, entry) - default: - return NewUnsupportedEntryKindError(entry.Kind) - } -} - -// Update implements registry.EntryListener -func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { - m.mu.Lock() - defer m.mu.Unlock() - - switch entry.Kind { - case config.Postgres, config.MySQL: - return m.handleStandardDBUpdate(ctx, entry) - case config.SQLite: - return m.handleSQLiteUpdate(ctx, entry) - default: - return NewUnsupportedEntryKindError(entry.Kind) - } +// deps bundles the manager's collaborators for the engine lifecycle. +func (m *Manager) deps() EngineDeps { + return EngineDeps{Transcoder: m.dtt, Env: m.env, Log: m.log} } -// Delete implements registry.EntryListener -func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { +// Add implements registry.EntryListener +func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { m.mu.Lock() defer m.mu.Unlock() - return m.handleDBDelete(ctx, entry) -} - -func (m *Manager) handleStandardDBAdd(ctx context.Context, entry registry.Entry) error { if _, exists := m.services[entry.ID]; exists { return NewServiceExistsError(entry.ID) } - cfg, err := entryutil.DecodeEntryConfig[config.DBConfig](ctx, m.dtt, entry) - if err != nil { - return NewInvalidConfigError(err) - } - - pool, err := m.factory.CreateStandardPool(ctx, entry.Kind, cfg) + pool, cfg, err := m.factory.CreatePool(ctx, m.deps(), entry) if err != nil { - return NewConnectionPoolCreationError(err) + return err } - return m.registerService(ctx, entry, pool, cfg.Lifecycle) + return m.registerService(ctx, entry, pool, cfg.LifecycleConfig()) } -func (m *Manager) handleSQLiteAdd(ctx context.Context, entry registry.Entry) error { - if _, exists := m.services[entry.ID]; exists { - return NewServiceExistsError(entry.ID) - } - - cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, m.dtt, entry) - if err != nil { - return NewInvalidConfigError(err) - } - - pool, err := m.factory.CreateSQLitePool(ctx, cfg) - if err != nil { - return NewSQLiteConnectionCreationError(err) - } - - return m.registerService(ctx, entry, pool, cfg.Lifecycle) -} +// Update implements registry.EntryListener +func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { + m.mu.Lock() + defer m.mu.Unlock() -func (m *Manager) handleStandardDBUpdate(ctx context.Context, entry registry.Entry) error { pool, exists := m.services[entry.ID] if !exists { return NewServiceNotFoundError(entry.ID) } - cfg, err := entryutil.DecodeEntryConfig[config.DBConfig](ctx, m.dtt, entry) + cfg, err := m.factory.UpdatePool(ctx, m.deps(), pool, entry) if err != nil { - return NewInvalidConfigError(err) - } - - if err := pool.UpdateConfig(cfg); err != nil { - return NewPoolUpdateError(err) + return err } - m.updateService(ctx, entry, cfg.Lifecycle) + m.updateService(ctx, entry, cfg.LifecycleConfig()) return nil } -func (m *Manager) handleSQLiteUpdate(ctx context.Context, entry registry.Entry) error { - pool, exists := m.services[entry.ID] - if !exists { - return NewServiceNotFoundError(entry.ID) - } - - cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, m.dtt, entry) - if err != nil { - return NewInvalidConfigError(err) - } - - if err := pool.UpdateConfig(cfg); err != nil { - return NewSQLiteUpdateError(err) - } - - m.updateService(ctx, entry, cfg.Lifecycle) - return nil -} +// Delete implements registry.EntryListener +func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { + m.mu.Lock() + defer m.mu.Unlock() -func (m *Manager) handleDBDelete(ctx context.Context, entry registry.Entry) error { _, exists := m.services[entry.ID] if !exists { return NewServiceNotFoundError(entry.ID) diff --git a/service/sql/manager_test.go b/service/sql/manager_test.go index 5d9caf6cd..6de8065da 100644 --- a/service/sql/manager_test.go +++ b/service/sql/manager_test.go @@ -104,6 +104,7 @@ func NewMockConnPool(kind registry.Kind) *ConnPool { pool := &ConnPool{ kind: kind, db: db, + driver: func() Driver { d, _ := testDriverFor(kind); return d }(), status: make(chan any, 1), closed: atomic.Bool{}, } @@ -143,16 +144,21 @@ func NewMockConnPool(kind registry.Kind) *ConnPool { return pool } +type standardPoolCall struct { + Cfg *apiconfig.DBConfig + Kind registry.Kind +} + +type sqlitePoolCall struct { + Cfg *apiconfig.SQLiteConfig + Kind registry.Kind +} + // Mock factory implementation type TestPoolFactory struct { - standardPoolCalls []struct { - Cfg *apiconfig.DBConfig - Kind registry.Kind - } - sqlitePoolCalls []struct { - Cfg *apiconfig.SQLiteConfig - } - shouldFailNext bool + standardPoolCalls []standardPoolCall + sqlitePoolCalls []sqlitePoolCall + shouldFailNext bool } func NewTestPoolFactory() *TestPoolFactory { @@ -162,32 +168,58 @@ func NewTestPoolFactory() *TestPoolFactory { } } -func (f *TestPoolFactory) CreateStandardPool(_ context.Context, kind registry.Kind, cfg *apiconfig.DBConfig) (*ConnPool, error) { - f.standardPoolCalls = append(f.standardPoolCalls, struct { - Cfg *apiconfig.DBConfig - Kind registry.Kind - }{ - Kind: kind, - Cfg: cfg, - }) - +func (f *TestPoolFactory) CreatePool(ctx context.Context, deps EngineDeps, entry registry.Entry) (*ConnPool, apiconfig.EngineConfig, error) { if f.shouldFailNext { - return nil, assert.AnError + return nil, nil, assert.AnError } - return NewMockConnPool(kind), nil -} -func (f *TestPoolFactory) CreateSQLitePool(_ context.Context, cfg *apiconfig.SQLiteConfig) (*ConnPool, error) { - f.sqlitePoolCalls = append(f.sqlitePoolCalls, struct { - Cfg *apiconfig.SQLiteConfig - }{ - Cfg: cfg, - }) + eng, ok := testDriverFor(entry.Kind) + if !ok { + return nil, nil, NewUnsupportedEntryKindError(entry.Kind) + } + cfg, err := eng.DecodeConfig(ctx, deps.Transcoder, entry) + if err != nil { + return nil, nil, err + } + if err := eng.ResolveEnv(ctx, deps, cfg); err != nil { + return nil, nil, err + } + if err := cfg.Validate(); err != nil { + return nil, nil, NewInvalidConfigError(err) + } + + if c, ok := cfg.(*apiconfig.SQLiteConfig); ok { + f.sqlitePoolCalls = append(f.sqlitePoolCalls, sqlitePoolCall{Kind: entry.Kind, Cfg: c}) + } else if c, ok := cfg.(*apiconfig.DBConfig); ok { + f.standardPoolCalls = append(f.standardPoolCalls, standardPoolCall{Kind: entry.Kind, Cfg: c}) + } + pool := NewMockConnPool(entry.Kind) + var cfgAny any = cfg + pool.config.Store(&cfgAny) + return pool, cfg, nil +} + +func (f *TestPoolFactory) UpdatePool(ctx context.Context, deps EngineDeps, pool *ConnPool, entry registry.Entry) (apiconfig.EngineConfig, error) { if f.shouldFailNext { return nil, assert.AnError } - return NewMockConnPool(apiconfig.SQLite), nil + + eng, ok := testDriverFor(entry.Kind) + if !ok { + return nil, NewUnsupportedEntryKindError(entry.Kind) + } + cfg, err := eng.DecodeConfig(ctx, deps.Transcoder, entry) + if err != nil { + return nil, err + } + if err := eng.ResolveEnv(ctx, deps, cfg); err != nil { + return nil, err + } + if err := pool.updateConfig(ctx, eng, cfg); err != nil { + return nil, err + } + return cfg, nil } // MockEnvRegistry implements envapi.Registry for testing @@ -255,7 +287,7 @@ func newTestManager(t *testing.T) (*Manager, event.Bus, *TestPoolFactory) { transcoder := &TestTranscoder{} factory := NewTestPoolFactory() - manager, err := NewManagerWithFactory(transcoder, bus, logger, factory) + manager, err := NewManagerWithFactory(transcoder, bus, logger, NewMockEnvRegistry(), factory) require.NoError(t, err) return manager, bus, factory } @@ -267,7 +299,7 @@ func TestNewManagerWithFactory(t *testing.T) { factory := NewTestPoolFactory() t.Run("Valid initialization", func(t *testing.T) { - manager, err := NewManagerWithFactory(transcoder, bus, logger, factory) + manager, err := NewManagerWithFactory(transcoder, bus, logger, NewMockEnvRegistry(), factory) assert.NoError(t, err) assert.NotNil(t, manager) assert.Equal(t, logger, manager.log) @@ -278,21 +310,21 @@ func TestNewManagerWithFactory(t *testing.T) { }) t.Run("Nil transcoder", func(t *testing.T) { - manager, err := NewManagerWithFactory(nil, bus, logger, factory) + manager, err := NewManagerWithFactory(nil, bus, logger, NewMockEnvRegistry(), factory) require.Error(t, err) assert.Nil(t, manager) assert.Contains(t, err.Error(), "transcoder is required") }) t.Run("Nil event bus", func(t *testing.T) { - manager, err := NewManagerWithFactory(transcoder, nil, logger, factory) + manager, err := NewManagerWithFactory(transcoder, nil, logger, NewMockEnvRegistry(), factory) require.Error(t, err) assert.Nil(t, manager) assert.Contains(t, err.Error(), "event bus is required") }) t.Run("Nil factory", func(t *testing.T) { - manager, err := NewManagerWithFactory(transcoder, bus, logger, nil) + manager, err := NewManagerWithFactory(transcoder, bus, logger, NewMockEnvRegistry(), nil) require.Error(t, err) assert.Nil(t, manager) assert.Contains(t, err.Error(), "pool factory is required") @@ -350,7 +382,12 @@ func TestManager_Add(t *testing.T) { id registry.ID shouldFail bool expectSuccess bool - }{} + }{ + {name: "add postgres", kind: apiconfig.Postgres, id: registry.NewID("test", "add-pg"), shouldFail: false, expectSuccess: true}, + {name: "add sqlite", kind: apiconfig.SQLite, id: registry.NewID("test", "add-lite"), shouldFail: false, expectSuccess: true}, + {name: "add failure", kind: apiconfig.Postgres, id: registry.NewID("test", "add-fail"), shouldFail: true, expectSuccess: false}, + {name: "unsupported kind", kind: "db.unsupported", id: registry.NewID("test", "add-bad"), shouldFail: false, expectSuccess: false}, + } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/service/terminal/host.go b/service/terminal/host.go index 97394c18a..16b5f8766 100644 --- a/service/terminal/host.go +++ b/service/terminal/host.go @@ -256,10 +256,23 @@ func (h *Host) Terminate(_ context.Context, processID pid.PID) error { // Send implements relay.Receiver. func (h *Host) Send(pkg *relay.Package) error { + return h.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender through the actor scheduler. +// Admission is non-blocking, so cancellation never requires a detached +// delivery goroutine. +func (h *Host) SendContext(ctx context.Context, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } if h.shutdown.Load() { return ErrHostShuttingDown } - return h.scheduler.Send(pkg) + return h.scheduler.SendContext(ctx, pkg) } // Start implements supervisor.Service. diff --git a/system/cdc/registry.go b/system/cdc/registry.go new file mode 100644 index 000000000..e7ab3f72b --- /dev/null +++ b/system/cdc/registry.go @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package cdc contains the process-local registry of configured CDC sources. +// It deliberately knows nothing about source construction or any particular +// database driver; service/cdc owns that responsibility. +package cdc + +import ( + "errors" + "reflect" + "sort" + "sync" + + "github.com/wippyai/runtime/api/registry" + api "github.com/wippyai/runtime/api/service/cdc" + "go.uber.org/zap" +) + +var ( + ErrSourceExists = errors.New("cdc source already registered") + ErrSourceMissing = errors.New("cdc source not registered") +) + +type entry struct { + source api.Source + kind registry.Kind +} + +// Registry is a concurrency-safe, driver-neutral source registry. The map is +// keyed only by canonical registry.ID values. Drivers must not introduce +// process-wide aliases such as PostgreSQL slot names into this layer. +type Registry struct { + log *zap.Logger + sources map[registry.ID]entry + mu sync.RWMutex +} + +func NewRegistry(log *zap.Logger) *Registry { + if log == nil { + log = zap.NewNop() + } + return &Registry{ + log: log, + sources: make(map[registry.ID]entry), + } +} + +// Register adds a source. It does not replace an existing source; callers +// must use Replace for an update so an accidental duplicate cannot orphan a +// running source. +func (r *Registry) Register(id registry.ID, source api.Source, kind registry.Kind) error { + if nilSource(source) { + return errors.New("cdc source is nil") + } + id = canonicalID(id) + + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.sources[id]; exists { + return ErrSourceExists + } + r.sources[id] = entry{source: source, kind: kind} + r.log.Debug("cdc source registered", zap.String("id", id.String()), zap.String("kind", kind)) + return nil +} + +// Replace atomically makes source visible under id and returns the previously +// visible source. The old source is not stopped here: lifecycle ownership stays +// with service/cdc, which can stop it after the visibility swap. +func (r *Registry) Replace(id registry.ID, source api.Source, kind registry.Kind) (api.Source, bool, error) { + if nilSource(source) { + return nil, false, errors.New("cdc source is nil") + } + id = canonicalID(id) + + r.mu.Lock() + old, exists := r.sources[id] + if !exists { + r.mu.Unlock() + return nil, false, ErrSourceMissing + } + r.sources[id] = entry{source: source, kind: kind} + r.mu.Unlock() + + r.log.Info("cdc source replaced", zap.String("id", id.String()), zap.String("kind", kind)) + return old.source, true, nil +} + +// Unregister removes a source and returns it to the lifecycle owner. +func (r *Registry) Unregister(id registry.ID) (api.Source, bool) { + id = canonicalID(id) + r.mu.Lock() + old, exists := r.sources[id] + if exists { + delete(r.sources, id) + } + r.mu.Unlock() + if exists { + r.log.Debug("cdc source unregistered", zap.String("id", id.String())) + return old.source, true + } + return nil, false +} + +func (r *Registry) Get(id registry.ID) (api.Source, bool) { + id = canonicalID(id) + r.mu.RLock() + item, ok := r.sources[id] + r.mu.RUnlock() + if !ok { + return nil, false + } + return item.source, true +} + +// List returns a deterministic snapshot. Metadata from the registry is merged +// over the source's own Info so a source cannot accidentally publish a +// different global identity or kind. +func (r *Registry) List() []api.SourceInfo { + r.mu.RLock() + items := make([]struct { + source api.Source + id registry.ID + kind registry.Kind + }, 0, len(r.sources)) + for id, item := range r.sources { + items = append(items, struct { + source api.Source + id registry.ID + kind registry.Kind + }{id: id, kind: item.kind, source: item.source}) + } + r.mu.RUnlock() + + sort.Slice(items, func(i, j int) bool { + return items[i].id.String() < items[j].id.String() + }) + out := make([]api.SourceInfo, 0, len(items)) + for _, item := range items { + info := item.source.Info() + info.ID = item.id + info.Kind = item.kind + info.Name = item.id.String() + out = append(out, info) + } + return out +} + +func canonicalID(id registry.ID) registry.ID { + return registry.ParseID(id.String()) +} + +func nilSource(source api.Source) bool { + if source == nil { + return true + } + v := reflect.ValueOf(source) + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return v.IsNil() + default: + return false + } +} + +var _ api.Registry = (*Registry)(nil) diff --git a/system/cdc/registry_test.go b/system/cdc/registry_test.go new file mode 100644 index 000000000..bf04130c3 --- /dev/null +++ b/system/cdc/registry_test.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "context" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/registry" + api "github.com/wippyai/runtime/api/service/cdc" +) + +type testSource struct { + info api.SourceInfo +} + +func (s *testSource) Info() api.SourceInfo { return s.info } + +func (s *testSource) Subscribe(context.Context, api.StreamOptions) (api.Stream, error) { + return nil, nil +} + +func newTestSource(name string) *testSource { + return &testSource{info: api.SourceInfo{Name: name}} +} + +func TestRegistryCanonicalIDAndDuplicateProtection(t *testing.T) { + r := NewRegistry(nil) + id := registry.NewID("app", "events") + representation := registry.ID{NS: "app", Name: "events"} + source := newTestSource("wrong-name") + + require.NoError(t, r.Register(id, source, "db.cdc.test")) + got, ok := r.Get(representation) + require.True(t, ok) + require.Same(t, source, got) + require.ErrorIs(t, r.Register(id, newTestSource("other"), "db.cdc.test"), ErrSourceExists) +} + +func TestRegistryRejectsTypedNilSource(t *testing.T) { + r := NewRegistry(nil) + id := registry.NewID("app", "events") + + var nilSource *testSource + require.Error(t, r.Register(id, nilSource, "db.cdc.test")) + + actual := newTestSource("actual") + require.NoError(t, r.Register(id, actual, "db.cdc.test")) + _, replaced, err := r.Replace(id, nilSource, "db.cdc.test") + require.Error(t, err) + require.False(t, replaced) + got, ok := r.Get(id) + require.True(t, ok) + require.Same(t, actual, got) +} + +func TestRegistryReplaceIsAtomicAndReturnsOld(t *testing.T) { + r := NewRegistry(nil) + id := registry.NewID("app", "events") + old := newTestSource("old") + newSource := newTestSource("new") + require.NoError(t, r.Register(id, old, "db.cdc.old")) + + previous, ok, err := r.Replace(registry.ParseID(id.String()), newSource, "db.cdc.new") + require.NoError(t, err) + require.True(t, ok) + require.Same(t, old, previous) + current, ok := r.Get(id) + require.True(t, ok) + require.Same(t, newSource, current) + + missing := registry.NewID("app", "missing") + _, replaced, err := r.Replace(missing, newTestSource("candidate"), "db.cdc.test") + require.ErrorIs(t, err, ErrSourceMissing) + require.False(t, replaced) + _, exists := r.Get(missing) + require.False(t, exists) +} + +func TestRegistryListIsSortedAndOverlaysIdentity(t *testing.T) { + r := NewRegistry(nil) + require.NoError(t, r.Register(registry.NewID("app", "z"), newTestSource("z"), "db.cdc.z")) + require.NoError(t, r.Register(registry.NewID("app", "a"), newTestSource("a"), "db.cdc.a")) + + infos := r.List() + require.Len(t, infos, 2) + require.Equal(t, "app:a", infos[0].ID.String()) + require.Equal(t, registry.Kind("db.cdc.a"), infos[0].Kind) + require.Equal(t, "app:a", infos[0].Name) + require.Equal(t, "app:z", infos[1].ID.String()) +} + +func TestRegistryConcurrentReplaceAndGet(t *testing.T) { + r := NewRegistry(nil) + id := registry.NewID("app", "events") + require.NoError(t, r.Register(id, newTestSource("initial"), "db.cdc.test")) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for j := 0; j < 100; j++ { + candidate := newTestSource("candidate") + _, _, _ = r.Replace(id, candidate, registry.Kind("db.cdc.test")) + _, _ = r.Get(id) + _ = i + } + }(i) + } + wg.Wait() + _, ok := r.Get(id) + require.True(t, ok) +} diff --git a/system/relay/errors.go b/system/relay/errors.go index ef5aa80e6..15040a86c 100644 --- a/system/relay/errors.go +++ b/system/relay/errors.go @@ -9,10 +9,21 @@ import ( ) var ( - ErrNilPackage = apierror.New(apierror.Invalid, "cannot send nil package").WithRetryable(apierror.False) - ErrAlreadyAttached = apierror.New(apierror.AlreadyExists, "receiver already attached").WithRetryable(apierror.False) + ErrNilPackage = apierror.New(apierror.Invalid, "cannot send nil package").WithRetryable(apierror.False) + ErrAlreadyAttached = apierror.New(apierror.AlreadyExists, "receiver already attached").WithRetryable(apierror.False) + ErrContextUnsupported = apierror.New(apierror.Unavailable, "receiver does not support cancellable delivery").WithRetryable(apierror.False) ) +// NewContextUnsupportedError identifies a host that cannot bind delivery to +// a caller context. SendContext must fail rather than invoke a legacy Send +// method that may block past the caller's lifecycle. +func NewContextUnsupportedError(hostID pid.HostID, nodeID pid.NodeID) apierror.Error { + return apierror.New(apierror.Unavailable, "receiver does not support cancellable delivery"). + WithRetryable(apierror.False). + WithDetails(attrs.NewBagFrom(map[string]any{"host_id": hostID, "node_id": nodeID})). + WithCause(ErrContextUnsupported) +} + // NewInvalidHostTypeError creates an error when host has invalid type. func NewInvalidHostTypeError(hostID pid.HostID, nodeID pid.NodeID) apierror.Error { return apierror.New(apierror.Internal, "invalid host type"). diff --git a/system/relay/mailbox.go b/system/relay/mailbox.go index 17c759692..ec67589fe 100644 --- a/system/relay/mailbox.go +++ b/system/relay/mailbox.go @@ -45,15 +45,68 @@ func WithLogger(logger *zap.Logger) MailboxOption { // Mailbox implements a local message relay with asynchronous delivery. // It routes packages to attached receivers via worker goroutines. type Mailbox struct { - config mailboxConfig - ctx context.Context - receivers sync.Map - jobQueues []chan *api.Package + config mailboxConfig + ctx context.Context + receivers sync.Map + jobQueues []chan mailboxJob + lifecycle sync.RWMutex + admissions sync.WaitGroup + closed bool +} + +type mailboxJob struct { + pkg *api.Package + receiver *mailboxReceiver +} + +// mailboxReceiver is one attachment incarnation. Deliveries hold an active +// reference while they are sending to the channel; Detach marks the +// incarnation closed, waits for those sends to finish, then drains anything +// they accepted. This makes a buffered channel safe to detach and reattach +// without letting an old delivery strand a package in the old channel. +type mailboxReceiver struct { + ch chan *api.Package + done chan struct{} + active sync.WaitGroup + mu sync.Mutex + detached bool +} + +func newMailboxReceiver(ch chan *api.Package) *mailboxReceiver { + return &mailboxReceiver{ch: ch, done: make(chan struct{})} +} + +func (r *mailboxReceiver) begin() bool { + r.mu.Lock() + defer r.mu.Unlock() + if r.detached { + return false + } + r.active.Add(1) + return true +} + +func (r *mailboxReceiver) end() { + r.active.Done() +} + +func (r *mailboxReceiver) stop() { + r.mu.Lock() + if !r.detached { + r.detached = true + close(r.done) + } + r.mu.Unlock() + r.active.Wait() } // NewMailbox creates a new Mailbox instance with the provided options. // The supplied context will cancel all workers when done. func NewMailbox(ctx context.Context, opts ...MailboxOption) *Mailbox { + if ctx == nil { + ctx = context.Background() + } + config := mailboxConfig{ workerCount: 1, logger: zap.NewNop(), @@ -67,9 +120,9 @@ func NewMailbox(ctx context.Context, opts ...MailboxOption) *Mailbox { config.workerCount = 1 } - jobQueues := make([]chan *api.Package, config.workerCount) + jobQueues := make([]chan mailboxJob, config.workerCount) for i := 0; i < config.workerCount; i++ { - jobQueues[i] = make(chan *api.Package, config.bufferSize) + jobQueues[i] = make(chan mailboxJob, config.bufferSize) } m := &Mailbox{ @@ -99,8 +152,14 @@ func hashString(s string) uint32 { // Attach attaches a receiver channel for Package messages. // Only one receiver may be attached per PID; if one already exists, an error is returned. func (m *Mailbox) Attach(p pid.PID, ch chan *api.Package) (context.CancelFunc, error) { + m.lifecycle.RLock() + defer m.lifecycle.RUnlock() + if m.closed { + return nil, m.ctx.Err() + } key := p.String() - _, loaded := m.receivers.LoadOrStore(key, ch) + receiver := newMailboxReceiver(ch) + _, loaded := m.receivers.LoadOrStore(key, receiver) if loaded { m.config.logger.Warn("attempt to attach an already existing package receiver", zap.String("pid", key), @@ -109,24 +168,58 @@ func (m *Mailbox) Attach(p pid.PID, ch chan *api.Package) (context.CancelFunc, e return nil, NewAlreadyAttachedError(p) } - return func() { m.receivers.Delete(key) }, nil + return func() { m.detach(key, receiver) }, nil } // Detach removes a receiver channel from a pid. func (m *Mailbox) Detach(p pid.PID) { key := p.String() - m.receivers.Delete(key) + m.detach(key, nil) m.config.logger.Debug("receiver detached", zap.String("pid", key)) } +// detach removes one receiver incarnation and waits for all deliveries that +// loaded it before the removal. expected is used by Attach's cancellation +// callback so an old callback cannot detach a newer reattachment. +func (m *Mailbox) detach(key string, expected *mailboxReceiver) { + m.lifecycle.Lock() + var receiver *mailboxReceiver + if value, ok := m.receivers.Load(key); ok { + current, valid := value.(*mailboxReceiver) + if valid && (expected == nil || current == expected) { + m.receivers.Delete(key) + receiver = current + } + } + m.lifecycle.Unlock() + if receiver == nil { + return + } + receiver.stop() + drainReceiver(receiver.ch) +} + // Send enqueues a package for delivery. Messages from the same source // are routed to the same worker to preserve per-sender FIFO ordering. func (m *Mailbox) Send(pkg *api.Package) error { + return m.SendContext(context.Background(), pkg) +} + +// SendContext enqueues a package until either the mailbox or caller context +// is canceled. The caller context is owned by the delivery operation; the +// mailbox context remains the lifecycle boundary for its workers. +func (m *Mailbox) SendContext(ctx context.Context, pkg *api.Package) error { if pkg == nil { return NewNilPackageError() } + if ctx == nil { + ctx = context.Background() + } + + if err := ctx.Err(); err != nil { + return err + } - // Check context before attempting to send to avoid sending to closed channels if err := m.ctx.Err(); err != nil { m.config.logger.Warn("send after mailbox shutdown", zap.String("pid", pkg.Target.String())) return err @@ -135,9 +228,36 @@ func (m *Mailbox) Send(pkg *api.Package) error { // Hash by Source.UniqID to preserve per-sender ordering workerIndex := int(hashString(pkg.Source.UniqID)) % m.config.workerCount + // The lifecycle read lock linearizes the receiver snapshot and admission + // with worker shutdown. It is released before the queue select: a full queue + // must not prevent Detach or shutdown from canceling the sender. + m.lifecycle.RLock() + if m.closed { + m.lifecycle.RUnlock() + return m.ctx.Err() + } + m.admissions.Add(1) + targetKey := pkg.Target.String() + + // Capture the attachment incarnation under the same lock as admission. A + // worker must never look up the target again after Detach/reattach, or a + // package accepted for an old channel could be delivered to a new one. + var receiver *mailboxReceiver + if value, ok := m.receivers.Load(targetKey); ok { + receiver, _ = value.(*mailboxReceiver) + } + job := mailboxJob{pkg: pkg, receiver: receiver} + defer func() { + m.lifecycle.Lock() + m.admissions.Done() + m.lifecycle.Unlock() + }() + m.lifecycle.RUnlock() select { - case m.jobQueues[workerIndex] <- pkg: + case m.jobQueues[workerIndex] <- job: return nil + case <-ctx.Done(): + return ctx.Err() case <-m.ctx.Done(): m.config.logger.Warn("send after mailbox shutdown", zap.String("pid", pkg.Target.String())) return m.ctx.Err() @@ -152,19 +272,69 @@ func (m *Mailbox) worker(queueIndex int) { for { select { - case pkg := <-queue: - m.deliver(pkg) + case job := <-queue: + m.deliver(job) case <-m.ctx.Done(): + m.shutdown() return } } } +// shutdown closes admission and releases packages still waiting in every +// worker queue. The queues remain open because SendContext may be racing the +// queue send; the admission count makes that race deterministic. +func (m *Mailbox) shutdown() { + m.lifecycle.Lock() + if m.closed { + m.lifecycle.Unlock() + return + } + m.closed = true + var receivers []*mailboxReceiver + m.receivers.Range(func(_, value any) bool { + if receiver, ok := value.(*mailboxReceiver); ok { + receivers = append(receivers, receiver) + } + return true + }) + m.lifecycle.Unlock() + + // SendContext may have captured an attachment and be waiting for queue + // capacity. Closed admission prevents new senders from entering; wait for + // existing senders before draining so every accepted job has a clear owner. + m.admissions.Wait() + for _, queue := range m.jobQueues { + drain: + for { + select { + case job := <-queue: + api.ReleasePackage(job.pkg) + default: + break drain + } + } + } + + // Stop every attachment outside lifecycle so Detach/Attach do not hold the + // global lock across a potentially blocked channel send. Each receiver's + // own active counter makes this wait finite once the mailbox context is + // canceled. + for _, receiver := range receivers { + receiver.stop() + drainReceiver(receiver.ch) + } +} + // deliver sends the package to the target's receiver channel. -func (m *Mailbox) deliver(pkg *api.Package) { +func (m *Mailbox) deliver(job mailboxJob) { + pkg := job.pkg + if pkg == nil { + return + } targetKey := pkg.Target.String() - rec, ok := m.receivers.Load(targetKey) - if !ok { + receiver := job.receiver + if receiver == nil { var topic string if len(pkg.Messages) > 0 { topic = pkg.Messages[0].Topic @@ -173,36 +343,66 @@ func (m *Mailbox) deliver(pkg *api.Package) { zap.String("target", targetKey), zap.String("source", pkg.Source.String()), zap.String("topic", topic)) + api.ReleasePackage(pkg) return } - ch, ok := rec.(chan *api.Package) - if !ok { - m.config.logger.Error("receiver has invalid type", - zap.String("target", targetKey)) + if !receiver.begin() { + api.ReleasePackage(pkg) return } - - m.deliverTo(ch, pkg, targetKey) + defer receiver.end() + m.deliverTo(receiver, pkg, targetKey) } // deliverTo performs the receiver-channel send. The channel is owned by the -// attached process, which may close it concurrently with this send (Detach only -// removes the map entry). A send on a closed channel panics, which must never -// take down the worker, so it is recovered: a closed receiver means the process -// is gone and the package is dropped. -func (m *Mailbox) deliverTo(ch chan *api.Package, pkg *api.Package, targetKey string) { +// attached process, which may close it concurrently with this send. Detach +// cancels the receiver-local delivery context; a send on a closed channel +// panics, which must never take down the worker, so it is recovered: a closed +// receiver means the process is gone and the package is dropped. +func (m *Mailbox) deliverTo(receiver *mailboxReceiver, pkg *api.Package, targetKey string) { + delivered := false defer func() { if r := recover(); r != nil { m.config.logger.Debug("dropped delivery to closed receiver", zap.String("target", targetKey)) } + if !delivered { + api.ReleasePackage(pkg) + } }() + if err := m.ctx.Err(); err != nil { + m.config.logger.Debug("delivery canceled", + zap.String("target", targetKey), zap.Error(err)) + return + } + select { - case ch <- pkg: + case receiver.ch <- pkg: + delivered = true + case <-receiver.done: + m.config.logger.Debug("delivery detached", + zap.String("target", targetKey)) case <-m.ctx.Done(): m.config.logger.Debug("delivery canceled", zap.String("target", targetKey)) } } + +// drainReceiver releases packages that were already accepted by an attached +// channel after its owner detaches. Detach serializes with deliverTo, so no +// package can become unreachable between the final send and this drain. +func drainReceiver(ch chan *api.Package) { + for { + select { + case pkg, ok := <-ch: + if !ok { + return + } + api.ReleasePackage(pkg) + default: + return + } + } +} diff --git a/system/relay/mailbox_test.go b/system/relay/mailbox_test.go index c679e093a..5a6289026 100644 --- a/system/relay/mailbox_test.go +++ b/system/relay/mailbox_test.go @@ -5,6 +5,8 @@ package relay import ( "context" "fmt" + "sync" + "sync/atomic" "testing" "time" @@ -15,6 +17,18 @@ import ( "go.uber.org/zap" ) +type mailboxReleaseProbe struct{ releases atomic.Int32 } + +func (p *mailboxReleaseProbe) Release() { p.releases.Add(1) } + +func probedPackage(target pidapi.PID) (*relay.Package, *mailboxReleaseProbe) { + probe := &mailboxReleaseProbe{} + msg := relay.AcquireMessage() + msg.Topic = "probe" + msg.SetRetentionLease(probe) + return relay.NewMessagePackage(pidapi.PID{}, target, msg), probe +} + func TestMailbox_NewMailbox(t *testing.T) { ctx := context.Background() logger := zap.NewNop() @@ -66,7 +80,7 @@ func TestMailbox_Attach(t *testing.T) { // Test cancellation cancel1() time.Sleep(time.Millisecond * 10) // Allow time for the delete operation - _, exists := mailbox.receivers.Load(pid) + _, exists := mailbox.receivers.Load(pid.String()) assert.False(t, exists) } @@ -133,6 +147,23 @@ func TestMailbox_SendCancelledContext(t *testing.T) { assert.NoError(t, err) } +func TestMailbox_SendContextHonorsCallerCancellation(t *testing.T) { + mailbox := NewMailbox(context.Background(), + WithBufferSize(1), + WithWorkerCount(0), // keep the queue full for the cancellation check + ) + + target := pidapi.PID{Host: "host1", UniqID: "uniq1"} + pkg := &relay.Package{Target: target} + require.NoError(t, mailbox.Send(pkg)) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + blocked := &relay.Package{Target: target} + err := mailbox.SendContext(ctx, blocked) + assert.ErrorIs(t, err, context.Canceled) +} + func TestMailbox_NoReceiver(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() @@ -149,14 +180,158 @@ func TestMailbox_NoReceiver(t *testing.T) { } // send message without attaching a receiver - pkg := &relay.Package{ - Target: pid, - Messages: []*relay.Message{ - {Topic: "test"}, - }, - } + pkg, probe := probedPackage(pid) err := mailbox.Send(pkg) assert.NoError(t, err) // send should succeed even without receiver + assert.Eventually(t, func() bool { return probe.releases.Load() == 1 }, time.Second, time.Millisecond, + "accepted package without a receiver must be released by the mailbox") +} + +func TestMailbox_DetachReleasesBufferedPackages(t *testing.T) { + mailbox := NewMailbox(context.Background(), WithBufferSize(4), WithWorkerCount(1)) + target := pidapi.PID{Node: "node1", Host: "host1", UniqID: "detach"} + receiverCh := make(chan *relay.Package, 4) + _, err := mailbox.Attach(target, receiverCh) + require.NoError(t, err) + + pkg, probe := probedPackage(target) + require.NoError(t, mailbox.Send(pkg)) + mailbox.Detach(target) + + assert.Eventually(t, func() bool { return probe.releases.Load() == 1 }, time.Second, time.Millisecond, + "detaching a receiver must release packages already accepted by its channel") +} + +func TestMailbox_DetachReattachDoesNotCrossReceiverGeneration(t *testing.T) { + mailbox := NewMailbox(context.Background(), WithBufferSize(8), WithWorkerCount(1)) + target := pidapi.PID{Node: "node1", Host: "host1", UniqID: "generation"} + oldCh := make(chan *relay.Package, 8) + _, err := mailbox.Attach(target, oldCh) + require.NoError(t, err) + + oldPkg, oldProbe := probedPackage(target) + require.NoError(t, mailbox.Send(oldPkg)) + mailbox.Detach(target) + require.Eventually(t, func() bool { return oldProbe.releases.Load() == 1 }, time.Second, time.Millisecond) + select { + case <-oldCh: + t.Fatal("detached receiver retained a package") + default: + } + + newCh := make(chan *relay.Package, 1) + _, err = mailbox.Attach(target, newCh) + require.NoError(t, err) + newPkg, newProbe := probedPackage(target) + require.NoError(t, mailbox.Send(newPkg)) + select { + case delivered := <-newCh: + relay.ReleasePackage(delivered) + case <-time.After(time.Second): + t.Fatal("reattached receiver did not receive its package") + } + require.Equal(t, int32(1), newProbe.releases.Load()) + select { + case <-oldCh: + t.Fatal("old receiver received a package after reattach") + default: + } + mailbox.Detach(target) +} + +func TestMailbox_DetachReattachConcurrentStress(t *testing.T) { + mailbox := NewMailbox(context.Background(), WithBufferSize(32), WithWorkerCount(2)) + target := pidapi.PID{Node: "node1", Host: "host1", UniqID: "stress"} + + for round := 0; round < 100; round++ { + oldCh := make(chan *relay.Package, 32) + _, err := mailbox.Attach(target, oldCh) + require.NoError(t, err) + probes := make([]*mailboxReleaseProbe, 8) + var sends sync.WaitGroup + for i := range probes { + pkg, probe := probedPackage(target) + probes[i] = probe + sends.Add(1) + go func(pkg *relay.Package) { + defer sends.Done() + _ = mailbox.Send(pkg) + }(pkg) + } + mailbox.Detach(target) + sends.Wait() + for _, probe := range probes { + require.Eventually(t, func() bool { return probe.releases.Load() == 1 }, time.Second, time.Millisecond) + } + newCh := make(chan *relay.Package, 1) + _, err = mailbox.Attach(target, newCh) + require.NoError(t, err) + mailbox.Detach(target) + } +} + +func TestMailbox_DetachDoesNotWaitForBlockedAdmission(t *testing.T) { + mailbox := NewMailbox(context.Background(), WithBufferSize(1), WithWorkerCount(1)) + target := pidapi.PID{Node: "node1", Host: "host1", UniqID: "blocked-admission"} + _, err := mailbox.Attach(target, make(chan *relay.Package)) + require.NoError(t, err) + + packages := make([]*relay.Package, 3) + probes := make([]*mailboxReleaseProbe, 3) + for i := range packages { + packages[i], probes[i] = probedPackage(target) + } + // The first job blocks in the unbuffered receiver; the second fills the + // mailbox queue, leaving the third sender blocked on admission. + require.NoError(t, mailbox.Send(packages[0])) + require.NoError(t, mailbox.Send(packages[1])) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + errCh := make(chan error, 1) + go func() { errCh <- mailbox.SendContext(ctx, packages[2]) }() + require.Eventually(t, func() bool { + return len(mailbox.jobQueues[0]) == 1 + }, time.Second, time.Millisecond, "second package did not remain queued behind blocked delivery") + + detached := make(chan struct{}) + go func() { + mailbox.Detach(target) + close(detached) + }() + select { + case <-detached: + case <-time.After(time.Second): + t.Fatal("Detach waited on a blocked queue admission") + } + if err := <-errCh; err != nil { + relay.ReleasePackage(packages[2]) + } + for _, probe := range probes { + require.Eventually(t, func() bool { return probe.releases.Load() == 1 }, time.Second, time.Millisecond) + } +} + +func TestMailbox_ShutdownReleasesQueuedPackages(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + mailbox := NewMailbox(ctx, WithBufferSize(8), WithWorkerCount(1)) + target := pidapi.PID{Node: "node1", Host: "host1", UniqID: "shutdown"} + // Keep the worker from handing packages to a receiver; shutdown must own + // and release packages remaining in its internal queue. + packages := make([]*relay.Package, 8) + probes := make([]*mailboxReleaseProbe, 8) + for i := range packages { + packages[i], probes[i] = probedPackage(target) + require.NoError(t, mailbox.Send(packages[i])) + } + cancel() + assert.Eventually(t, func() bool { + for _, probe := range probes { + if probe.releases.Load() != 1 { + return false + } + } + return true + }, time.Second, time.Millisecond, "mailbox shutdown leaked queued packages") } func TestMailbox_DetachDuringDelivery(t *testing.T) { @@ -197,10 +372,10 @@ func TestMailbox_DetachDuringDelivery(t *testing.T) { // Message should be dropped without error } -// A receiver's owner closes its own channel on teardown (Detach only removes the -// map entry). A delivery racing that close hits a send-on-closed-channel, which -// must be dropped, never panicking the worker — proven by a subsequent delivery -// to a live receiver on the same worker still arriving. +// A receiver's owner closes its own channel on teardown. A delivery racing that +// close hits a send-on-closed-channel, which must be dropped, never panicking +// the worker — proven by a subsequent delivery to a live receiver on the same +// worker still arriving. func TestMailbox_ClosedReceiverDoesNotKillWorker(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() @@ -318,6 +493,12 @@ func TestMailbox_Shutdown(t *testing.T) { } err = mailbox.Send(pkg) assert.NoError(t, err) + select { + case delivered := <-receiverCh: + relay.ReleasePackage(delivered) + case <-time.After(time.Second): + t.Fatal("timeout waiting for pre-shutdown delivery") + } // Now cancel the mailbox context cancel() diff --git a/system/relay/node.go b/system/relay/node.go index 6940221f1..749fcb3b0 100644 --- a/system/relay/node.go +++ b/system/relay/node.go @@ -64,9 +64,26 @@ func (n *Node) GetHost(hostID pid.HostID) (api.Receiver, bool) { // Send delivers a package to its destination. The destination must be a host // registered within this node. func (n *Node) Send(pkg *api.Package) error { + return n.send(context.Background(), pkg) +} + +// SendContext delivers a package while honoring cancellation in a +// context-aware host. Hosts that do not expose ContextSender retain the +// historical synchronous receiver contract. +func (n *Node) SendContext(ctx context.Context, pkg *api.Package) error { + if ctx == nil { + ctx = context.Background() + } + return n.send(ctx, pkg) +} + +func (n *Node) send(ctx context.Context, pkg *api.Package) error { if pkg == nil { return NewNilPackageError() } + if err := ctx.Err(); err != nil { + return err + } if pkg.Target.Node != "" && pkg.Target.Node != n.nodeID { return NewExternalNodeError(pkg.Target.Node) @@ -82,6 +99,12 @@ func (n *Node) Send(pkg *api.Package) error { return NewInvalidHostTypeError(pkg.Target.Host, n.nodeID) } + if sender, ok := receiver.(api.ContextSender); ok { + return sender.SendContext(ctx, pkg) + } + if ctx.Done() != nil { + return NewContextUnsupportedError(pkg.Target.Host, n.nodeID) + } return receiver.Send(pkg) } diff --git a/system/relay/node_test.go b/system/relay/node_test.go index 96bd0c436..948542426 100644 --- a/system/relay/node_test.go +++ b/system/relay/node_test.go @@ -5,8 +5,10 @@ package relay import ( "context" "errors" + "sync" "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,6 +38,24 @@ func (d *dummyHost) Detach(_ pidapi.PID) { // No-op for testing } +type blockingHost struct { + entered chan struct{} + release chan struct{} + once sync.Once +} + +func (h *blockingHost) Send(_ *relay.Package) error { + h.once.Do(func() { close(h.entered) }) + <-h.release + return nil +} + +func (h *blockingHost) Attach(_ pidapi.PID, _ chan *relay.Package) (context.CancelFunc, error) { + return func() {}, nil +} + +func (h *blockingHost) Detach(_ pidapi.PID) {} + func TestNodeSendLocal(t *testing.T) { // Create a dummy host and register it with the node. dhost := &dummyHost{} @@ -71,6 +91,28 @@ func TestNodeSendLocal(t *testing.T) { assert.Equal(t, int32(2), dhost.sendCalled) } +func TestNodeSendContextRejectsBlockingLegacyReceiver(t *testing.T) { + host := &blockingHost{entered: make(chan struct{}), release: make(chan struct{})} + node := NewNode("node1") + require.NoError(t, node.RegisterHost("host1", host)) + pkg := &relay.Package{Target: pidapi.PID{Host: "host1", UniqID: "process"}} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + result := make(chan error, 1) + go func() { result <- node.SendContext(ctx, pkg) }() + select { + case err := <-result: + require.ErrorIs(t, err, ErrContextUnsupported) + case <-host.entered: + close(host.release) + t.Fatal("SendContext called a blocking legacy receiver") + case <-time.After(time.Second): + close(host.release) + t.Fatal("SendContext did not fail fast for legacy receiver") + } +} + func TestNodeSendHostNotFound(t *testing.T) { node := NewNode("node1") pid := pidapi.PID{ diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 415612042..df18b051a 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -12,6 +12,7 @@ import ( "github.com/wippyai/runtime/api/attrs" "github.com/wippyai/runtime/api/dispatcher" + apierror "github.com/wippyai/runtime/api/error" "github.com/wippyai/runtime/api/payload" "github.com/wippyai/runtime/api/pid" "github.com/wippyai/runtime/api/process" @@ -24,6 +25,8 @@ import ( type Option func(*Scheduler) +var errNilPackage = apierror.New(apierror.Invalid, "cannot send nil package").WithRetryable(apierror.False) + func WithWorkers(n int) Option { return func(s *Scheduler) { if n > 0 { @@ -431,6 +434,24 @@ func (s *Scheduler) ReleaseProcessor(proc *Processor) { // Send implements relay.Receiver. Routes package to target process. // Wakes the process if it's idle or blocked waiting for messages. func (s *Scheduler) Send(pkg *relay.Package) error { + return s.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender. Admission into a process queue +// is non-blocking, so cancellation is checked before the target lookup and +// before admission. Once PushMessage accepts a package, ownership transfers +// to the process queue and a later cancellation cannot undo that transfer. +func (s *Scheduler) SendContext(ctx context.Context, pkg *relay.Package) error { + if pkg == nil { + return errNilPackage + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + target := pkg.Target // copy before push - pkg may be released after queue receives it v, ok := s.byPID.Load(target.String()) @@ -453,12 +474,16 @@ func (s *Scheduler) Send(pkg *relay.Package) error { // callers holding an out-of-band snapshot never deliver to a different process // that has since inherited the slot. Returns whether the push succeeded. func (s *Scheduler) deliverToProc(proc *Processor, gen uint64, pkg *relay.Package) bool { - if !proc.queue.Push(process.Event{ + admission := proc.queue.PushMessage(process.Event{ Type: process.EventMessage, Data: pkg, - }, gen) { + }, gen) + if admission == process.MessageRejected { return false } + if admission == process.MessageDropped { + relay.ReleasePackage(pkg) + } // Wake process if waiting for messages. // CAS ensures exactly-once wake even with concurrent senders. diff --git a/system/scheduler/actor/send_context_test.go b/system/scheduler/actor/send_context_test.go new file mode 100644 index 000000000..e6389b165 --- /dev/null +++ b/system/scheduler/actor/send_context_test.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" +) + +func TestSchedulerSendContextUnknownTargetKeepsCallerOwnership(t *testing.T) { + s := NewScheduler(nil, WithWorkers(1)) + pkg := relay.NewPackage(pid.PID{}, pid.PID{UniqID: "missing"}, "test") + + err := s.SendContext(context.Background(), pkg) + require.ErrorIs(t, err, process.ErrProcessNotFound) + // A failed admission leaves the package with the caller, so it is safe to + // inspect/release it here rather than relying on a detached sender. + require.Len(t, pkg.Messages, 1) + relay.ReleasePackage(pkg) +} + +func TestSchedulerSendContextCanceledBeforeAdmission(t *testing.T) { + s := NewScheduler(nil, WithWorkers(1)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + pkg := relay.NewPackage(pid.PID{}, pid.PID{UniqID: "missing"}, "test") + + err := s.SendContext(ctx, pkg) + require.ErrorIs(t, err, context.Canceled) + require.Len(t, pkg.Messages, 1) + relay.ReleasePackage(pkg) +} + +func TestSchedulerSendRejectsStaleGeneration(t *testing.T) { + s := NewScheduler(nil, WithWorkers(1)) + target := pid.PID{UniqID: "stale-generation"} + proc, err := s.Submit(context.Background(), target, &IdleProcess{}, "", nil) + require.NoError(t, err) + + oldGeneration := proc.gen.Load() + proc.queue.Reset() + pkg := relay.NewPackage(pid.PID{}, target, "test") + require.False(t, s.deliverToProc(proc, oldGeneration, pkg)) + // The stale sender never transfers ownership. + require.Len(t, pkg.Messages, 1) + relay.ReleasePackage(pkg) + + s.completeNoPool(proc, nil, context.Canceled) + _, ok := s.byPID.Load(target.String()) + require.False(t, ok) +} + +func TestSchedulerSendContextAcceptedQueueOwnsPackage(t *testing.T) { + s := NewScheduler(nil, WithWorkers(1)) + target := pid.PID{UniqID: "accepted"} + proc, err := s.Submit(context.Background(), target, &IdleProcess{}, "", nil) + require.NoError(t, err) + + pkg := relay.NewPackage(pid.PID{}, target, "test") + require.NoError(t, s.SendContext(context.Background(), pkg)) + // Queue admission transfers ownership. Closing the queue must release the + // package exactly once and must not require the sender to release it. + proc.queue.Close() + require.Empty(t, pkg.Messages) + + s.completeNoPool(proc, nil, context.Canceled) + if err := s.SendContext(context.Background(), pkg); !errors.Is(err, process.ErrProcessNotFound) { + t.Fatalf("expected completed process to be absent, got %v", err) + } +} diff --git a/system/scheduler/pool/adaptive/adaptive.go b/system/scheduler/pool/adaptive/adaptive.go index 123958155..00b7ea80c 100644 --- a/system/scheduler/pool/adaptive/adaptive.go +++ b/system/scheduler/pool/adaptive/adaptive.go @@ -222,11 +222,22 @@ func (a *Pool) QueueLen() int { // Send implements relay.Receiver for message routing. func (a *Pool) Send(pkg *relay.Package) error { + return a.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender for message routing. +func (a *Pool) SendContext(ctx context.Context, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } v, ok := a.active.Load(pkg.Target.UniqID) if !ok { return process.ErrProcessNotFound } - return v.(*pool.Executor).Send(pkg) + return v.(*pool.Executor).SendContext(ctx, pkg) } // Call executes a function call using an available worker. diff --git a/system/scheduler/pool/inline/inline.go b/system/scheduler/pool/inline/inline.go index 5a59324c8..d6438de19 100644 --- a/system/scheduler/pool/inline/inline.go +++ b/system/scheduler/pool/inline/inline.go @@ -98,11 +98,23 @@ func (i *Pool) replaceProcess() error { // Send implements relay.Receiver. Routes package to target execution. func (i *Pool) Send(pkg *relay.Package) error { + return i.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender. Routes package to target +// execution while honoring cancellation before queue admission. +func (i *Pool) SendContext(ctx context.Context, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } v, ok := i.active.Load(pkg.Target.UniqID) if !ok { return process.ErrProcessNotFound } - return v.(*pool.Executor).Send(pkg) + return v.(*pool.Executor).SendContext(ctx, pkg) } // Start is a no-op for inline execution. diff --git a/system/scheduler/pool/lazy/lazy.go b/system/scheduler/pool/lazy/lazy.go index 7ea5db001..e67de5cb8 100644 --- a/system/scheduler/pool/lazy/lazy.go +++ b/system/scheduler/pool/lazy/lazy.go @@ -146,11 +146,23 @@ func (l *Pool) Call(ctx context.Context, method string, input payload.Payloads) // Send implements relay.Receiver. Routes package to target execution. func (l *Pool) Send(pkg *relay.Package) error { + return l.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender. Routes package to target +// execution while honoring cancellation before queue admission. +func (l *Pool) SendContext(ctx context.Context, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } v, ok := l.activeExec.Load(pkg.Target.UniqID) if !ok { return process.ErrProcessNotFound } - return v.(*pool.Executor).Send(pkg) + return v.(*pool.Executor).SendContext(ctx, pkg) } // acquire gets an idle process or creates a new one. diff --git a/system/scheduler/pool/pool.go b/system/scheduler/pool/pool.go index 6f5e9386e..e69ef1271 100644 --- a/system/scheduler/pool/pool.go +++ b/system/scheduler/pool/pool.go @@ -121,13 +121,37 @@ func (e *Executor) CompleteYield(tag uint64, data any, err error) { // Send implements relay.Receiver. Delivers message via EventQueue. // Safe to call concurrently. Messages can be queued before Run() starts. func (e *Executor) Send(pkg *relay.Package) error { - // Push message event to queue with generation check - if !e.queue.Push(process.Event{ + return e.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender. EventQueue admission is +// non-blocking, so cancellation is checked before admission. Once the queue +// accepts a package it owns it; cancellation after that point cannot revoke +// ownership or leave a detached sender behind. +func (e *Executor) SendContext(ctx context.Context, pkg *relay.Package) error { + if pkg == nil { + return process.ErrProcessNotFound + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + + // Push through the bounded message admission path. A dropped package has + // already caused one terminal event to be queued; the original pooled + // package is no longer owned by the queue and must be released here. + admission := e.queue.PushMessage(process.Event{ Type: process.EventMessage, Data: pkg, - }, e.gen.Load()) { + }, e.gen.Load()) + if admission == process.MessageRejected { return process.ErrProcessNotFound } + if admission == process.MessageDropped { + relay.ReleasePackage(pkg) + } // Signal wake select { case e.wake <- struct{}{}: diff --git a/system/scheduler/pool/static/static.go b/system/scheduler/pool/static/static.go index cfdfaade3..3f1db6e14 100644 --- a/system/scheduler/pool/static/static.go +++ b/system/scheduler/pool/static/static.go @@ -138,11 +138,22 @@ func (s *Pool) Stop() { // Send implements relay.Receiver for message routing. func (s *Pool) Send(pkg *relay.Package) error { + return s.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender for message routing. +func (s *Pool) SendContext(ctx context.Context, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } v, ok := s.active.Load(pkg.Target.UniqID) if !ok { return process.ErrProcessNotFound } - return v.(*pool.Executor).Send(pkg) + return v.(*pool.Executor).SendContext(ctx, pkg) } // Call executes a function call using an available worker. diff --git a/system/supervisor/controller.go b/system/supervisor/controller.go index 6bf77a769..63621fd76 100644 --- a/system/supervisor/controller.go +++ b/system/supervisor/controller.go @@ -98,12 +98,25 @@ func NewController( // Start initiates the service and transitions it to the running state. func (c *Controller) Start() error { + return c.startContext(context.Background()) +} + +// startContext starts the service while honoring the caller's lifecycle +// context. The controller root remains independent so a canceled supervisor +// run can still perform a bounded Stop afterward. +func (c *Controller) startContext(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } c.state.setDesiredStatus(supervisor.StatusRunning) if c.securityErr != nil { c.updateState(supervisor.StatusExited, c.securityErr) return c.securityErr } - return c.runCommand(ctrlOp{kind: ctrlStart}) + return c.runCommand(ctrlOp{kind: ctrlStart, ctx: ctx}) } // Stop gracefully stops the service and transitions it to the stopped state. @@ -139,6 +152,14 @@ func (c *Controller) cancelStart() { } } +// close releases the controller's supervision context after the service has +// been stopped or detached from the supervisor. Stop intentionally does not +// cancel this context because callers may retry a failed stop; once a +// controller is replaced or removed there is no future lifecycle work for it. +func (c *Controller) close() { + c.cancel() +} + func (c *Controller) setStartCancel(cancel context.CancelFunc) { c.startMu.Lock() c.startCancel = cancel @@ -274,6 +295,13 @@ func (c *Controller) supervise() { continue case ctrlStart: + if op.ctx != nil { + if startErr := op.ctx.Err(); startErr != nil { + c.state.setDesiredStatus(supervisor.StatusStopped) + err = startErr + break + } + } if c.state.getDesiredStatus() != supervisor.StatusRunning { err = context.Canceled break @@ -282,10 +310,29 @@ func (c *Controller) supervise() { break } ctx, cancel = context.WithCancel(c.ctx) + var stopStartPropagation func() bool + if op.ctx != nil { + stopStartPropagation = context.AfterFunc(op.ctx, cancel) + } c.setStartCancel(cancel) detailsCh, sErr := c.tryStart(ctx, cancel) + if stopStartPropagation != nil { + stopStartPropagation() + } c.clearStartCancel() if sErr != nil { + if op.ctx != nil && op.ctx.Err() != nil { + // The lifecycle operation was canceled by the supervisor + // run context. Do not leave Desired=Running, otherwise the + // retry policy can resurrect this generation after commit + // cancellation. The root controller context remains alive + // for an independent bounded Stop. + c.state.setDesiredStatus(supervisor.StatusStopped) + err = op.ctx.Err() + c.updateState(supervisor.StatusStopped, err) + respondAndCancel(err) + break + } if startCh == nil && op.result != nil { startCh = op.result op.result = nil diff --git a/system/supervisor/sequencer.go b/system/supervisor/sequencer.go index 2a79eabd9..eb3875041 100644 --- a/system/supervisor/sequencer.go +++ b/system/supervisor/sequencer.go @@ -149,7 +149,13 @@ func (sp *sequencer) processStartOperations(ctx context.Context, operations []op sp.logger.Info("starting service", zap.String("service_id", op.id)) - if err := op.controller.Start(); err != nil { + var err error + if contextual, ok := op.controller.(contextStartControllable); ok { + err = contextual.startContext(ctx) + } else { + err = op.controller.Start() + } + if err != nil { resultCh <- startResult{ serviceID: op.id, err: NewServiceStartError(op.id, err), @@ -227,6 +233,10 @@ type startStateChangeNotifier interface { startStateChanged() <-chan struct{} } +type contextStartControllable interface { + startContext(context.Context) error +} + func forwardStartStateChanges( ctx context.Context, done <-chan struct{}, diff --git a/system/supervisor/supervisor.go b/system/supervisor/supervisor.go index 02dd16f51..ed1a18e5c 100644 --- a/system/supervisor/supervisor.go +++ b/system/supervisor/supervisor.go @@ -242,6 +242,9 @@ func (s *Supervisor) StopContext(ctx context.Context) error { s.stopErr = ctx.Err() } } + for _, ctrl := range controllers { + ctrl.close() + } s.wg.Wait() @@ -619,8 +622,12 @@ func (s *Supervisor) resolveServiceDependencyRefs( return services, blockers, nil } -// execute processes the transaction by creating new services, -// stopping removed services, and starting auto-start services. +// execute processes a registry transaction through the normal lifecycle +// sequencer. A remove/register pair for one ID is a replacement: the old +// controller is stopped and detached before the new controller is created. +// This keeps controller configuration, dependency ordering, and desired-state +// handling in one lifecycle path instead of introducing an out-of-band +// reconfiguration operation. // // All iterations of tx.register, tx.remove, and s.controllers traverse a // pre-sorted slice of IDs. The supervisor feeds the sequencer in this order, @@ -630,9 +637,57 @@ func (s *Supervisor) resolveServiceDependencyRefs( func (s *Supervisor) execute(ctx context.Context, tx *regTx) (err error) { registerIDs := sortedRegisterIDs(tx.register) removeIDs := sortedRemoveIDs(tx.remove) + oldControllers := s.snapshotControllers() + oldStates := make(map[string]State, len(removeIDs)) + for _, id := range removeIDs { + if ctrl := oldControllers[id]; ctrl != nil { + oldStates[id] = ctrl.State() + } + } + + // Stop removed controllers first. In particular, a replacement must not + // expose a new controller while the old service still owns resources or + // dependency edges. A stop failure leaves the old registry untouched and + // the transaction can be retried by the caller. + stopOperations := make([]operation, 0, len(removeIDs)) + for _, id := range removeIDs { + if ctrl := oldControllers[id]; ctrl != nil { + deps, resolveErr := s.resolveDependencies(oldControllers, id) + if resolveErr != nil { + return NewDependencyResolveError(id, resolveErr) + } + stopOperations = append(stopOperations, operation{ + kind: opStop, + id: id, + controller: ctrl, + dependencies: deps, + }) + } + } + if err := s.runTransition(ctx, stopOperations); err != nil { + return NewTransitionError(err) + } + + // Detach successfully stopped controllers before constructing replacements. + // Closing the controller context releases its frame/supervision resources; + // Stop alone intentionally leaves a controller retryable for callers that + // need to recover a failed stop. + s.mu.Lock() + for _, id := range removeIDs { + if old := oldControllers[id]; old != nil && s.controllers[id] == old { + delete(s.controllers, id) + } + } + s.mu.Unlock() + for _, id := range removeIDs { + if old := oldControllers[id]; old != nil { + old.close() + } + } - // Mutate controller registry under lock, then run potentially long transitions - // lock-free so state readers are never blocked behind start/stop timeouts. + // Construct new controllers only after the stop phase has committed. This + // is also what makes remove/register a true generation handoff rather than + // silently retaining the old controller. created := make(map[string]*Controller, len(registerIDs)) s.mu.Lock() for _, id := range registerIDs { @@ -665,31 +720,24 @@ func (s *Supervisor) execute(ctx context.Context, tx *regTx) (err error) { }() controllers := s.snapshotControllers() - var operations []operation - - // Queue stop operations for services being removed - for _, id := range removeIDs { - if ctrl, exists := controllers[id]; exists { - deps, err := s.resolveDependencies(controllers, id) - if err != nil { - return NewDependencyResolveError(id, err) - } - operations = append(operations, operation{ - kind: opStop, - id: id, - controller: ctrl, - dependencies: deps, - }) - } - } - roots := make([]startRoot, 0, len(registerIDs)) for _, id := range registerIDs { entry := tx.register[id] - if entry.Config.AutoStart { + shouldStart := entry.Config.AutoStart + required := entry.Config.StartupRequired() + // Preserve a running/desired-running generation across a config or + // service replacement even when the replacement's AutoStart is false. + // This is an update of an active service, not an implicit user stop. + if previous, ok := oldStates[id]; ok && + (previous.Desired == supervisor.StatusRunning || + previous.Status == supervisor.StatusRunning || + previous.Status == supervisor.StatusStarting) { + shouldStart = true + } + if shouldStart { roots = append(roots, startRoot{ id: id, - required: entry.Config.StartupRequired(), + required: required, }) } } @@ -697,20 +745,13 @@ func (s *Supervisor) execute(ctx context.Context, tx *regTx) (err error) { if err != nil { return NewStartOperationsError(err) } - operations = append(operations, startOps...) - // Execute transitions in dependency order - if err := s.runTransition(ctx, operations); err != nil { + // Start additions and replacements through the same dependency-aware + // sequencer used by explicit ServiceStart actions. + if err := s.runTransition(ctx, startOps); err != nil { return NewTransitionError(err) } - // Done stopped services - s.mu.Lock() - for _, id := range removeIDs { - delete(s.controllers, id) - } - s.mu.Unlock() - return nil } diff --git a/system/supervisor/supervisor_test.go b/system/supervisor/supervisor_test.go index 4cd945197..04302de50 100644 --- a/system/supervisor/supervisor_test.go +++ b/system/supervisor/supervisor_test.go @@ -35,6 +35,17 @@ type testService struct { stopped bool } +type dependencyCheckingService struct { + *testService + dependency *testService + dependencyReady atomic.Bool +} + +func (s *dependencyCheckingService) Start(ctx context.Context) (<-chan any, error) { + s.dependencyReady.Store(s.dependency.IsStarted()) + return s.testService.Start(ctx) +} + type blockingStartService struct { startedCh chan struct{} releaseCh chan struct{} @@ -43,6 +54,44 @@ type blockingStartService struct { stoppedOnce sync.Once } +type cancellationAwareStartService struct { + startEntered chan struct{} + startCanceled chan struct{} + + secondAttempt chan struct{} + startOnce sync.Once + cancelOnce sync.Once + secondOnce sync.Once + startAttempts atomic.Int32 + startCompleted atomic.Bool +} + +func newCancellationAwareStartService() *cancellationAwareStartService { + return &cancellationAwareStartService{ + startEntered: make(chan struct{}), + startCanceled: make(chan struct{}), + secondAttempt: make(chan struct{}), + } +} + +func (s *cancellationAwareStartService) Start(ctx context.Context) (<-chan any, error) { + if s.startAttempts.Add(1) > 1 { + s.secondOnce.Do(func() { close(s.secondAttempt) }) + } + s.startOnce.Do(func() { close(s.startEntered) }) + <-ctx.Done() + s.cancelOnce.Do(func() { close(s.startCanceled) }) + return nil, ctx.Err() +} + +func (s *cancellationAwareStartService) Stop(context.Context) error { + return nil +} + +func (s *cancellationAwareStartService) IsStarted() bool { + return s.startCompleted.Load() +} + func newBlockingStartService() *blockingStartService { return &blockingStartService{ startedCh: make(chan struct{}), @@ -326,6 +375,163 @@ func TestSupervisor_BasicLifecycle(t *testing.T) { h.assertLog("supervisor stopped") } +func TestSupervisor_SameIDReplacementUsesNormalLifecycleTransaction(t *testing.T) { + h := newTestHarness(t) + h.start(context.Background()) + + h.registerServices(map[string]bool{ + "dependency": false, + "service": true, + }) + old := h.services["service"] + old.WaitForStart(t) + oldController := func() *Controller { + h.sup.mu.RLock() + defer h.sup.mu.RUnlock() + return h.sup.controllers["service"] + }() + + dependency := h.services["dependency"] + replacementBase := newTestService() + replacement := &dependencyCheckingService{ + testService: replacementBase, + dependency: dependency, + } + + h.sup.handleEvent(event.Event{System: registry.System, Kind: registry.TxBegin}) + h.sup.handleEvent(event.Event{System: supervisor.System, Kind: supervisor.ServiceRemove, Path: "service"}) + h.sup.handleEvent(event.Event{ + System: supervisor.System, + Kind: supervisor.ServiceRegister, + Path: "service", + Data: &supervisor.Entry{ + Service: replacement, + Config: supervisor.LifecycleConfig{ + AutoStart: false, + Requires: []string{"dependency"}, + StartTimeout: time.Second, + StopTimeout: time.Second, + }, + }, + }) + h.sup.handleEvent(event.Event{System: registry.System, Kind: registry.TxCommit}) + + replacement.WaitForStart(t) + dependency.WaitForStart(t) + old.WaitForStop(t) + require.True(t, replacement.dependencyReady.Load(), "replacement must start after its required dependency") + + h.sup.mu.RLock() + newController := h.sup.controllers["service"] + h.sup.mu.RUnlock() + require.NotSame(t, oldController, newController) + require.Same(t, replacement, newController.service) + require.False(t, newController.config.AutoStart) + state, err := h.sup.GetState("service") + require.NoError(t, err) + require.Equal(t, supervisor.StatusRunning, state.Status, "replacement preserves a running generation") + select { + case <-oldController.ctx.Done(): + default: + t.Fatal("replaced controller supervision context was not released") + } + + h.stop() +} + +func TestSupervisor_SameIDReplacementStopFailureRetainsOldController(t *testing.T) { + old := newTestService() + replacement := newTestService() + oldController := NewController(context.Background(), old, supervisor.LifecycleConfig{ + AutoStart: true, + StartTimeout: time.Second, + StopTimeout: time.Second, + }, nil) + defer oldController.close() + require.NoError(t, oldController.Start()) + old.stopErr = errors.New("old stop failed") + + bus := eventbus.NewBus() + defer bus.Stop() + sup := NewSupervisor(bus, zap.NewNop()) + id := "service" + sup.controllers[id] = oldController + tx := newRegTx(zap.NewNop()) + tx.open = true + tx.remove[id] = struct{}{} + tx.register[id] = &supervisor.Entry{ + Service: replacement, + Config: supervisor.LifecycleConfig{ + AutoStart: true, + StartTimeout: time.Second, + StopTimeout: time.Second, + RetryPolicy: supervisor.RetryPolicy{MaxAttempts: 1}, + }, + } + + err := sup.execute(context.Background(), tx) + require.Error(t, err) + sup.mu.RLock() + current := sup.controllers[id] + sup.mu.RUnlock() + require.Same(t, oldController, current, "failed replacement must retain the old controller") + require.False(t, replacement.IsStarted(), "candidate must not start after old stop fails") + + old.stopErr = nil + require.NoError(t, oldController.Stop()) +} + +func TestSupervisor_SameIDReplacementStartFailureKeepsNewController(t *testing.T) { + old := newTestService() + replacement := newTestService() + replacement.startErr = errors.New("replacement start failed") + oldController := NewController(context.Background(), old, supervisor.LifecycleConfig{ + AutoStart: true, + StartTimeout: time.Second, + StopTimeout: time.Second, + }, nil) + defer oldController.close() + require.NoError(t, oldController.Start()) + + bus := eventbus.NewBus() + defer bus.Stop() + sup := NewSupervisor(bus, zap.NewNop()) + sup.ctx = context.Background() + id := "service" + sup.controllers[id] = oldController + tx := newRegTx(zap.NewNop()) + tx.open = true + tx.remove[id] = struct{}{} + tx.register[id] = &supervisor.Entry{ + Service: replacement, + Config: supervisor.LifecycleConfig{ + AutoStart: true, + StartTimeout: time.Second, + StopTimeout: time.Second, + RetryPolicy: supervisor.RetryPolicy{MaxAttempts: 1}, + }, + } + + err := sup.execute(context.Background(), tx) + require.Error(t, err) + sup.mu.RLock() + current := sup.controllers[id] + sup.mu.RUnlock() + require.NotSame(t, oldController, current) + require.Same(t, replacement, current.service) + state := current.State() + require.Equal(t, supervisor.StatusFailed, state.Status) + require.False(t, replacement.IsStarted()) + select { + case <-oldController.ctx.Done(): + default: + t.Fatal("old controller supervision context was not released after commit") + } + + require.NoError(t, current.Stop()) + current.close() +} + func TestSupervisor_MultipleServices(t *testing.T) { h := newTestHarness(t) ctx := context.Background() @@ -977,22 +1183,48 @@ func TestSupervisor_ContextCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) h.start(ctx) - // Register a service that takes time to start - svc := h.service("slow-service") - svc.startDelay = 2 * time.Second - - h.registerServices(map[string]bool{ - "slow-service": true, + // Register a service that cannot complete until its Start context is + // canceled. Waiting for startEntered makes the cancellation race + // deterministic: the supervisor has already admitted the start operation. + svc := newCancellationAwareStartService() + h.sup.handleEvent(event.Event{System: registry.System, Kind: registry.TxBegin}) + h.sup.handleEvent(event.Event{ + System: supervisor.System, + Kind: supervisor.ServiceRegister, + Path: "slow-service", + Data: &supervisor.Entry{ + Service: svc, + Config: supervisor.LifecycleConfig{ + AutoStart: true, + StartTimeout: time.Second, + StopTimeout: time.Second, + RetryPolicy: supervisor.RetryPolicy{ + InitialDelay: 10 * time.Millisecond, + MaxDelay: 10 * time.Millisecond, + }, + }, + }, }) + h.sup.handleEvent(event.Event{System: registry.System, Kind: registry.TxCommit}) - // Cancel context while service is starting + select { + case <-svc.startEntered: + case <-time.After(time.Second): + t.Fatal("timed out waiting for service Start") + } cancel() - // Wait a bit to ensure cancellation is processed - time.Sleep(100 * time.Millisecond) - - // Verify service was not started - require.False(t, svc.IsStarted(), "Service should not be started after context cancellation") + select { + case <-svc.startCanceled: + case <-time.After(time.Second): + t.Fatal("timed out waiting for in-flight Start cancellation") + } + select { + case <-svc.secondAttempt: + t.Fatal("canceled start scheduled a retry") + case <-time.After(200 * time.Millisecond): + } + require.False(t, svc.IsStarted(), "service should not be started after context cancellation") h.stop() } diff --git a/system/supervisor/transaction.go b/system/supervisor/transaction.go index d29847113..f0abbdec6 100644 --- a/system/supervisor/transaction.go +++ b/system/supervisor/transaction.go @@ -10,17 +10,19 @@ import ( ) type regTx struct { - register map[string]*supervisor.Entry - remove map[string]struct{} - logger *zap.Logger - open bool + register map[string]*supervisor.Entry + remove map[string]struct{} + registeredBeforeRemove map[string]bool + logger *zap.Logger + open bool } func newRegTx(logger *zap.Logger) *regTx { return ®Tx{ - register: make(map[string]*supervisor.Entry), - remove: make(map[string]struct{}), - logger: logger, + register: make(map[string]*supervisor.Entry), + remove: make(map[string]struct{}), + registeredBeforeRemove: make(map[string]bool), + logger: logger, } } @@ -30,8 +32,7 @@ func (th *regTx) begin() { } th.open = true - th.register = make(map[string]*supervisor.Entry) - th.remove = make(map[string]struct{}) + th.resetChanges() } func (th *regTx) commit(removeFn func(string) error, registerFn func(string, *supervisor.Entry) error) error { @@ -88,7 +89,17 @@ func (th *regTx) registerService(id string, entry *supervisor.Entry) error { return supervisor.ErrOutsideTransaction } - delete(th.remove, id) + if _, removed := th.remove[id]; removed { + // A register following a remove is a replacement when the remove was + // already pending before this transaction saw a registration. Keep both + // operations so commit stops the old controller before installing the + // new one. A register/remove/register sequence is a canceled + // registration and retains the historical cancellation behavior. + if th.registeredBeforeRemove[id] { + delete(th.remove, id) + delete(th.registeredBeforeRemove, id) + } + } th.register[id] = entry // always use the latest entry return nil } @@ -98,19 +109,29 @@ func (th *regTx) removeService(id string) error { return supervisor.ErrOutsideTransaction } - // duplicate check + // A duplicate remove is idempotent, but a remove after a replacement's + // register cancels that new registration while retaining removal of the old + // controller. This preserves the final event in the transaction. if _, exists := th.remove[id]; exists { + delete(th.register, id) return nil } + _, registered := th.register[id] delete(th.register, id) th.remove[id] = struct{}{} + th.registeredBeforeRemove[id] = registered return nil } func (th *regTx) reset() { th.open = false + th.resetChanges() +} + +func (th *regTx) resetChanges() { th.register = make(map[string]*supervisor.Entry) th.remove = make(map[string]struct{}) + th.registeredBeforeRemove = make(map[string]bool) } diff --git a/system/supervisor/transaction_test.go b/system/supervisor/transaction_test.go index 1db3e44fb..74ef967fb 100644 --- a/system/supervisor/transaction_test.go +++ b/system/supervisor/transaction_test.go @@ -242,3 +242,27 @@ func TestTransactionHelper_RemoveService_NoTransaction(t *testing.T) { t.Error("removeService should return error outside of transaction") } } + +func TestTransactionHelper_SameIDRemoveThenRegisterIsReplacement(t *testing.T) { + th := newRegTx(noopLogger()) + th.begin() + + entry := &supervisor.Entry{} + assert.NoError(t, th.removeService("service1")) + assert.NoError(t, th.registerService("service1", entry)) + + var sequence []string + assert.NoError(t, th.commit( + func(id string) error { + sequence = append(sequence, "remove:"+id) + return nil + }, + func(id string, got *supervisor.Entry) error { + assert.Same(t, entry, got) + sequence = append(sequence, "register:"+id) + return nil + }, + )) + + assert.Equal(t, []string{"remove:service1", "register:service1"}, sequence) +} diff --git a/test.sh b/test.sh index e14ad03ba..ff5266e79 100755 --- a/test.sh +++ b/test.sh @@ -32,6 +32,7 @@ go test \ ./system/env/... \ ./service/env/... \ ./service/sql/... \ + ./service/cdc/sqlite \ ./service/cdc/postgres \ ./runtime/lua/modules/cdc \ ./runtime/lua/modules/hub \ @@ -40,6 +41,9 @@ go test \ ./boot/... \ ./system/registry/... +echo "running sqlite cdc implementation and integration tests (local temp file, no docker)" +make test-cdc-sqlite + if [[ -n "${WIPPY_CDC_IT_REPL_DSN:-}" && -n "${WIPPY_CDC_IT_ADMIN_DSN:-}" ]]; then go test -tags integration ./service/cdc/postgres exit 0