From f78e2b80d640aa1584a35d92f0bdb706b19e2aae Mon Sep 17 00:00:00 2001 From: Howard van Rooijen Date: Fri, 17 Jul 2026 08:20:58 +0100 Subject: [PATCH] Automind/Universalis neural-computer POC on Reaqtor (.NET 10) Implements Erik Meijer's Automind/Universalis papers (ACM Queue 2024/2025) as a durable neural computer: a local Ollama model as the branch predictor, a pure kernel step function intercepting generation at [...] hedge boundaries, and Nuqleon/Bonsai/Reaqtive/Reaqtor as the load-bearing substrate - a reasoning derivation is a checkpointed standing query that survives process death and resumes exactly-once. - Universalis.Core: hedge scanner/parser (+ live-derived instinct teachings and heals), literate recognizer, evaluator (patterns, zip lifting, conditionals, query comprehensions -> LINQ), contracts, Bonsai rule compilation - Automind.Kernel: pure derivation step machine, prompt assembly, tree-of-thought backtracking + restart, completion contracts, protocol repairs (phantom guard, post-answer-noise skip, escalation), Mode B whole-program synthesis through the same execution machinery - Automind.Reaqtor: checkpointing engine host, kill-safe atomic file store (version-gated snapshot skip), derivation driver with idempotency- gated recovery re-issue and transport self-heal, think-tag suppression, full OpenTelemetry instrumentation (kernel trace bridge, JSONL sink) - Automind.Mcp: MCP stdio servers' tools as Universalis predicates - invoked through the durable hedge protocol, never native function calling; specs persist and reconnect on resume - Automind.Memory: in-process ONNX embeddings (bge-micro-v2) as the RAG virtual-memory pager - Automind.Cli: Spectre repl/ask/resume/rules/learn-doc/store/demo, six paper demos, chaos kill switch, generation-visibility UX - scripts: demo-pack runner (per-demo telemetry, failed-attempt log preservation), model-swap matrix, embedding model fetch - docs/model-matrix.md: measured per-model conformance verdicts (qwen2.5-coder:7b fully conformant; gemma4 rescued by Mode B) Restores entirely from public nuget.org (Reaqtor 1.0.0-beta.24; transitive Newtonsoft.Json advisory lifted to 13.0.4). Hardened by a max-effort adversarial code review (15 verified findings fixed) and runtime-profiled clean (flat heap, 21ms total GC pause, no CPU hotspots). 176 fast tests + Ollama/embedding-gated live suites; demo pack 6/6 first-attempt. --- .gitignore | 14 + Automind.slnx | 21 + Directory.Build.props | 13 + Directory.Packages.props | 56 + NuGet.config | 7 + README.md | 188 ++ docs/model-matrix.md | 40 + global.json | 9 + scripts/fetch-embedding-model.ps1 | 27 + scripts/run-demos.ps1 | 171 ++ scripts/run-model-matrix.ps1 | 187 ++ src/Automind.Cli/Automind.Cli.csproj | 23 + src/Automind.Cli/AutomindHost.cs | 539 +++++ src/Automind.Cli/CheckpointCoordinator.cs | 113 + src/Automind.Cli/Demos.cs | 308 +++ src/Automind.Cli/OtelFileLog.cs | 133 ++ src/Automind.Cli/Program.cs | 431 ++++ src/Automind.Cli/TraceRenderer.cs | 248 +++ src/Automind.Kernel/Automind.Kernel.csproj | 7 + .../Contract/DerivationEvents.cs | 82 + .../Contract/DerivationState.cs | 145 ++ .../Contract/QuestionEnvelope.cs | 60 + src/Automind.Kernel/Contract/TraceEvents.cs | 77 + src/Automind.Kernel/DerivationStep.cs | 1809 +++++++++++++++++ .../Prompting/PromptAssembler.cs | 330 +++ src/Automind.Kernel/Prompting/PromptState.cs | 30 + src/Automind.Mcp/Automind.Mcp.csproj | 11 + src/Automind.Mcp/McpBridgedTool.cs | 69 + src/Automind.Mcp/McpPredicateMapper.cs | 179 ++ src/Automind.Mcp/McpToolBridge.cs | 147 ++ src/Automind.Memory/Automind.Memory.csproj | 18 + src/Automind.Memory/OnnxMemoryPager.cs | 164 ++ src/Automind.Reaqtor/Automind.Reaqtor.csproj | 26 + .../Catalog/ConversationCatalog.cs | 80 + .../Catalog/DocCatalogStore.cs | 62 + .../Catalog/McpCatalogStore.cs | 51 + .../Catalog/RuleCatalogStore.cs | 68 + .../ToolRegistryStepContextProvider.cs | 27 + .../Client/AutomindClientContext.cs | 47 + .../Engine/AutomindArtifacts.cs | 55 + .../Engine/AutomindEngineFactory.cs | 69 + .../Engine/AutomindServices.cs | 25 + src/Automind.Reaqtor/Engine/AutomindUris.cs | 23 + .../IO/AutomindIngressEgressManager.cs | 109 + src/Automind.Reaqtor/IO/IReliableSubject.cs | 11 + src/Automind.Reaqtor/IO/ReliableSubject.cs | 142 ++ src/Automind.Reaqtor/Llm/ILlmService.cs | 16 + src/Automind.Reaqtor/Llm/OllamaLlmService.cs | 59 + .../Llm/OllamaSegmentStreamer.cs | 176 ++ src/Automind.Reaqtor/Llm/ThinkFilter.cs | 99 + .../Reactive/ConsoleObserver.cs | 16 + .../Reactive/DerivationObservable.cs | 473 +++++ .../Reactive/DerivationOutput.cs | 43 + .../Reactive/EgressObserver.cs | 70 + .../Reactive/IngressObservable.cs | 124 ++ .../Reactive/TimerObservable.cs | 52 + .../Reactive/ToolInvokeObservable.cs | 104 + .../Store/FileQueryEngineStateStore.cs | 594 ++++++ .../Telemetry/AutomindDiagnostics.cs | 114 ++ src/Automind.Tools/Automind.Tools.csproj | 7 + src/Automind.Tools/DemoTools.cs | 281 +++ src/Automind.Tools/ITool.cs | 77 + src/Automind.Tools/PrimitiveTools.cs | 116 ++ .../Compilation/BonsaiCompiler.cs | 343 ++++ .../Compilation/BonsaiSerialization.cs | 38 + src/Universalis.Core/Evaluation/EvalEnv.cs | 76 + src/Universalis.Core/Evaluation/EvalTypes.cs | 125 ++ src/Universalis.Core/Evaluation/Evaluator.cs | 933 +++++++++ src/Universalis.Core/Evaluation/NumericOps.cs | 249 +++ .../Evaluation/PatternMatcher.cs | 262 +++ .../Evaluation/QueryPipeline.cs | 319 +++ src/Universalis.Core/Ir/IrJson.cs | 30 + src/Universalis.Core/Ir/PaperShape.cs | 168 ++ src/Universalis.Core/Ir/Program.cs | 130 ++ src/Universalis.Core/Ir/Statements.cs | 56 + src/Universalis.Core/Ir/Terms.cs | 97 + .../Parsing/ComprehensionCompiler.cs | 281 +++ src/Universalis.Core/Parsing/HedgeParser.cs | 840 ++++++++ src/Universalis.Core/Parsing/HedgeScanner.cs | 162 ++ .../Parsing/LiterateRecognizer.cs | 521 +++++ .../Parsing/UniversalisParser.cs | 68 + .../Rendering/AnswerAssembler.cs | 65 + .../Rendering/ConcreteRenderer.cs | 262 +++ src/Universalis.Core/Universalis.Core.csproj | 9 + .../Automind.Integration.Tests.csproj | 18 + .../MemoryPagerTests.cs | 86 + .../OllamaLiveTests.cs | 402 ++++ .../Automind.Kernel.Tests.csproj | 7 + .../DerivationHarness.cs | 95 + .../GoldenScenarioTests.cs | 787 +++++++ tests/Automind.Kernel.Tests/ModeBTests.cs | 242 +++ tests/Automind.Kernel.Tests/RuleTests.cs | 385 ++++ .../Automind.Mcp.Tests.csproj | 7 + .../Automind.Mcp.Tests/LoopbackBridgeTests.cs | 114 ++ .../McpPredicateMapperTests.cs | 136 ++ .../Automind.Reaqtor.Tests.csproj | 7 + .../EngineSmokeTests.cs | 106 + tests/Automind.Reaqtor.Tests/Fakes.cs | 92 + .../Automind.Reaqtor.Tests/FileStoreTests.cs | 157 ++ .../KillRecoverTests.cs | 320 +++ .../SubstrateHarness.cs | 176 ++ .../ThinkFilterTests.cs | 81 + tests/Universalis.Core.Tests/BonsaiTests.cs | 109 + .../ComprehensionTests.cs | 247 +++ .../Universalis.Core.Tests/EvaluatorTests.cs | 249 +++ .../HedgeParserTests.cs | 298 +++ .../HedgeScannerTests.cs | 106 + tests/Universalis.Core.Tests/LiftingTests.cs | 148 ++ .../LiterateRecognizerTests.cs | 175 ++ .../PatternMatcherTests.cs | 139 ++ .../Universalis.Core.Tests/RoundTripTests.cs | 139 ++ .../Universalis.Core.Tests.csproj | 7 + tools/McpSampleServer/McpSampleServer.csproj | 11 + tools/McpSampleServer/Program.cs | 32 + 114 files changed, 18784 insertions(+) create mode 100644 .gitignore create mode 100644 Automind.slnx create mode 100644 Directory.Build.props create mode 100644 Directory.Packages.props create mode 100644 NuGet.config create mode 100644 README.md create mode 100644 docs/model-matrix.md create mode 100644 global.json create mode 100644 scripts/fetch-embedding-model.ps1 create mode 100644 scripts/run-demos.ps1 create mode 100644 scripts/run-model-matrix.ps1 create mode 100644 src/Automind.Cli/Automind.Cli.csproj create mode 100644 src/Automind.Cli/AutomindHost.cs create mode 100644 src/Automind.Cli/CheckpointCoordinator.cs create mode 100644 src/Automind.Cli/Demos.cs create mode 100644 src/Automind.Cli/OtelFileLog.cs create mode 100644 src/Automind.Cli/Program.cs create mode 100644 src/Automind.Cli/TraceRenderer.cs create mode 100644 src/Automind.Kernel/Automind.Kernel.csproj create mode 100644 src/Automind.Kernel/Contract/DerivationEvents.cs create mode 100644 src/Automind.Kernel/Contract/DerivationState.cs create mode 100644 src/Automind.Kernel/Contract/QuestionEnvelope.cs create mode 100644 src/Automind.Kernel/Contract/TraceEvents.cs create mode 100644 src/Automind.Kernel/DerivationStep.cs create mode 100644 src/Automind.Kernel/Prompting/PromptAssembler.cs create mode 100644 src/Automind.Kernel/Prompting/PromptState.cs create mode 100644 src/Automind.Mcp/Automind.Mcp.csproj create mode 100644 src/Automind.Mcp/McpBridgedTool.cs create mode 100644 src/Automind.Mcp/McpPredicateMapper.cs create mode 100644 src/Automind.Mcp/McpToolBridge.cs create mode 100644 src/Automind.Memory/Automind.Memory.csproj create mode 100644 src/Automind.Memory/OnnxMemoryPager.cs create mode 100644 src/Automind.Reaqtor/Automind.Reaqtor.csproj create mode 100644 src/Automind.Reaqtor/Catalog/ConversationCatalog.cs create mode 100644 src/Automind.Reaqtor/Catalog/DocCatalogStore.cs create mode 100644 src/Automind.Reaqtor/Catalog/McpCatalogStore.cs create mode 100644 src/Automind.Reaqtor/Catalog/RuleCatalogStore.cs create mode 100644 src/Automind.Reaqtor/Catalog/ToolRegistryStepContextProvider.cs create mode 100644 src/Automind.Reaqtor/Client/AutomindClientContext.cs create mode 100644 src/Automind.Reaqtor/Engine/AutomindArtifacts.cs create mode 100644 src/Automind.Reaqtor/Engine/AutomindEngineFactory.cs create mode 100644 src/Automind.Reaqtor/Engine/AutomindServices.cs create mode 100644 src/Automind.Reaqtor/Engine/AutomindUris.cs create mode 100644 src/Automind.Reaqtor/IO/AutomindIngressEgressManager.cs create mode 100644 src/Automind.Reaqtor/IO/IReliableSubject.cs create mode 100644 src/Automind.Reaqtor/IO/ReliableSubject.cs create mode 100644 src/Automind.Reaqtor/Llm/ILlmService.cs create mode 100644 src/Automind.Reaqtor/Llm/OllamaLlmService.cs create mode 100644 src/Automind.Reaqtor/Llm/OllamaSegmentStreamer.cs create mode 100644 src/Automind.Reaqtor/Llm/ThinkFilter.cs create mode 100644 src/Automind.Reaqtor/Reactive/ConsoleObserver.cs create mode 100644 src/Automind.Reaqtor/Reactive/DerivationObservable.cs create mode 100644 src/Automind.Reaqtor/Reactive/DerivationOutput.cs create mode 100644 src/Automind.Reaqtor/Reactive/EgressObserver.cs create mode 100644 src/Automind.Reaqtor/Reactive/IngressObservable.cs create mode 100644 src/Automind.Reaqtor/Reactive/TimerObservable.cs create mode 100644 src/Automind.Reaqtor/Reactive/ToolInvokeObservable.cs create mode 100644 src/Automind.Reaqtor/Store/FileQueryEngineStateStore.cs create mode 100644 src/Automind.Reaqtor/Telemetry/AutomindDiagnostics.cs create mode 100644 src/Automind.Tools/Automind.Tools.csproj create mode 100644 src/Automind.Tools/DemoTools.cs create mode 100644 src/Automind.Tools/ITool.cs create mode 100644 src/Automind.Tools/PrimitiveTools.cs create mode 100644 src/Universalis.Core/Compilation/BonsaiCompiler.cs create mode 100644 src/Universalis.Core/Compilation/BonsaiSerialization.cs create mode 100644 src/Universalis.Core/Evaluation/EvalEnv.cs create mode 100644 src/Universalis.Core/Evaluation/EvalTypes.cs create mode 100644 src/Universalis.Core/Evaluation/Evaluator.cs create mode 100644 src/Universalis.Core/Evaluation/NumericOps.cs create mode 100644 src/Universalis.Core/Evaluation/PatternMatcher.cs create mode 100644 src/Universalis.Core/Evaluation/QueryPipeline.cs create mode 100644 src/Universalis.Core/Ir/IrJson.cs create mode 100644 src/Universalis.Core/Ir/PaperShape.cs create mode 100644 src/Universalis.Core/Ir/Program.cs create mode 100644 src/Universalis.Core/Ir/Statements.cs create mode 100644 src/Universalis.Core/Ir/Terms.cs create mode 100644 src/Universalis.Core/Parsing/ComprehensionCompiler.cs create mode 100644 src/Universalis.Core/Parsing/HedgeParser.cs create mode 100644 src/Universalis.Core/Parsing/HedgeScanner.cs create mode 100644 src/Universalis.Core/Parsing/LiterateRecognizer.cs create mode 100644 src/Universalis.Core/Parsing/UniversalisParser.cs create mode 100644 src/Universalis.Core/Rendering/AnswerAssembler.cs create mode 100644 src/Universalis.Core/Rendering/ConcreteRenderer.cs create mode 100644 src/Universalis.Core/Universalis.Core.csproj create mode 100644 tests/Automind.Integration.Tests/Automind.Integration.Tests.csproj create mode 100644 tests/Automind.Integration.Tests/MemoryPagerTests.cs create mode 100644 tests/Automind.Integration.Tests/OllamaLiveTests.cs create mode 100644 tests/Automind.Kernel.Tests/Automind.Kernel.Tests.csproj create mode 100644 tests/Automind.Kernel.Tests/DerivationHarness.cs create mode 100644 tests/Automind.Kernel.Tests/GoldenScenarioTests.cs create mode 100644 tests/Automind.Kernel.Tests/ModeBTests.cs create mode 100644 tests/Automind.Kernel.Tests/RuleTests.cs create mode 100644 tests/Automind.Mcp.Tests/Automind.Mcp.Tests.csproj create mode 100644 tests/Automind.Mcp.Tests/LoopbackBridgeTests.cs create mode 100644 tests/Automind.Mcp.Tests/McpPredicateMapperTests.cs create mode 100644 tests/Automind.Reaqtor.Tests/Automind.Reaqtor.Tests.csproj create mode 100644 tests/Automind.Reaqtor.Tests/EngineSmokeTests.cs create mode 100644 tests/Automind.Reaqtor.Tests/Fakes.cs create mode 100644 tests/Automind.Reaqtor.Tests/FileStoreTests.cs create mode 100644 tests/Automind.Reaqtor.Tests/KillRecoverTests.cs create mode 100644 tests/Automind.Reaqtor.Tests/SubstrateHarness.cs create mode 100644 tests/Automind.Reaqtor.Tests/ThinkFilterTests.cs create mode 100644 tests/Universalis.Core.Tests/BonsaiTests.cs create mode 100644 tests/Universalis.Core.Tests/ComprehensionTests.cs create mode 100644 tests/Universalis.Core.Tests/EvaluatorTests.cs create mode 100644 tests/Universalis.Core.Tests/HedgeParserTests.cs create mode 100644 tests/Universalis.Core.Tests/HedgeScannerTests.cs create mode 100644 tests/Universalis.Core.Tests/LiftingTests.cs create mode 100644 tests/Universalis.Core.Tests/LiterateRecognizerTests.cs create mode 100644 tests/Universalis.Core.Tests/PatternMatcherTests.cs create mode 100644 tests/Universalis.Core.Tests/RoundTripTests.cs create mode 100644 tests/Universalis.Core.Tests/Universalis.Core.Tests.csproj create mode 100644 tools/McpSampleServer/McpSampleServer.csproj create mode 100644 tools/McpSampleServer/Program.cs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3f6834e --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +.kb/ + +# build output +bin/ +obj/ +artifacts/ +*.user +.vs/ +TestResults/ + +# runtime state + local models +data/ +models/ +demo-data/ \ No newline at end of file diff --git a/Automind.slnx b/Automind.slnx new file mode 100644 index 0000000..be84f1d --- /dev/null +++ b/Automind.slnx @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000..74e464f --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,13 @@ + + + + net10.0 + 14.0 + enable + enable + true + true + false + + + diff --git a/Directory.Packages.props b/Directory.Packages.props new file mode 100644 index 0000000..05d4342 --- /dev/null +++ b/Directory.Packages.props @@ -0,0 +1,56 @@ + + + + true + false + + 1.0.0-beta.24 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/NuGet.config b/NuGet.config new file mode 100644 index 0000000..765346e --- /dev/null +++ b/NuGet.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/README.md b/README.md new file mode 100644 index 0000000..4bab5f9 --- /dev/null +++ b/README.md @@ -0,0 +1,188 @@ +# Automind / Universalis on Reaqtor + +A .NET 10 implementation of Erik Meijer's **neural computer** — the architecture from *Virtual Machinations: Using Large Language Models as Neural Computers* (ACM Queue, 2024) and *Unleashing the Power of End-User Programmable AI* (ACM Queue, 2025) — with **Nuqleon / Bonsai / Reaqtive / Reaqtor as the load-bearing substrate**, integrated with **Microsoft.Extensions.AI** against a **local Ollama** model (granite3.3:8b). + +The defining property: **a reasoning derivation is a durable standing query**. Kill the process mid-thought — mid-LLM-call, mid-tool-call, even mid-rule — restart it, and the derivation resumes from the last checkpoint and completes, re-issuing exactly the in-flight work that was lost. + +## The papers' concepts → this implementation + +| Neural computer (papers) | Realization | +|-------------------------------------------------|---------------------------------------------------------------------------------| +| Reasoning engine (control unit) | Pure `DerivationStep` state machine hosted in a checkpointed Reaqtor operator | +| LLM as branch predictor | `IChatClient` (OllamaSharp) with streaming hedge-cut + assistant-prefill resume | +| Environment σ (register set) | `var → JSON` map in checkpointed operator state; the model NEVER sees values | +| Tools as instruction set (relations) | URI-identified artifacts (`automind://tools/*`) invoked by the engine | +| Intentional representation | Structured IR (JSON) + **Bonsai expression trees** over `universalis://` params | +| Query comprehensions (paper: Kotlin DataFrames) | **LINQ** pipelines over JSON rows — nested results, HAVING and all | +| RAG as virtual memory | In-process ONNX embeddings (bge-micro-v2) paging rules/docs into the context | +| Tree-of-thought backtracking | Choice points + value-sanitized hints + temperature ladder | +| Pre/post-condition contracts | Vanilla Universalis clauses; pre gates the run, post restarts the derivation | +| Self-learning | Successful derivations become durable rules (IR + Bonsai) that run LLM-free | + +## Quickstart + +Prereqs: .NET 10 SDK and Ollama running with `granite3.3:8b`. Everything else — including the +Reaqtor/Reaqtive/Nuqleon stack (`1.0.0-beta.24`) — restores from nuget.org. + +```pwsh +dotnet test Automind.slnx --filter "TestCategory!=RequiresOllama" # 176 tests, no model needed +dotnet test Automind.slnx # + live Ollama & embedding tests + +# optional: enable RAG (fetches bge-micro-v2, ~66 MB, one-time) +pwsh scripts/fetch-embedding-model.ps1 + +dotnet run --project src/Automind.Cli -- ask "What is the current weather in Palo Alto?" --data .\data +dotnet run --project src/Automind.Cli -- repl --data .\data +``` + +## The flagship demo — durable reasoning + +```pwsh +# Act 1: the process kills itself the instant the first tool call is durably checkpointed +$env:AUTOMIND_CHAOS = "tool" +dotnet run --project src/Automind.Cli -- ask "What is the current weather in Palo Alto?" --data .\demo +# ☠ AUTOMIND_CHAOS: killing the process now (state is checkpointed) + +# Act 2: restart; the engine recovers the standing query, re-issues the in-flight call, finishes +$env:AUTOMIND_CHAOS = $null +dotnet run --project src/Automind.Cli -- resume --data .\demo +# ⟲ resuming derivation … — replaying its trace +# @weatherPaloAlto ← "Sunny and 80°F" (via WEATHER) +# ⇝ Sunny and 80°F +# ╭─ answer … ─╮ (exactly once) +``` + +`AUTOMIND_CHAOS=segment:N` kills mid-generation instead. Other commands: `rules` (learned-rule library), `learn-doc ` (ingest into virtual memory), `store` (dump the durable tables), `ask --learn ` (save the derivation as a reusable rule). + +## Mode B — whole-program synthesis + +`ask --mode-b` trades the hedge-by-hedge interception protocol for **one schema-constrained completion** returning the complete program in the papers' `{comment|expression}[]` interchange form (streamed and accumulated — grammar-constrained decoding is slow, and a non-streaming call times out). `PaperShape.Import` re-parses every expression through the same hedge grammar and the walk feeds (prose, hedge) pairs through the same literate recognizer, so conditionals, queries, tools, rules, teachings, and contracts behave **exactly as in Mode A** — including durable tool suspension mid-walk. Only the backtracking is coarse: any failure regenerates the whole program with the escalation-counted failure as feedback; the same failure three times fails fast, a generation cut at the token cap is taught truthfully (write a shorter program) while the cap grows across attempts, and the request budget binds on every path. Mode B is the conformance floor for models that cannot hold the interception protocol — measured, it is exactly what rescues gemma4 (see the model matrix below). + +## MCP — external tools as Universalis predicates + +`--mcp ""` (repeatable; `--mcp-tools` allowlists a subset) bridges an MCP stdio server into the tool registry: tool names uppercase to the catalog convention while **parameter names stay verbatim** (the engine keys call arguments by them); required schema properties become In-params plus one Out-param; array-typed parameters take whole lists (no zip-lifting); results pass through **raw when they are JSON** — so pattern destructuring works on them — and string-encode when prose. Idempotency comes only from the server's own `readOnlyHint`/`idempotentHint` annotations: recovery synthesizes a failure and backtracks instead of re-firing a non-idempotent call that was in flight at a kill. Tools whose schemas use `allOf`/`$ref` composition are skipped loudly at connect time, and the bridged server commands persist with the data directory, so a resume without `--mcp` reconnects them automatically. + +Invocation flows through the durable hedge protocol, **not** the chat client's native function calling — register discipline, checkpointed suspension, and backtracking apply to bridged tools unchanged. (Live-proven: the model *guessed* a bridged tool's result inside a guard, and the engine evaluated the guess false and crossed the branch out — the model never possesses the value.) A self-contained sample server lives at `tools/McpSampleServer`: + +```pwsh +dotnet run --project src/Automind.Cli -- ask "Reverse the word 'reaqtor' and bind it as @reversed." ` + --mcp "dotnet tools/McpSampleServer/bin/Debug/net10.0/McpSampleServer.dll" --data .\d-mcp +``` + +## Observability + +`--otel` exports traces/metrics to console/OTLP; `--otel-log ` appends spans and metric snapshots as JSON lines (`run-demos.ps1` passes it automatically — every demo leaves `\otel.jsonl`, and a retried demo preserves the failed attempt's log as `.attemptN.otel.jsonl`). The money metric for durability is `automind.llm.reissues` — nonzero after recovery is the substrate visibly re-driving in-flight reasoning. The reasoning-quality signals are `automind.backtracks` / `automind.restarts` / `automind.repairs` / `automind.derivation.failures`; `automind.llm.segment.duration` (min/max included in the file export) profiles generation speed, which swings 6–8× with the model server's mood. + +The kernel is pure and hosts no telemetry — the substrate observes around it: every kernel trace event rides the `derivation.step` spans as `automind.trace.*` events (segments, bindings, guards, backtracks, repairs), so the span stream reconstructs the complete internal logic flow. Alongside: `llm.complete` (model, temperature, seed, prefill/segment sizes, hedge cuts), `tool.invoke` (URI, route, status — the derivation path and the router artifact) with `mcp.call` children for bridged tools, `engine.create` / `engine.recover` (in-flight and conversation counts), `engine.checkpoint`, `store.open` (a backup fallback is an **Error**-status span, never silent), `store.persist` (bytes, edit count, `File.Replace` retries, the `automind.snapshot.bytes` histogram — and a commit that changed nothing skips the rewrite entirely, tagged `automind.store.skipped=true`), `rule.define`, and `memory.recall` / `memory.index` (hits, best score). Deliberately uninstrumented: the kernel and Universalis.Core (pure by design — observed via the bridged traces), per-output egress delivery, and the `AUTOMIND_CHAOS` kill switch (nothing can flush across a deliberate `FailFast`). + +The CLI surfaces generation wall time directly: a heartbeat line when a generation exceeds 20 s of silence, duration annotations on slow segments, and an `LLM: N segment(s), X s total, slowest Y s` footer on every answer — a healthy slow run must never read as a hang. + +## The samples from the papers + +Every worked example in the two papers is implemented — the interactive ones as runnable `demo` scenarios, the rest as golden tests, live tests, or few-shot exemplars. This section maps each paper sample to where it lives. + +### The runnable demo pack + +`demo list` enumerates the scenarios: + +```pwsh +dotnet run --project src/Automind.Cli -- demo weather-rule --data .\d1 + +# or run the whole pack: builds, preflights Ollama, gives each demo a fresh +# state dir, and prints a pass/fail summary. -Retries 1 reruns a failed roll +# once (8B derivations are stochastic). +pwsh scripts/run-demos.ps1 -Retries 1 +``` + +A demo whose *point* is a refusal (`contracts-violation`) declares `ExpectFailure` and exits 0 when the derivation is refused. + +| Scenario | Paper | What it proves live | +|-----------------------|-------|---------------------------------------------------------------------------------------------------------| +| `weather-rule` | 1 | The composed WEATHER rule (GEO_CODE → WEATHER_GOV → HTTP_GET, JSON patterns digging out each field) **shadows** the primitive tool; one invocation → three chained tool calls, zero LLM calls inside the rule | +| `bulk-pdf` | 2 | Loopless bulk processing: TO_PDF takes ONE file, the model calls it once with the LIST — `⚙ TO_PDF(...) ×3`, zip lifting fans it out, real stub PDFs land on disk | +| `contracts` | 2 | Pre-conditions gate the profit question; post-conditions check `@P == ((@S-@B)/@B)*100` after the fact | +| `contracts-violation` | 2 | `@B = 0` violates `[@B > 0]` → instant, traced refusal — **zero** LLM calls spent on an invalid ask | +| `btc-decision` | 2 | The conditional checklist: the engine evaluates every guard (`? … → false`), runs the taken branch, crosses out the other (`✗`), and the declared output `@btcLeft` rides into the answer | +| `team-selection` | 2 | The flagship grouped query as an **intentional program**: a stored STRONG_GROUPS rule compiles group-by / mean / min / collect / HAVING to a LINQ pipeline over JSON rows — fully nested `members` in the result, model channels one hedge | + +### Complete sample coverage + +| Paper sample | Where it lives | +|---|---| +| **1** — WEATHER relation, `"Palo Alto" ⇓ "Sunny and 80°F"` (table 1) | The canned WEATHER primitive returns exactly that; used by every weather test and demo | +| **1** — variable-passing transcript (`@weatherPaloAlto`, `⇝` display, "the city between Mountain View and Menlo Park") | Byte-for-byte in the `Weather_HappyPath` golden test, the live Ollama test, and the flagship chaos demo | +| **1** — ReAct interception (cut before `]`, ignore the hallucinated value) | The interception protocol itself: streaming hedge-cut + engine-appended `]` + prefill resume | +| **1** — the composed WEATHER rule (GEO_CODE → WEATHER_GOV → HTTP_GET with `{... "forecast": @url ...}` patterns) | `demo weather-rule` + rule-frame tests | +| **1** — the naked-model denial ("I don't have access to realtime information…") | Reproduced verbatim on raw granite3.3 (no tools); see below | +| **1** — the 10000°F model-interference example | Prevented architecturally by the register discipline; see below | +| **1** — RAG as virtual memory (context-stuffed forecast Q&A) | `Automind.Memory` pager + `learn-doc`; doc-grounded questions answer from paged chunks | +| **1** — `[@Z is @X+"hello"]` type error → no derivation → backtrack | Evaluator type failure feeds tree-of-thought backtracking (golden tests) | +| **1** — big-step ReAct semantics, fixed-mode Prolog, tree of thought | These *are* `DerivationStep`, the evaluator's mode discipline, and choice-point backtracking | +| **2** — apples profit, including the paper's own `(@D/@B)*100)` paren typo | Golden + live tests, `demo contracts`; the parser heals the exact typo (dedicated test) | +| **2** — formulas⇄values "live programming" rendering | The renderer's two modes over σ (`[@D is (@S-@B)]` ⇄ `[7 is (17-10)]`) in the CLI trace | +| **2** — pre-conditions `[@B>0]`, `[@S>=0]` and implication-form post-conditions | `demo contracts` (holds) and `demo contracts-violation` (instant engine refusal, zero LLM) | +| **2** — BTC/MSFT conditional checklist (TODAY, STOCK `{..."close":...}`, SEARCH) | `demo btc-decision` — guards engine-evaluated, untaken branch crossed out | +| **2** — toPdf/listFiles loopless bulk conversion | `demo bulk-pdf` — `⚙ TO_PDF(...) ×3` from a single hedge | +| **2** — customers-in-Palo-Alto count query | Few-shot exemplar + passing live integration test | +| **2** — World Cup players grouped query (group/mean/min/collect/HAVING, nested players) | `demo team-selection` + query-pipeline unit tests on the exact query | +| **2** — STOCK("IBM") messy JSON blob + `{ ... "volume": @V ... }` pattern | Pattern-matcher tests bind @V/@P/@X from the paper's blob | +| **2** — the intentional representation (`{comment}/{expression}` JSON array) | `PaperShape` import/export with round-trip tests (the durable form adds Bonsai) | +| **2** — self-learning via Tennent abstraction | `ask --learn `, the `rules` command, Bonsai round-trip + kill-survival tests | + +### The two "anti-examples": denial and interference + +Paper 1 opens with the model **denying** it can answer ("I'm sorry, but I don't have access to realtime information…"). Raw granite3.3 reproduces this almost verbatim when asked `What is the current temperature in Palo Alto?` with no tools — and the same model, on the same question, calls `[WEATHER("Palo Alto", @weatherPaloAlto)]` inside Automind. The before/after is the whole thesis in one exchange. If a completion ever *were* denial-shaped (prose, no hedges), the essay guard backtracks it with "act instead of describing" steering — live-proven, since essays are the same failure class. + +Paper 1's **10000°F interference** example (vanilla ReAct lets the model "correct" a tool result it finds implausible) is prevented by construction rather than detection: tool results live in σ inside checkpointed operator state and are never serialized into any prompt — the model reads back only `[WEATHER("Palo Alto", @weatherPaloAlto)]` and the engine-appended `]`, and user-visible values are substituted engine-side at display hedges. To corrupt a value the model would have to generate it, and it never possesses it. (The golden test asserts the invariant: the prompt after binding must not contain the value.) Two deliberate boundaries: *question inputs* are disclosed to the model — they're the user's own values, and σ stays authoritative since a conflicting re-bind is an error — and the architecture guarantees displayed values, not the model's prose beside them. + +### Deliberate deviations from the papers + +- Query comprehensions compile to **LINQ**, not the paper's Kotlin DataFrames (this project's substrate is Meijer's own .NET lineage). +- GEO_CODE / WEATHER_GOV / HTTP_GET / STOCK / SEARCH are **canned facts** shaped like the real APIs (NWS, TwelveData) so demos run deterministic and offline; LIST_FILES / TO_PDF do real file I/O with stub PDFs. +- The paper's aside that users can ask for the **equivalent SQL** of a query has no renderer here. +- The apples-worth-in-gold illustration is covered as the intentional-representation *shape* only — WOLFRAM / TO_DOUBLE are not registered tools. +- `team-selection` has the model **invoke** the stored query rather than author it live: granite3.3:8b reliably channels programs but reimplements (SQL, Python, accumulators) when asked to author or even transcribe the five-bullet form — the paper's authoring transcript came from a frontier model. +- The 10000°F scenario has no dedicated end-to-end regression test — the register discipline is asserted structurally in the golden tests instead. + +## How a derivation runs + +1. A question becomes a **Reaqtor subscription**: `automind://derivation(id, envelope)` → egress topic. The subscription — including the question — is write-ahead-logged. +2. The model generates literate Universalis; the bridge streams it through a bracket-depth scanner and **cuts before the closing `]`** of each hedge (client-side abort — a `]` stop sequence would false-cut inside JSON array patterns). +3. The kernel parses the hedge and executes it: bindings/`is`/patterns inline; tool calls emit effects and suspend; display hedges `[@x]` show values to the *user only* (⇝). The engine appends `]` and resumes generation — the model narrates over *names*, never values. +4. Every step mutates checkpointed operator state (`σ`, program IR, choice points, rule frames, pending effects). The host checkpoints per step (100 ms debounce) + every 5 s. +5. Recovery re-issues pending effects with the *same request ids*; the deterministic step function drops stale completions. At-least-once end to end, deduplicated by sequence ids. + +## Solution layout + +``` +src/Universalis.Core the language: IR, hedge scanner/parser, literate recognizer, evaluator (σ, is, one-way patterns w/ DFS key search, zip lifting), LINQ query pipeline, contracts, Bonsai compiler + serialization, renderer +src/Automind.Kernel the control unit: pure Step phase machine, prompting (LAWS + few-shots), backtracking, rule frames, learning — no I/O, no clocks, deterministic +src/Automind.Reaqtor the substrate: engine host (Shebang-derived), kill-safe file store, DerivationObservable driver, tool router artifact, reliable ingress/ egress, catalogs (conversations/rules/docs), Ollama bridge, telemetry +src/Automind.Tools primitive tools ("facts"): WEATHER (canned/deterministic), TODAY, MATH + the papers' demo facts (GEO_CODE, WEATHER_GOV, HTTP_GET, STOCK, SEARCH, LIST_FILES, TO_PDF) +src/Automind.Mcp MCP bridge: schema→signature mapper, AIFunction→ITool adapter, stdio client lifecycle +src/Automind.Memory RAG virtual memory: in-process BERT-ONNX embeddings + cosine recall +src/Automind.Cli Spectre.Console host: repl/ask/resume/rules/learn-doc/store/demo, chaos switch, JSONL telemetry sink +tools/McpSampleServer self-contained stdio MCP server (reverse, word_count) for tests and demos +scripts/ run-demos.ps1 (demo pack + per-demo telemetry), run-model-matrix.ps1, fetch-embedding-model.ps1 +docs/model-matrix.md measured per-model conformance verdicts +tests/ 176 fast tests (golden transcripts, kill/recover matrix, store atomicity, in-proc MCP loopback, paper examples as fixtures) + live suites (Ollama / embedding-model gated) +``` + +## Live-verified against granite3.3:8b + +Arithmetic chains, tool loops, conditional checklists (guards traced, untaken branches crossed out), query comprehensions, zip-lifted batches, and stored-rule invocation all run live. Several engine behaviors exist *because* live transcripts demanded them: bracket-free steering (models parrot feedback), duplicate-call and redundant-re-bind no-ops (models re-narrate what already happened), `[@x is TOOL(...)]` syntax tolerance, named arguments, extra-out-arg trimming, sentence-start section detection, block-level backtracking that rewinds to before the block, essay/phantom-variable detection (a completion that *narrates* results without computing them backtracks), **identical-failure escalation** (a model looping on the same error burns extra attempts and is forced to a deeper rewind instead of riding the whole request budget), **rule-frame scope isolation** (a rule body sees only its own locals — a caller's `@lat` must never collide with the body's `@lat`, or deterministic re-execution fails every retry), the `$` money-sigil tolerance, targeted parse teachings for `for`-loop / accumulator / conjunction / bracketed-`If`-bullet / imperative-verb instincts, **query loop-walkthrough tolerance** (a model that states a query declaratively and then re-narrates it as a loop — fused `Retain … and increment …` bullets, `- Initially, [@total = 0]` counter inits, `- If …` restatements of registered filters — compiles to the filter+count it means), and the **always-true-filter guard** (a filter over a field the row pattern never destructured *binds* instead of testing and would count every row — it now fails with the destructuring teach instead of answering wrongly), and the **If-sentence guard** (a σ-mutating hedge inside an "If …" *sentence* — as opposed to a checklist bullet — would execute unconditionally, poisoning σ with the untaken arm's value; it now backtracks with the checklist teach). + +Calibration for an 8B: granite3.3 reliably *channels* programs and *invokes* stored rules but cannot reliably *author* the five-bullet grouped query form live (it reimplements instead of transcribing) — which is why `team-selection` demonstrates the papers' intentional-program mechanism rather than live query authoring, and why derivations are stochastic: an occasional roll exhausts its retry budget where an immediate rerun passes. + +`scripts/run-model-matrix.ps1` measures any installed model against the protocol: the conformance gate (prefill continuation + hedge cut), four Mode A derivations, and the Mode B floor — verdicts land in [docs/model-matrix.md](docs/model-matrix.md). Reasoning-tuned models (qwen3) are handled by a streaming `…` suppressor in front of the hedge scanner: chain-of-thought must never execute, only the answer does. Measured verdicts (2026-07-16): **qwen2.5-coder:7b fully conformant and fastest across every cell** (a coding-tuned 7B holds the formal protocol best), granite3.3:8b Mode A (the session workhorse), qwen3:8b Mode A but fragile (thinking eats segment budget and wall clock), and gemma4 **Mode B only** — protocol-conformant but too slow per segment for multi-request derivations, rescued by the single whole-program generation, which is exactly the fallback Mode B exists to be. + +Resilience policies that keep stochastic rolls honest *and* alive: **declared outputs gate completion** (finishing without computing one backtracks with placement-aware teaching), **post-answer-noise skipping** (once every declared output is bound, a failing trailing hedge is skipped — never unwinds a finished derivation), and **tree-of-thought restart** (exhausting the search — choice points *or* backtracking depth — with budget to spare restarts the derivation from the question, engine notes carrying the accumulated teachings forward). A max-effort adversarial review round hardened the loop further: recovery never re-fires non-idempotent tools, terminal egress signals always resolve the waiting CLI (a drained-answer recovery can no longer hang `ask`), the LLM request budget binds on every issue path — success loops included — and an LLM transport outage self-heals with capped backoff instead of wedging until a restart. + +Measured over repeated full-pack runs: four demos pass first-attempt in 6–20 seconds near-deterministically *at typical generation speeds*, `btc-decision` (live JSON-extraction arithmetic, the hardest micro-skill for an 8B) passes ~75% first-attempt and reliably with the script's `-Retries 1` — and wall clock scales with the model server's mood (the same pack measured 6–8× slower on the same day with identical trajectories; the telemetry and the CLI's LLM footer make the difference legible). + +## Known limitations (v1) + +- Tool and rule calls inside conditional *branches* backtrack with a teaching hint (call before the checklist); comprehensions and rule bodies support them fully. +- Rules persist as durable catalog rows (IR + Bonsai) rather than engine-defined artifacts. +- Answers can carry model prose verbatim, including its occasional echo debris. +- MCP bridging is stdio-only and maps required parameters only (optional MCP parameters are dropped — fixed-mode arity). diff --git a/docs/model-matrix.md b/docs/model-matrix.md new file mode 100644 index 0000000..2980dcd --- /dev/null +++ b/docs/model-matrix.md @@ -0,0 +1,40 @@ +# Model-swap matrix + +Measured 2026-07-16 22:44 against http://localhost:11434 on HVR-SLS. +Each cell is one gated live test (pass/fail, seconds). Verdicts: **Mode A** = conformance gate ++ ≥3/4 Mode A derivations; **Mode A (fragile)** = the derivations pass but the gate's tight +segment budget does not; **Mode B only** = whole-program synthesis works where the +interception protocol does not. + +| model | conformance | apples A | ticket A | customers A | weather A | apples B | verdict | +|---|---|---|---|---|---|---|---| +| gemma4:latest | ✓ 40s | ✗ 244s | ✗ 244s | ✗ 244s | ✗ 244s | ✓ 150s | **Mode B only** | +| granite3.3:8b | ✓ 37s | ✓ 71s | ✗ 244s † | ✗ 106s † | ✓ 10s | ✓ 10s | **Mode A** † | +| qwen2.5-coder:7b | ✓ 11s | ✓ 12s | ✓ 13s | ✓ 19s | ✓ 8s | ✓ 9s | **Mode A** | +| qwen3:8b | ✗ 13s ‡ | ✓ 34s | ✓ 62s | ✗ 243s | ✓ 117s | ✗ 244s | **Mode A (fragile)** ‡ | + +† granite's two ✗ cells were cold-swap artifacts: this run cycled four models through GPU +memory and both cells hit their 240 s test timeout cold; **both pass warm on immediate rerun** +(and granite passes the full live suite 5/5 routinely — see the session log). The script now +warms each model outside the test timeouts before its battery. Verdict corrected accordingly. + +‡ qwen3's conformance failure is real and instructive: with thinking enabled, the gate's +64-token echo budget is consumed entirely by `` content — the filtered reply is EMPTY. +Real derivations pass (3/4) because 512-token segments leave room to think *and* emit, but the +overhead is paid on every segment (34–117 s vs qwen2.5-coder's 8–19 s) and it timed out the +customers query and the grammar-constrained Mode B generation. Usable, with caveats. + +## Findings + +- **qwen2.5-coder:7b is the surprise headline**: 6/6, fully conformant, and the fastest across + every cell — a coding-tuned 7B holds the formal hedge protocol better than anything else + measured, including the general 8Bs. Worth making it the default for protocol-heavy work. +- **gemma4 is the Mode B story validated**: conformant at the byte level but too slow per + segment for multi-request Mode A derivations inside the 240 s test window — yet the single + 150 s whole-program generation passes. Exactly the fallback Mode B was built to be. +- **Thinking models pay the protocol tax twice** (qwen3): thought consumes segment budget and + wall clock; the `` suppressor keeps chain-of-thought from ever executing, but cannot + refund the time. + +Rerun a single cell with: +`$env:AUTOMIND_OLLAMA_MODEL=''; dotnet test tests/Automind.Integration.Tests --filter "FullyQualifiedName~"`. diff --git a/global.json b/global.json new file mode 100644 index 0000000..bcdddea --- /dev/null +++ b/global.json @@ -0,0 +1,9 @@ +{ + "sdk": { + "version": "10.0.300", + "rollForward": "latestPatch" + }, + "test": { + "runner": "Microsoft.Testing.Platform" + } +} diff --git a/scripts/fetch-embedding-model.ps1 b/scripts/fetch-embedding-model.ps1 new file mode 100644 index 0000000..f092c14 --- /dev/null +++ b/scripts/fetch-embedding-model.ps1 @@ -0,0 +1,27 @@ +# Fetches the bge-micro-v2 embedding model (BERT ONNX) for fully LOCAL, in-process embeddings. +# No Ollama, no server, no generative model. Files land in \models\bge-micro-v2. +param( + [string]$Destination = (Join-Path $PSScriptRoot "..\models\bge-micro-v2") +) + +$ErrorActionPreference = "Stop" +New-Item -ItemType Directory -Force $Destination | Out-Null + +$files = @( + @{ Url = "https://huggingface.co/TaylorAI/bge-micro-v2/resolve/main/onnx/model.onnx"; Name = "model.onnx" }, + @{ Url = "https://huggingface.co/TaylorAI/bge-micro-v2/resolve/main/vocab.txt"; Name = "vocab.txt" } +) + +foreach ($file in $files) { + $target = Join-Path $Destination $file.Name + if (Test-Path $target) { + Write-Host "already present: $target" + continue + } + + Write-Host "downloading $($file.Url) ..." + Invoke-WebRequest -Uri $file.Url -OutFile $target + Write-Host ("saved: {0} ({1} MB)" -f $target, [math]::Round((Get-Item $target).Length / 1MB, 1)) +} + +Write-Host "done - Automind probes '\..\models\bge-micro-v2' and AUTOMIND_EMBEDDINGS." diff --git a/scripts/run-demos.ps1 b/scripts/run-demos.ps1 new file mode 100644 index 0000000..5ce3ecc --- /dev/null +++ b/scripts/run-demos.ps1 @@ -0,0 +1,171 @@ +#Requires -Version 7 +<# +.SYNOPSIS + Runs the Automind paper-example demos end to end against local Ollama. + +.DESCRIPTION + Builds the CLI (unless -SkipBuild), verifies Ollama and the chat model are + reachable, then runs each demo scenario serially with a fresh durable-state + directory and prints a pass/fail summary. + + Demos are stochastic: an 8B model occasionally exhausts its retry budget on + a bad trajectory where an immediate rerun passes. Use -Retries 1 to rerun a + failed demo once (on a fresh directory) before counting it as failed. + +.PARAMETER Name + Subset of demos to run (default: all six, in pack order). + +.PARAMETER DataRoot + Root under which each demo gets its own fresh --data directory. + +.PARAMETER Retries + Extra attempts per demo after a failure (default 0). + +.EXAMPLE + pwsh scripts/run-demos.ps1 +.EXAMPLE + pwsh scripts/run-demos.ps1 -Name bulk-pdf,team-selection -Retries 1 +#> +[CmdletBinding()] +param( + [string[]]$Name, + [string]$DataRoot = (Join-Path $PSScriptRoot '..' 'demo-data'), + [string]$Endpoint = 'http://localhost:11434', + [string]$Model = 'granite3.3:8b', + # Generation speed swings 6-8x with Ollama's mood (measured live: the same demo at 15 s and + # 10 min on the same day) — 900 s would kill a CONVERGING btc roll on a slow day. + [int]$TimeoutSeconds = 1200, + [int]$Retries = 0, + [switch]$SkipBuild +) + +$ErrorActionPreference = 'Stop' + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +$exe = Join-Path $repoRoot 'src' 'Automind.Cli' 'bin' 'Debug' 'net10.0' 'Automind.Cli.exe' + +$allDemos = @('weather-rule', 'bulk-pdf', 'contracts', 'contracts-violation', 'btc-decision', 'team-selection') + +$demos = if ($Name) { + # `pwsh -File` passes "a,b" as one literal string (only interactive sessions split it). + $requested = @($Name | ForEach-Object { $_ -split ',' } | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + + foreach ($n in $requested) { + if ($n -notin $allDemos) { + throw "Unknown demo '$n'. Available: $($allDemos -join ', ')" + } + } + $allDemos | Where-Object { $_ -in $requested } # keep pack order +} else { + $allDemos +} + +# ---------------------------------------------------------------- preflight + +if (-not $SkipBuild -or -not (Test-Path $exe)) { + Write-Host 'building Automind.slnx …' -ForegroundColor DarkGray + dotnet build (Join-Path $repoRoot 'Automind.slnx') --verbosity quiet --nologo + if ($LASTEXITCODE -ne 0) { throw 'build failed' } +} + +try { + $tags = Invoke-RestMethod "$Endpoint/api/tags" -TimeoutSec 5 +} catch { + throw "Ollama is not reachable at $Endpoint — start it first ($($_.Exception.Message))" +} + +if (-not ($tags.models.name -contains $Model)) { + throw "model '$Model' is not installed in Ollama (have: $($tags.models.name -join ', ')). Run: ollama pull $Model" +} + +if (-not (Test-Path (Join-Path $repoRoot 'models' 'bge-micro-v2' 'model.onnx'))) { + Write-Host 'note: embedding model missing — the memory pager will be off (optional; scripts/fetch-embedding-model.ps1 fetches it)' -ForegroundColor DarkYellow +} + +# ---------------------------------------------------------------- run loop + +$results = foreach ($demo in $demos) { + $attempts = 0 + $exit = $null + $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + + while ($attempts -le $Retries) { + $attempts++ + + $dataDir = Join-Path $DataRoot $demo + + if ($attempts -eq 1) { + # Stale evidence from a previous pack run. + Get-ChildItem $DataRoot -Filter "$demo.attempt*.otel.jsonl" -ErrorAction SilentlyContinue | Remove-Item -Force + } + + if (Test-Path $dataDir) { + # A retry wipes the data directory — preserve the FAILED attempt's telemetry first; + # the failure log is the evidence the retry exists to route around. + $prevLog = Join-Path $dataDir 'otel.jsonl' + if ($attempts -gt 1 -and (Test-Path $prevLog)) { + Move-Item $prevLog (Join-Path $DataRoot ("{0}.attempt{1}.otel.jsonl" -f $demo, ($attempts - 1))) -Force + } + + Remove-Item -Recurse -Force $dataDir + } + + Write-Host '' + Write-Host ('═' * 78) -ForegroundColor Cyan + Write-Host (" demo {0} (attempt {1}/{2})" -f $demo, $attempts, ($Retries + 1)) -ForegroundColor Cyan + Write-Host ('═' * 78) -ForegroundColor Cyan + + # Every run leaves a telemetry artifact: spans (with the automind.trace.* logic-flow + # events) + metric snapshots, as JSON lines next to the demo's durable state. + $otelLog = Join-Path $dataDir 'otel.jsonl' + + $p = Start-Process $exe ` + -ArgumentList 'demo', $demo, '--data', $dataDir, '--endpoint', $Endpoint, '--model', $Model, '--quiet', '--otel-log', $otelLog ` + -NoNewWindow -PassThru + + if (-not $p.WaitForExit($TimeoutSeconds * 1000)) { + $p.Kill($true) + $exit = -1 + Write-Host " ⏱ timed out after $TimeoutSeconds s (killed; state is checkpointed in $dataDir)" -ForegroundColor Red + } else { + $exit = $p.ExitCode + } + + if ($exit -eq 0) { break } + + if ($attempts -le $Retries) { + Write-Host ' ↻ failed roll — retrying on a fresh directory (8B derivations are stochastic)' -ForegroundColor DarkYellow + } + } + + $stopwatch.Stop() + + [pscustomobject]@{ + Demo = $demo + Result = if ($exit -eq 0) { 'pass' } elseif ($exit -eq -1) { 'timeout' } else { 'fail' } + Attempts = $attempts + Duration = '{0:mm\:ss}' -f $stopwatch.Elapsed + } +} + +# ---------------------------------------------------------------- summary + +Write-Host '' +Write-Host ('═' * 78) -ForegroundColor Cyan +Write-Host ' summary' -ForegroundColor Cyan +Write-Host ('═' * 78) -ForegroundColor Cyan + +$results | Format-Table -AutoSize | Out-String | Write-Host + +$failed = @($results | Where-Object Result -ne 'pass') + +if ($failed.Count -gt 0) { + Write-Host ("{0} of {1} demo(s) did not pass. A failed roll usually passes on rerun:" -f $failed.Count, $results.Count) -ForegroundColor Yellow + foreach ($f in $failed) { + Write-Host (" pwsh scripts/run-demos.ps1 -Name {0}" -f $f.Demo) -ForegroundColor Yellow + } + exit 1 +} + +Write-Host 'all demos passed.' -ForegroundColor Green +exit 0 diff --git a/scripts/run-model-matrix.ps1 b/scripts/run-model-matrix.ps1 new file mode 100644 index 0000000..5081be9 --- /dev/null +++ b/scripts/run-model-matrix.ps1 @@ -0,0 +1,187 @@ +<# +.SYNOPSIS + Runs the model-swap matrix: the live conformance gate plus the derivation battery + against every candidate Ollama model, and writes the verdict table. + +.DESCRIPTION + Per model, each live test runs as its own `dotnet test` invocation with + AUTOMIND_OLLAMA_MODEL set: the protocol conformance gate (prefill continuation + + hedge cut), four Mode A derivations (arithmetic, conditional checklist, query + comprehension, tool loop), and the Mode B whole-program synthesis floor. + + Verdicts: 'Mode A' (conformance + >=3/4 Mode A derivations), 'Mode B only' + (whole-program synthesis passes where the interception protocol does not), + 'unusable'. First test per model pays the model-load cost — durations are + indicative, not benchmarks. + +.EXAMPLE + pwsh scripts/run-model-matrix.ps1 + pwsh scripts/run-model-matrix.ps1 -Models granite3.3:8b,qwen3:8b +#> +param( + [string[]]$Models, + [string]$Endpoint = ($env:AUTOMIND_OLLAMA ?? 'http://localhost:11434'), + [string]$ReportPath = (Join-Path $PSScriptRoot '..' 'docs' 'model-matrix.md'), + [int]$TimeoutSeconds = 420, + [switch]$SkipBuild +) + +$ErrorActionPreference = 'Stop' +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..') + +# The battery: display name -> FullyQualifiedName fragment. Order matters — conformance first. +$battery = [ordered]@{ + 'conformance' = 'Conformance_PrefillContinuation_And_HedgeCut' + 'apples A' = 'LiveDerivation_Apples_PureArithmetic' + 'ticket A' = 'LiveDerivation_Conditional_ChecklistDecision' + 'customers A' = 'LiveDerivation_Comprehension_CustomerCount' + 'weather A' = 'LiveDerivation_Weather_ToolCallLoop' + 'apples B' = 'LiveDerivation_ModeB_WholeProgramSynthesis' +} + +# ---------------------------------------------------------------- preflight + +try { + $tags = Invoke-RestMethod -Uri "$Endpoint/api/tags" -TimeoutSec 5 +} catch { + throw "Ollama is not reachable at $Endpoint" +} + +$installed = @($tags.models | ForEach-Object { $_.name }) + +if (-not $Models) { + # Default: every installed chat model (embedding models can't chat). + $Models = $installed | Where-Object { $_ -notmatch 'embed' } +} + +Write-Host "matrix candidates: $($Models -join ', ')" -ForegroundColor Cyan + +if (-not $SkipBuild) { + Write-Host 'building Automind.slnx …' -ForegroundColor DarkGray + dotnet build (Join-Path $repoRoot 'Automind.slnx') --verbosity quiet --nologo + if ($LASTEXITCODE -ne 0) { throw 'build failed' } +} + +$testProject = Join-Path $repoRoot 'tests' 'Automind.Integration.Tests' +$env:NO_COLOR = '1' +$env:AUTOMIND_OLLAMA = $Endpoint + +# ---------------------------------------------------------------- run + +$results = foreach ($model in $Models) { + if ($model -notin $installed) { + Write-Host " ⤫ $model is not installed — skipped" -ForegroundColor DarkYellow + [pscustomobject]@{ Model = $model; Cells = $null; Verdict = 'not installed'; Seconds = 0 } + continue + } + + Write-Host '' + Write-Host ('═' * 70) -ForegroundColor Cyan + Write-Host " model $model" -ForegroundColor Cyan + Write-Host ('═' * 70) -ForegroundColor Cyan + + $env:AUTOMIND_OLLAMA_MODEL = $model + $cells = [ordered]@{} + $modelWatch = [System.Diagnostics.Stopwatch]::StartNew() + + # Warm the model OUTSIDE the test timeouts: cycling four models through GPU memory made + # cold cells time out and read as protocol failures (observed: granite3.3 'failed' two + # cells cold that it passes warm every time). + try { + $null = Invoke-RestMethod -Method Post -Uri "$Endpoint/api/generate" -TimeoutSec 300 -ContentType 'application/json' ` + -Body (@{ model = $model; prompt = 'hi'; stream = $false } | ConvertTo-Json) + } catch { + Write-Host " (warm-up failed: $($_.Exception.Message))" -ForegroundColor DarkYellow + } + + foreach ($name in $battery.Keys) { + $fqn = $battery[$name] + $stdout = New-TemporaryFile + $watch = [System.Diagnostics.Stopwatch]::StartNew() + + $p = Start-Process dotnet ` + -ArgumentList 'test', $testProject, '--no-build', '--filter', "FullyQualifiedName~$fqn" ` + -RedirectStandardOutput $stdout.FullName -NoNewWindow -PassThru + + $finished = $p.WaitForExit($TimeoutSeconds * 1000) + if (-not $finished) { $p.Kill($true) } + $watch.Stop() + + $clean = (Get-Content $stdout.FullName -Raw -ErrorAction SilentlyContinue) -replace "`e\[[0-9;]*m", '' + Remove-Item $stdout -Force -ErrorAction SilentlyContinue + + $pass = $finished -and $clean -match 'succeeded:\s*1' -and $clean -match 'failed:\s*0' + $skipped = $clean -match 'skipped:\s*[1-9]' # Ollama unreachable → inconclusive + + $cells[$name] = [pscustomobject]@{ + Pass = $pass + Skipped = $skipped + Seconds = [math]::Round($watch.Elapsed.TotalSeconds) + } + + $glyph = if ($pass) { '✓' } elseif ($skipped) { '~' } else { '✗' } + $color = if ($pass) { 'Green' } elseif ($skipped) { 'DarkYellow' } else { 'Red' } + Write-Host (" {0} {1,-12} {2,4}s" -f $glyph, $name, $cells[$name].Seconds) -ForegroundColor $color + } + + $modelWatch.Stop() + + $modeAPasses = @('apples A', 'ticket A', 'customers A', 'weather A' | ForEach-Object { $cells[$_] } | Where-Object Pass).Count + $verdict = + if ($cells['conformance'].Pass -and $modeAPasses -ge 3) { 'Mode A' } + elseif ($modeAPasses -ge 3) { 'Mode A (fragile)' } # derivations work; the gate's tight segment budget does not (thinking models eat it) + elseif ($cells['apples B'].Pass) { 'Mode B only' } + else { 'unusable' } + + Write-Host " verdict: $verdict" -ForegroundColor Magenta + + [pscustomobject]@{ + Model = $model + Cells = $cells + Verdict = $verdict + Seconds = [math]::Round($modelWatch.Elapsed.TotalSeconds) + } +} + +Remove-Item Env:\AUTOMIND_OLLAMA_MODEL -ErrorAction SilentlyContinue + +# ---------------------------------------------------------------- report + +$lines = [System.Collections.Generic.List[string]]::new() +$lines.Add('# Model-swap matrix') +$lines.Add('') +$lines.Add("Measured $(Get-Date -Format 'yyyy-MM-dd HH:mm') against $Endpoint on $env:COMPUTERNAME.") +$lines.Add('Each cell is one gated live test (pass/fail, seconds); the first test per model pays the') +$lines.Add('model-load cost. Verdicts: **Mode A** = conformance gate + >=3/4 Mode A derivations;') +$lines.Add('**Mode B only** = whole-program synthesis works where the interception protocol does not.') +$lines.Add('') +$lines.Add('| model | ' + (($battery.Keys | ForEach-Object { $_ }) -join ' | ') + ' | verdict |') +$lines.Add('|---|' + (('---|' * $battery.Count)) + '---|') + +foreach ($r in $results) { + if ($null -eq $r.Cells) { + $lines.Add("| $($r.Model) | " + (('— | ' * $battery.Count)) + "$($r.Verdict) |") + continue + } + + $cellText = ($battery.Keys | ForEach-Object { + $c = $r.Cells[$_] + if ($c.Pass) { "✓ $($c.Seconds)s" } elseif ($c.Skipped) { '~' } else { "✗ $($c.Seconds)s" } + }) -join ' | ' + + $lines.Add("| $($r.Model) | $cellText | **$($r.Verdict)** |") +} + +$lines.Add('') +$lines.Add('A ✗ on a stochastic derivation is one roll, not a verdict — the verdict thresholds absorb that;') +$lines.Add('rerun a single cell with: `AUTOMIND_OLLAMA_MODEL= dotnet test tests/Automind.Integration.Tests --filter "FullyQualifiedName~"`.') + +New-Item -ItemType Directory -Force (Split-Path $ReportPath) | Out-Null +Set-Content -Path $ReportPath -Value ($lines -join "`n") -Encoding utf8 + +Write-Host '' +Write-Host "report written to $ReportPath" -ForegroundColor Cyan + +$results | Format-Table Model, Verdict, Seconds -AutoSize + +exit ($results | Where-Object { $_.Verdict -eq 'unusable' } | Measure-Object).Count diff --git a/src/Automind.Cli/Automind.Cli.csproj b/src/Automind.Cli/Automind.Cli.csproj new file mode 100644 index 0000000..8f0940b --- /dev/null +++ b/src/Automind.Cli/Automind.Cli.csproj @@ -0,0 +1,23 @@ + + + + Exe + + + + + + + + + + + + + + + + + + + diff --git a/src/Automind.Cli/AutomindHost.cs b/src/Automind.Cli/AutomindHost.cs new file mode 100644 index 0000000..3b3a014 --- /dev/null +++ b/src/Automind.Cli/AutomindHost.cs @@ -0,0 +1,539 @@ +using System.Collections.Concurrent; +using System.Collections.Immutable; + +using Automind.Kernel; +using Automind.Kernel.Contract; +using Automind.Reaqtor.Catalog; +using Automind.Reaqtor.Client; +using Automind.Reaqtor.Engine; +using Automind.Reaqtor.IO; +using Automind.Reaqtor.Llm; +using Automind.Reaqtor.Reactive; +using Automind.Reaqtor.Store; +using Automind.Tools; + +using OllamaSharp; + +using Reaqtive.Scheduler; + +using Reaqtor.Shebang.Service; + +using Spectre.Console; + +namespace Automind.Cli; + +public sealed record HostOptions(string DataDirectory, string OllamaEndpoint, string Model, bool Verbose) +{ + /// Extra tools to register (demo scenarios add the papers' primitive facts here). + public Action? ConfigureTools { get; init; } + + /// Pre-stored rules (session-scoped, not persisted) — e.g. the papers' composed WEATHER rule. + public IReadOnlyList? SeedRules { get; init; } + + /// MCP stdio servers to bridge — one command line each ("dotnet path/Server.dll"). + public IReadOnlyList? McpServers { get; init; } + + /// Optional comma-separated allowlist of MCP tool names (MCP or predicate spelling). + public string? McpToolFilter { get; init; } +} + +public sealed record DerivationResult(string DerivationId, bool Succeeded, string Payload); + +/// +/// The Automind process host: opens the durable store, stands up (or recovers) the engine, +/// re-attaches in-flight conversations, runs the checkpoint policy, and — when +/// AUTOMIND_CHAOS is set — kills the process deterministically right after a durable +/// checkpoint at the requested point, for the durable-reasoning demo. +/// +public sealed class AutomindHost : IAsyncDisposable +{ + private readonly PhysicalScheduler _scheduler; + private readonly FileQueryEngineStateStore _store; + private readonly AutomindIngressEgressManager _iemgr = new(); + private readonly TraceRenderer _renderer; + private readonly ConcurrentDictionary> _completions = new(); + private readonly ConcurrentDictionary _terminalSeen = new(); + private readonly string? _chaos; + private int _chaosSegments; + + private SimplerCheckpointingQueryEngine _engine = null!; + private CheckpointCoordinator _checkpoints = null!; + private Automind.Mcp.McpToolBridge? _mcpBridge; + + public ConversationCatalog Conversations { get; } + + public RuleCatalogStore Rules { get; } + + public DocCatalogStore Docs { get; } + + public Automind.Memory.IMemoryPager? Memory { get; private set; } + + private ToolRegistryStepContextProvider _stepContextProvider = null!; + + public bool Recovered { get; private set; } + + public IReadOnlyList ResumedDerivations { get; private set; } = []; + + private AutomindHost(HostOptions options) + { + _scheduler = PhysicalScheduler.Create(); + _store = FileQueryEngineStateStore.Open(options.DataDirectory); + Conversations = new ConversationCatalog(_store); + Rules = new RuleCatalogStore(_store); + Docs = new DocCatalogStore(_store); + _renderer = new TraceRenderer { Verbose = options.Verbose }; + _chaos = Environment.GetEnvironmentVariable("AUTOMIND_CHAOS"); + } + + public static async Task StartAsync(HostOptions options) + { + var host = new AutomindHost(options); + + var tools = PrimitiveTools.CreateDefault(); + options.ConfigureTools?.Invoke(tools); + + // Captured BEFORE anything writes to the store: persisting the MCP specs below makes a + // brand-new store non-empty, and deciding create-vs-recover afterwards sent a fresh + // directory down the RECOVER path with no engine state to recover (caught live). + var isFreshStore = host._store.IsEmpty; + + // Recovery re-issues in-flight tool calls by URI, and "kill and re-run to resume" must + // not require repeating the flags — so the bridged server commands persist with the + // data directory and reconnect on a resume that omits --mcp (review finding). + var mcpCatalog = new McpCatalogStore(host._store); + var mcpServers = options.McpServers; + + if (mcpServers is { Count: > 0 }) + { + await mcpCatalog.SaveAsync(mcpServers); + } + else if (!isFreshStore && mcpCatalog.Servers() is { Count: > 0 } recorded) + { + mcpServers = recorded; + AnsiConsole.MarkupLineInterpolated($"[grey]reconnecting {recorded.Count} MCP server(s) recorded for this data dir[/]"); + } + + // MCP servers bridge in AFTER the primitives, and primitives win name collisions — + // the papers' instruction set stays stable no matter what a server advertises. + if (mcpServers is { Count: > 0 }) + { + var allowlist = options.McpToolFilter is { Length: > 0 } filter + ? new HashSet(filter.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase) + : null; + + host._mcpBridge = await Automind.Mcp.McpToolBridge.ConnectAsync( + mcpServers, + allowlist, + message => AnsiConsole.MarkupLineInterpolated($"[yellow]{message}[/]")); + + var bridged = 0; + + foreach (var tool in host._mcpBridge.Tools) + { + if (tools.Resolve(ToolRegistry.UriFor(tool.Signature.Name)) is not null) + { + AnsiConsole.MarkupLineInterpolated($"[yellow]MCP tool '{tool.Signature.Name}' collides with an existing predicate — skipped[/]"); + continue; + } + + tools.Add(tool); + bridged++; + } + + if (bridged > 12) + { + AnsiConsole.MarkupLineInterpolated($"[yellow]{bridged} MCP tools bridged — a prompt this wide strains an 8B; consider --mcp-tools to allowlist the ones you need[/]"); + } + else if (bridged > 0) + { + AnsiConsole.MarkupLineInterpolated($"[grey]{bridged} MCP tool(s) bridged as Universalis predicates[/]"); + } + } + + var chat = new OllamaApiClient(new Uri(options.OllamaEndpoint), options.Model); + + host._stepContextProvider = new ToolRegistryStepContextProvider(tools); + + // Rehydrate the learned-rule library from the durable catalog, then any seed rules + // (seeds come last so they shadow — the papers' rules-over-tools story). + foreach (var stored in host.Rules.All()) + { + host._stepContextProvider.AddRule(stored.Definition); + } + + foreach (var seed in options.SeedRules ?? []) + { + host._stepContextProvider.AddRule(seed); + } + + var services = new AutomindServices( + new DerivationStep(), + host._stepContextProvider, + new OllamaLlmService(chat, options.Model), + tools); + + if (isFreshStore) + { + using var activity = Automind.Reaqtor.Telemetry.AutomindDiagnostics.ActivitySource.StartActivity("engine.create"); + host._engine = await AutomindEngineFactory.CreateNewAsync( + host._store, host._scheduler, services.ToDictionary(), host._iemgr); + } + else + { + using var activity = Automind.Reaqtor.Telemetry.AutomindDiagnostics.ActivitySource.StartActivity("engine.recover"); + host.Recovered = true; + + // Recovery order: topics (and their renderers) must exist BEFORE the engine + // recovers — recovered egress observers resolve topics inside SetContext. + var active = host.Conversations.Active(); + + foreach (var record in host.Conversations.All()) + { + host.Attach(record.DerivationId, record.Topic); + } + + host._engine = await AutomindEngineFactory.RecoverAsync( + host._store, host._scheduler, services.ToDictionary(), host._iemgr); + + host.ResumedDerivations = [.. active.Select(r => r.DerivationId)]; + activity?.SetTag("automind.recover.in_flight", host.ResumedDerivations.Count); + activity?.SetTag("automind.recover.conversations", host.Conversations.All().Count); + } + + host._checkpoints = new CheckpointCoordinator(host._engine); + + await host.StartMemoryAsync(options); + + return host; + } + + /// + /// Stands up the virtual-memory pager (in-process ONNX embeddings — no Ollama involvement) + /// and indexes the durable rule library and document store. Silently absent when the model + /// files aren't fetched yet. + /// + private async Task StartMemoryAsync(HostOptions options) + { + var modelDirectory = Environment.GetEnvironmentVariable("AUTOMIND_EMBEDDINGS") + ?? Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "models", "bge-micro-v2"); + + try + { + Memory = await Automind.Memory.OnnxMemoryPager.TryCreateAsync(Path.GetFullPath(modelDirectory)); + } + catch (Exception ex) + { + AnsiConsole.MarkupLineInterpolated($"[yellow]memory pager unavailable: {ex.Message}[/]"); + return; + } + + if (Memory is null) + { + AnsiConsole.MarkupLine("[grey]memory pager off — run scripts\\fetch-embedding-model.ps1 to enable RAG[/]"); + return; + } + + var indexed = 0; + + foreach (var rule in Rules.All()) + { + var definition = rule.Definition; + await Memory.IndexAsync(new Automind.Memory.MemoryChunk( + "rule:" + rule.Name, + Automind.Memory.MemoryChunk.RuleKind, + rule.Name, + definition.Signature.Description + "\n" + definition.HeadProse)); + indexed++; + } + + foreach (var doc in Docs.All()) + { + foreach (var chunk in Automind.Memory.OnnxMemoryPager.ChunkDocument(doc.Name, doc.Text)) + { + await Memory.IndexAsync(chunk); + indexed++; + } + } + + AnsiConsole.MarkupLineInterpolated($"[grey]memory pager on — {indexed} chunk(s) indexed (in-process ONNX embeddings)[/]"); + } + + /// Ingests a document into durable storage and the vector index. + public async Task LearnDocumentAsync(string name, string text) + { + await Docs.SaveAsync(name, text); + + if (Memory is not null) + { + foreach (var chunk in Automind.Memory.OnnxMemoryPager.ChunkDocument(name, text)) + { + await Memory.IndexAsync(chunk); + } + } + } + + // ---------------------------------------------------------------- asking + + public Task AskAsync(string question, string? learnRuleName = null, bool modeB = false) + { + var envelope = QuestionEnvelope.ForText(question) with + { + LearnRuleOnSuccess = learnRuleName is not null, + RuleName = learnRuleName, + ModeB = modeB, + }; + + return AskAsync(envelope); + } + + public async Task AskAsync(QuestionEnvelope envelope) + { + // RAG as virtual memory: the ENGINE pages relevant knowledge into the context — recalled + // once per question and carried in the durable envelope (deterministic replay). + if (Memory is not null) + { + var recalled = await Memory.RecallAsync(envelope.Text, top: 4); + + var ruleNames = recalled + .Where(c => c.Kind == Automind.Memory.MemoryChunk.RuleKind) + .Select(c => c.Title) + .ToImmutableArray(); + + var chunks = recalled + .Where(c => c.Kind == Automind.Memory.MemoryChunk.DocKind) + .Select(c => new RecalledChunk(c.Title, c.Text)) + .ToImmutableArray(); + + envelope = envelope with + { + Context = chunks.IsEmpty ? envelope.Context : chunks, + RecalledRules = ruleNames.IsEmpty ? envelope.RecalledRules : ruleNames, + }; + } + + var derivationId = Guid.NewGuid().ToString("N")[..8]; + var topic = $"automind/out/{derivationId}"; + + Attach(derivationId, topic); + + await Conversations.UpsertAsync(new ConversationRecord( + derivationId, topic, envelope.Text, ConversationRecord.PendingStatus)); + + var ctx = AutomindClientContext.For(_engine); + + await ctx.Derivation(derivationId, envelope.ToJson()).SubscribeAsync( + ctx.Egress(topic), + new Uri($"automind://derivations/{derivationId}"), + state: null, + CancellationToken.None); + + _checkpoints.Request(); + + return derivationId; + } + + public Task WaitForAsync(string derivationId) => + _completions.GetOrAdd(derivationId, _ => new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously)).Task; + + private void Attach(string derivationId, string topic) + { + _completions.GetOrAdd(derivationId, _ => new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously)); + + // Terminal signals must resolve the waiter too (review finding): a recovered derivation + // whose answer was already delivered-and-drained before a kill re-emits ONLY OnCompleted + // — swallowing it left `ask` hung forever and the conversation permanently 'pending'. + // Fallback-only: after a normal Answer/Failed payload the waiter is already resolved and + // the trailing OnCompleted must not rewrite the persisted status. + _iemgr.GetOrCreateSubject(topic) + .Subscribe(new DelegateObserver<(long SequenceId, DerivationOutput Item)>( + v => OnOutput(derivationId, v.Item), + onCompleted: () => + { + if (!_terminalSeen.ContainsKey(derivationId)) + { + Complete( + derivationId, + new DerivationResult(derivationId, true, "(completed in a previous run — its answer was already delivered)"), + ConversationRecord.CompletedStatus); + } + }, + onError: ex => + { + if (!_terminalSeen.ContainsKey(derivationId)) + { + Complete(derivationId, new DerivationResult(derivationId, false, ex.Message), ConversationRecord.FailedStatus); + } + })); + } + + private void OnOutput(string derivationId, DerivationOutput output) + { + _renderer.Render(output); + + // Per-step checkpointing: every observed output marks progress worth persisting. + _checkpoints.Request(); + + MaybeChaosKill(output); + + switch (output.Kind) + { + case DerivationOutput.AnswerKind: + _terminalSeen[derivationId] = true; + Complete(derivationId, new DerivationResult(derivationId, true, output.PayloadJson), ConversationRecord.CompletedStatus); + break; + + case DerivationOutput.FailedKind: + _terminalSeen[derivationId] = true; + Complete(derivationId, new DerivationResult(derivationId, false, output.PayloadJson), ConversationRecord.FailedStatus); + break; + + case DerivationOutput.RuleKind: + OnRuleLearned(output.PayloadJson); + break; + } + } + + /// Persists a learned rule and makes it immediately invocable by the next step. + private void OnRuleLearned(string payloadJson) + { + Task.Run(async () => + { + using var activity = Automind.Reaqtor.Telemetry.AutomindDiagnostics.ActivitySource.StartActivity("rule.define"); + + try + { + var wrapper = System.Text.Json.JsonDocument.Parse(payloadJson); + var name = wrapper.RootElement.GetProperty("name").GetString()!; + var inner = System.Text.Json.JsonDocument.Parse(wrapper.RootElement.GetProperty("bonsai").GetString()!); + var ir = inner.RootElement.GetProperty("ir").GetString()!; + var bonsai = inner.RootElement.GetProperty("bonsai").GetString()!; + + activity?.SetTag("automind.rule.name", name); + activity?.SetTag("automind.rule.bonsai_bytes", bonsai.Length); + + await Rules.SaveAsync(name, ir, bonsai); + _stepContextProvider.AddRule(Universalis.Core.Ir.IrJson.DeserializeRule(ir)); + + AnsiConsole.MarkupLineInterpolated($"[blue]★ rule '{name}' saved to the durable library[/]"); + } + catch (Exception ex) + { + activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, ex.Message); + AnsiConsole.MarkupLineInterpolated($"[red]rule persistence failed: {ex.Message}[/]"); + } + }); + } + + private void Complete(string derivationId, DerivationResult result, string status) + { + Task.Run(async () => + { + try + { + await Conversations.SetStatusAsync(derivationId, status); + await _checkpoints.ForceAsync(); + } + catch (Exception ex) + { + AnsiConsole.MarkupLineInterpolated($"[red]finalization error: {ex.Message}[/]"); + } + finally + { + _completions.GetOrAdd(derivationId, _ => new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously)).TrySetResult(result); + } + }); + } + + // ---------------------------------------------------------------- chaos + + /// + /// AUTOMIND_CHAOS=tool kills after the first tool invocation is durably checkpointed + /// (state: AwaitingTools, call in flight). AUTOMIND_CHAOS=segment:N kills after the + /// N-th generation segment (state: Synthesizing, LLM request in flight). The kill is + /// — no unload, no cleanup: a true crash. + /// + private void MaybeChaosKill(DerivationOutput output) + { + if (_chaos is null || output.Kind != DerivationOutput.TraceKind) + { + return; + } + + TraceEvent trace; + try + { + trace = TraceEvent.FromJson(output.PayloadJson); + } + catch (System.Text.Json.JsonException) + { + return; + } + + var triggered = _chaos.ToLowerInvariant() switch + { + "tool" => trace is ToolInvoked, + var s when s.StartsWith("segment:", StringComparison.Ordinal) && + int.TryParse(s["segment:".Length..], out var n) => + trace is SegmentReceived && Interlocked.Increment(ref _chaosSegments) >= n, + _ => false, + }; + + if (!triggered) + { + return; + } + + Task.Run(async () => + { + await _checkpoints.ForceAsync(); // make the crash point durable, then die + AnsiConsole.MarkupLine("[red bold]☠ AUTOMIND_CHAOS: killing the process now (state is checkpointed)[/]"); + Console.Out.Flush(); + Environment.FailFast("AUTOMIND_CHAOS kill"); + }); + } + + // ---------------------------------------------------------------- lifecycle + + public string StoreDebugView => _store.DebugView; + + public int CheckpointsTaken => _checkpoints.CheckpointsTaken; + + public async ValueTask DisposeAsync() + { + await _checkpoints.DisposeAsync(); + await _engine.CheckpointAsync(); + await _engine.UnloadAsync(); + _engine.Dispose(); + _scheduler.Dispose(); + Memory?.Dispose(); + _renderer.Dispose(); // stops the heartbeat timer + + if (_mcpBridge is not null) + { + await _mcpBridge.DisposeAsync(); // shuts down the stdio child processes + } + } + + private sealed class DelegateObserver : IObserver + { + private readonly Action _onNext; + private readonly Action? _onCompleted; + private readonly Action? _onError; + + public DelegateObserver(Action onNext, Action? onCompleted = null, Action? onError = null) + { + _onNext = onNext; + _onCompleted = onCompleted; + _onError = onError; + } + + public void OnCompleted() => _onCompleted?.Invoke(); + + public void OnError(Exception error) => _onError?.Invoke(error); + + public void OnNext(T value) => _onNext(value); + } +} diff --git a/src/Automind.Cli/CheckpointCoordinator.cs b/src/Automind.Cli/CheckpointCoordinator.cs new file mode 100644 index 0000000..d9017bd --- /dev/null +++ b/src/Automind.Cli/CheckpointCoordinator.cs @@ -0,0 +1,113 @@ +using System.Threading.Channels; + +using Reaqtor.Shebang.Service; + +namespace Automind.Cli; + +/// +/// The host-side checkpoint policy: per-step (every derivation output requests one, debounced +/// 100 ms) plus a 5-second safety timer. Serialized — the engine tolerates only one checkpoint +/// at a time — and never runs on the engine scheduler (which checkpointing must pause). +/// +public sealed class CheckpointCoordinator : IAsyncDisposable +{ + private readonly SimplerCheckpointingQueryEngine _engine; + private readonly Channel _signal = Channel.CreateBounded( + new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite }); + private readonly CancellationTokenSource _cts = new(); + private readonly SemaphoreSlim _gate = new(1, 1); // the engine rejects overlapping checkpoints + private readonly Task _loop; + + public int CheckpointsTaken { get; private set; } + + public event Action? CheckpointCompleted; + + public CheckpointCoordinator(SimplerCheckpointingQueryEngine engine) + { + _engine = engine; + _loop = Task.Run(RunAsync); + } + + /// Requests a checkpoint soon (debounced/coalesced). + public void Request() => _signal.Writer.TryWrite(0); + + /// + /// Takes a checkpoint NOW and returns when it is durable — the chaos kill waits on this. + /// All checkpoint requests (loop, completion, chaos) serialize on one gate: the engine + /// throws on overlapping checkpoints (observed live from Complete() racing the loop). + /// + public async Task ForceAsync() + { + await _gate.WaitAsync().ConfigureAwait(false); + + try + { + using var activity = Automind.Reaqtor.Telemetry.AutomindDiagnostics.ActivitySource.StartActivity("engine.checkpoint"); + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + await _engine.CheckpointAsync().ConfigureAwait(false); + stopwatch.Stop(); + + CheckpointsTaken++; + activity?.SetTag("automind.checkpoint.number", CheckpointsTaken); + Automind.Reaqtor.Telemetry.AutomindDiagnostics.CheckpointDuration.Record(stopwatch.Elapsed.TotalMilliseconds); + CheckpointCompleted?.Invoke(stopwatch.Elapsed); + } + finally + { + _gate.Release(); + } + } + + private async Task RunAsync() + { + using var timer = new PeriodicTimer(TimeSpan.FromSeconds(5)); + + var timerTask = Task.Run(async () => + { + while (await timer.WaitForNextTickAsync(_cts.Token).ConfigureAwait(false)) + { + Request(); + } + }); + + try + { + while (await _signal.Reader.WaitToReadAsync(_cts.Token).ConfigureAwait(false)) + { + _signal.Reader.TryRead(out _); + + // Debounce: coalesce the burst of outputs one step produces. + await Task.Delay(100, _cts.Token).ConfigureAwait(false); + _signal.Reader.TryRead(out _); + + try + { + await ForceAsync().ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Spectre.Console.AnsiConsole.MarkupLineInterpolated($"[red]checkpoint failed: {ex.Message}[/]"); + } + } + } + catch (OperationCanceledException) + { + // Shutdown. + } + } + + public async ValueTask DisposeAsync() + { + await _cts.CancelAsync(); + + try + { + await _loop.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + + _cts.Dispose(); + } +} diff --git a/src/Automind.Cli/Demos.cs b/src/Automind.Cli/Demos.cs new file mode 100644 index 0000000..110ff24 --- /dev/null +++ b/src/Automind.Cli/Demos.cs @@ -0,0 +1,308 @@ +using System.Collections.Immutable; + +using Automind.Kernel.Contract; +using Automind.Tools; + +using Universalis.Core.Ir; +using Universalis.Core.Parsing; + +namespace Automind.Cli; + +/// One runnable scenario derived from the papers' worked examples. +public sealed record DemoScenario( + string Name, + string Title, + string Concept, + string[] Blurb, + Action? Tools, + IReadOnlyList? SeedRules, + Func CreateEnvelope, + Func? AfterNotes) +{ + /// A refused derivation IS this demo's success (e.g. a pre-condition violation). + public bool ExpectFailure { get; init; } +} + +public static class Demos +{ + public static IReadOnlyList All => + [ + WeatherRule(), + BulkPdf(), + Contracts(), + ContractsViolation(), + BtcDecision(), + TeamSelection(), + ]; + + public static DemoScenario? Find(string name) => + All.FirstOrDefault(d => string.Equals(d.Name, name, StringComparison.OrdinalIgnoreCase)); + + // ================================================================ paper 1: the composed WEATHER rule + + private static DemoScenario WeatherRule() + { + // The canonical literate-Prolog rule from paper 1: WEATHER is DEFINED in Universalis as + // a chain of three primitive facts, with JSON patterns digging the fields out of the + // messy API responses. The stored rule SHADOWS the primitive WEATHER tool. + var rule = new RuleDefinition( + new RuleSignature("WEATHER", [ + new RuleParam("city", ParamMode.In), + new RuleParam("weather", ParamMode.Out), + ], "current weather via the National Weather Service (geo-code, point lookup, forecast fetch)"), + "Find the current weather in a city using the National Weather Service API", + new UniversalisProgram( + [ + new Comment("To find the current weather, we first need the coordinates of the city "), + Hedge("GEO_CODE(@city, @lat, @lon)"), + new Comment(". Given the coordinates, we look up the forecast URL "), + Hedge("WEATHER_GOV(@lat, @lon, { ... \"forecast\": @url ... })"), + new Comment(", and finally we fetch the detailed forecast "), + Hedge("HTTP_GET(@url, { ... \"detailedForecast\": @weather ... })"), + new Comment("."), + ], [], [])); + + return new DemoScenario( + "weather-rule", + "The composed WEATHER rule (paper 1)", + "rules as new tools · JSON pattern matching · zero LLM calls inside a rule", + [ + "Paper 1 defines WEATHER not as a primitive but as a Universalis rule chaining", + "GEO_CODE → WEATHER_GOV → HTTP_GET, destructuring each nested API response with", + "{ ... \"field\": @var ... } patterns. Here that rule is pre-stored and SHADOWS the", + "primitive WEATHER tool. Watch: one rule invocation → three tool calls, no LLM", + "between them — and the answer is the NWS-style detailed forecast, not the", + "primitive tool's short string (proof the rule ran).", + ], + tools => tools.Add(DemoTools.GeoCode()).Add(DemoTools.WeatherGov()).Add(DemoTools.HttpGet()), + [rule], + _ => QuestionEnvelope.ForText("What is the current weather in Palo Alto?"), + AfterNotes: null); + } + + // ================================================================ paper 2: loopless bulk processing + + private static DemoScenario BulkPdf() + { + return new DemoScenario( + "bulk-pdf", + "Convert all files to PDF (paper 2)", + "loopless programming · implicit zip lifting over collections", + [ + "Paper 2's bulk-processing example: TO_PDF converts ONE file, but the model calls", + "it once with a LIST — no loop, no map. The engine's zip lifting fans the single", + "hedge out into one invocation per file (watch the ⚙ TO_PDF(...) ×N trace) and", + "binds the outputs back as a list. Real stub .pdf files appear on disk.", + ], + tools => tools.Add(DemoTools.ListFiles()).Add(DemoTools.ToPdf()), + SeedRules: null, + dataDir => + { + var docs = Path.Combine(dataDir, "docs"); + Directory.CreateDirectory(docs); + File.WriteAllText(Path.Combine(docs, "notes.txt"), "meeting notes\n"); + File.WriteAllText(Path.Combine(docs, "report.txt"), "quarterly report\n"); + File.WriteAllText(Path.Combine(docs, "summary.txt"), "executive summary\n"); + + foreach (var stale in Directory.GetFiles(docs, "*.pdf")) + { + File.Delete(stale); + } + + // The path rides in σ as @dir (the papers' live-programming inputs) — the model + // manipulates the NAME and never has to re-type a fragile Windows path. + return QuestionEnvelope.ForText("Convert all files in the directory @dir to PDF.") with + { + InitialBindings = new Dictionary + { + ["dir"] = System.Text.Json.JsonSerializer.Serialize(docs), + }.ToImmutableDictionary(), + }; + }, + dataDir => + { + var docs = Path.Combine(dataDir, "docs"); + var pdfs = Directory.GetFiles(docs, "*.pdf").Select(Path.GetFileName).OrderBy(f => f).ToArray(); + + return + [ + $"files now in {docs}:", + .. Directory.GetFiles(docs).Select(f => " " + Path.GetFileName(f)).OrderBy(s => s), + pdfs.Length > 0 + ? $"→ {pdfs.Length} PDF(s) created by ONE hedge, fanned out by zip lifting." + : "→ no PDFs were created (the derivation likely failed — see the trace).", + ]; + }); + } + + // ================================================================ paper 2: contracts + + private static DemoScenario Contracts() + { + return new DemoScenario( + "contracts", + "Apples with pre/post-conditions (paper 2)", + "contracts as AI safety · pre gates the run · post verifies the result", + [ + "Paper 2 attaches data-validation-style contracts to the apples question. The", + "pre-condition (the buying price must be positive) is checked BEFORE any LLM", + "call; the post-condition re-derives the profit formula and verifies the model's", + "answer against it. Run `demo contracts-violation` to see the pre-condition", + "refuse bad input without spending a single token.", + ], + Tools: null, + SeedRules: null, + _ => new QuestionEnvelope( + "Alice bought a kilo of apples for $B and sold them for $S. How much percent profit or loss did Alice make? Name the result @P.", + InitialBindings: new Dictionary { ["B"] = "10", ["S"] = "17" }.ToImmutableDictionary(), + ExpectedOutputs: ["P"], + LearnRuleOnSuccess: false, + Pre: + [ + new ContractClauseText("@B > 0", "the buying price must be greater than 0 — Alice paid a positive amount for the apples"), + new ContractClauseText("@S >= 0", "the selling price must be non-negative"), + ], + Post: + [ + new ContractClauseText("@P == ((@S - @B) / @B) * 100", "the reported percentage must equal the profit formula"), + ]), + AfterNotes: null); + } + + /// The violation variant, exposed as its own name for discoverability. + public static DemoScenario ContractsViolation() => Contracts() with + { + Name = "contracts-violation", + Title = "Pre-condition violation (paper 2)", + Blurb = + [ + "The same apples question with B=0: the pre-condition @B > 0 fails, so the", + "derivation is refused immediately — zero LLM calls, exactly like Excel rejecting", + "input that breaks a data-validation rule. The rationale is shown to the user.", + ], + CreateEnvelope = _ => new QuestionEnvelope( + "Alice got a kilo of apples for $B and sold them for $S. How much percent profit did she make?", + InitialBindings: new Dictionary { ["B"] = "0", ["S"] = "5" }.ToImmutableDictionary(), + ExpectedOutputs: ["P"], + LearnRuleOnSuccess: false, + Pre: [new ContractClauseText("@B > 0", "the buying price must be greater than 0 — Alice paid a positive amount for the apples")]), + ExpectFailure = true, + }; + + // ================================================================ paper 2: conditional decision + + private static DemoScenario BtcDecision() + { + return new DemoScenario( + "btc-decision", + "BTC or MSFT? (paper 2)", + "conditionals as checklists · tools + patterns feeding a decision", + [ + "Paper 2's decision example: fetch the MSFT price (nested in a messy quote blob),", + "find the BTC price via search, compare the totals, and decide via a checklist —", + "the engine evaluates the guards, runs the taken branch, and crosses out the", + "other. With 0.05 BTC (~$2,162) against 10 MSFT (~$4,289), Erik keeps his BTC.", + ], + tools => tools.Add(DemoTools.Stock()).Add(DemoTools.Search()), + SeedRules: null, + _ => new QuestionEnvelope( + // @shares, not @msft: models read "@msft" as a price/symbol and invent + // conversions around it (observed live: "@sharePrice is 10", COIN_TO_USD). + "Erik has @btc BTC. If that is enough to buy @shares MSFT shares, he should buy them; otherwise he keeps the BTC. Decide, and show the remaining BTC either way as @btcLeft.", + InitialBindings: new Dictionary { ["btc"] = "0.05", ["shares"] = "10" }.ToImmutableDictionary(), + ExpectedOutputs: ["btcLeft"], + LearnRuleOnSuccess: false), + AfterNotes: null); + } + + // ================================================================ paper 2: the big query + + private static DemoScenario TeamSelection() + { + const string Players = """ + [{"position":"Forward","stats":120,"games":10,"age":25}, + {"position":"Forward","stats":110,"games":8,"age":27}, + {"position":"Midfielder","stats":105,"games":9,"age":24}, + {"position":"Midfielder","stats":98,"games":11,"age":26}, + {"position":"Defender","stats":92,"games":12,"age":24}, + {"position":"Defender","stats":88,"games":10,"age":28}, + {"position":"Goalie","stats":150,"games":2,"age":30}] + """; + + // Paper 2's flagship query stored as an INTENTIONAL PROGRAM: the checklist compiled to + // IR, saved as a rule. Live 8B models channel simple queries (see the customers few-shot) + // but reliably mangle the five-bullet grouped form — so this demo shows the papers' other + // mechanism: the query as a durable named program the model merely invokes. + var query = new ComprehensionBlock( + ItemVar: "p", + ItemPattern: new ObjectPatternTerm([ + new PatternField("position", new VarTerm("position")), + new PatternField("stats", new VarTerm("stats")), + new PatternField("games", new VarTerm("games")), + ], IsOpen: true), + SourceVar: "players", + Ops: + [ + new GroupByOp("position", "Group each player [@p] by its position [@position]."), + new AggregateOp(AggregateFn.Mean, "stats", "averageStats", "averageStats", + "Determine the average stats [@stats] as [{ \"averageStats\": @averageStats }]."), + new AggregateOp(AggregateFn.Min, "games", "minGames", "minGames", + "Find the minimum games [@games] as [{ \"minGames\": @minGames }]."), + new CollectOp("p", "members", "members", + "Collect the players [@p] as [{ \"members\": @members }]."), + new FilterOp(new Comparison(CompareOp.Gt, new VarTerm("averageStats"), new NumTerm(100)), + "Keep only groups where [@averageStats > 100]."), + new FilterOp(new Comparison(CompareOp.Gt, new VarTerm("minGames"), new NumTerm(3)), + "Keep only groups where [@minGames > 3]."), + ], + IntoVar: "strongGroups"); + + var rule = new RuleDefinition( + new RuleSignature("STRONG_GROUPS", [ + new RuleParam("players", ParamMode.In), + new RuleParam("strongGroups", ParamMode.Out), + ], "the strong position groups of a squad: average stats above 100 and minimum games above 3, with each group's members collected"), + "Group players by position, aggregate each group, and keep only the strong ones", + new UniversalisProgram( + [ + new Comment("Consider each player, group by position, aggregate each group, and keep the strong ones "), + query, + new Comment("."), + ], [], [])); + + return new DemoScenario( + "team-selection", + "World Cup team selection (paper 2)", + "query comprehensions · group / aggregate / collect / HAVING · intentional programs", + [ + "Paper 2's flagship query: group players by position, average each group's stats,", + "find its minimum games, COLLECT the group's players (fully nested results — the", + "anti-SQL superpower), and keep only strong, experienced groups. The query is", + "stored as an intentional program (a STRONG_GROUPS rule) that compiles to a LINQ", + "pipeline over the JSON rows; the model channels ONE invocation hedge and the", + "engine runs the whole query — no LLM calls inside. The Goalie group tops the", + "stats chart yet is dropped: minimum games is 2.", + ], + Tools: null, + [rule], + _ => new QuestionEnvelope( + "Call STRONG_GROUPS on the players in @Players and show the result.", + InitialBindings: new Dictionary { ["Players"] = Players }.ToImmutableDictionary(), + // The assembler appends "@strongGroups = " to the answer — the model + // narrates the call but rarely emits a display hedge for a structured result. + ExpectedOutputs: ["strongGroups"], + LearnRuleOnSuccess: false), + AfterNotes: null); + } + + // ================================================================ helpers + + private static HedgeItem Hedge(string content) + { + var parsed = HedgeParser.Parse(content); + + return parsed.Success + ? new HedgeItem(parsed.Statement!, content) + : throw new InvalidOperationException($"demo rule hedge failed to parse: [{content}] — {parsed.Error}"); + } +} diff --git a/src/Automind.Cli/OtelFileLog.cs b/src/Automind.Cli/OtelFileLog.cs new file mode 100644 index 0000000..fcd4695 --- /dev/null +++ b/src/Automind.Cli/OtelFileLog.cs @@ -0,0 +1,133 @@ +using System.Diagnostics; +using System.Text.Json; + +using OpenTelemetry; +using OpenTelemetry.Metrics; + +namespace Automind.Cli; + +/// +/// A minimal JSON-lines telemetry sink: every finished span (with its automind.trace.* +/// events — the kernel's full logic flow) and every periodic metric snapshot appends one JSON +/// object per line. OpenTelemetry for .NET ships no file exporter; this keeps the POC +/// self-contained and gives every run a greppable observability artifact. +/// +public sealed class OtelFileLog : IDisposable +{ + private static readonly JsonSerializerOptions s_json = new() { DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; + + private readonly StreamWriter _writer; + private readonly Lock _gate = new(); + + public OtelFileLog(string path) + { + var full = Path.GetFullPath(path); + + if (Path.GetDirectoryName(full) is { Length: > 0 } directory) + { + Directory.CreateDirectory(directory); + } + + _writer = new StreamWriter(new FileStream(full, FileMode.Append, FileAccess.Write, FileShare.Read)); + } + + public void WriteLine(object record) + { + var line = JsonSerializer.Serialize(record, s_json); + + lock (_gate) + { + _writer.WriteLine(line); + _writer.Flush(); // a crash mid-run must not lose the telemetry that explains it + } + } + + public void Dispose() + { + lock (_gate) + { + _writer.Dispose(); + } + } +} + +internal sealed class FileActivityExporter(OtelFileLog log) : BaseExporter +{ + public override ExportResult Export(in Batch batch) + { + foreach (var activity in batch) + { + log.WriteLine(new + { + type = "span", + name = activity.DisplayName, + traceId = activity.TraceId.ToString(), + spanId = activity.SpanId.ToString(), + start = activity.StartTimeUtc, + durationMs = Math.Round(activity.Duration.TotalMilliseconds, 3), + status = activity.Status == ActivityStatusCode.Unset ? null : activity.Status.ToString(), + statusDescription = activity.StatusDescription, + tags = activity.TagObjects.ToDictionary(tag => tag.Key, tag => tag.Value), + events = activity.Events.Any() + ? activity.Events.Select(evt => new + { + name = evt.Name, + time = evt.Timestamp.UtcDateTime, + tags = evt.Tags.ToDictionary(tag => tag.Key, tag => tag.Value), + }).ToArray() + : null, + }); + } + + return ExportResult.Success; + } +} + +internal sealed class FileMetricExporter(OtelFileLog log) : BaseExporter +{ + public override ExportResult Export(in Batch batch) + { + foreach (var metric in batch) + { + foreach (ref readonly var point in metric.GetMetricPoints()) + { + object? value = metric.MetricType switch + { + MetricType.LongSum => point.GetSumLong(), + MetricType.DoubleSum => point.GetSumDouble(), + // min/max included: "slowest segment" questions kept falling back to span + // scans without them. + MetricType.Histogram => point.TryGetHistogramMinMaxValues(out var min, out var max) + ? new + { + count = point.GetHistogramCount(), + sum = Math.Round(point.GetHistogramSum(), 3), + min = (double?)Math.Round(min, 3), + max = (double?)Math.Round(max, 3), + } + : new + { + count = point.GetHistogramCount(), + sum = Math.Round(point.GetHistogramSum(), 3), + min = (double?)null, + max = (double?)null, + }, + MetricType.LongGauge => point.GetGaugeLastValueLong(), + MetricType.DoubleGauge => point.GetGaugeLastValueDouble(), + _ => null, + }; + + log.WriteLine(new + { + type = "metric", + name = metric.Name, + unit = metric.Unit is { Length: > 0 } unit ? unit : null, + value, + at = DateTime.UtcNow, + }); + } + } + + return ExportResult.Success; + } +} diff --git a/src/Automind.Cli/Program.cs b/src/Automind.Cli/Program.cs new file mode 100644 index 0000000..67eb66f --- /dev/null +++ b/src/Automind.Cli/Program.cs @@ -0,0 +1,431 @@ +using System.ComponentModel; + +using Automind.Cli; + +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; + +using Spectre.Console; +using Spectre.Console.Cli; + +// The trace glyphs (⚙ ▣ ⇝ ✓ ↩) need UTF-8 — legacy Windows console code pages render them +// as '?' / '␦'. Harmless if the handle is redirected or the encoding cannot be changed. +try +{ + Console.OutputEncoding = System.Text.Encoding.UTF8; +} +catch (Exception) +{ + // non-console host; keep whatever encoding is in effect +} + +var app = new CommandApp(); + +app.Configure(config => +{ + config.SetApplicationName("automind"); + + config.AddCommand("ask") + .WithDescription("Ask one question, stream the derivation live, print the answer, exit.") + .WithExample("ask", "\"What is the current weather in Palo Alto?\""); + + config.AddCommand("repl") + .WithDescription("Interactive session. In-flight derivations resume automatically after a crash/restart."); + + config.AddCommand("resume") + .WithDescription("Recover the engine and finish any in-flight derivations, then exit."); + + config.AddCommand("rules") + .WithDescription("List the learned rules in the durable library."); + + config.AddCommand("learn-doc") + .WithDescription("Ingest a text/markdown file into the durable virtual memory (RAG)."); + + config.AddCommand("demo") + .WithDescription("Run a worked example from the papers ('demo list' shows them all).") + .WithExample("demo", "weather-rule"); + + config.AddCommand("store") + .WithDescription("Dump the durable store's tables (engine artifacts + Automind catalogs)."); +}); + +return await app.RunAsync(args); + +public class HostSettings : CommandSettings +{ + [CommandOption("--data ")] + [Description("Durable state directory (the thing that survives kills).")] + public string DataDirectory { get; init; } = "data"; + + [CommandOption("--endpoint ")] + [Description("Ollama endpoint.")] + public string Endpoint { get; init; } = + Environment.GetEnvironmentVariable("AUTOMIND_OLLAMA") ?? "http://localhost:11434"; + + [CommandOption("--model ")] + [Description("Ollama chat model.")] + public string Model { get; init; } = + Environment.GetEnvironmentVariable("AUTOMIND_OLLAMA_MODEL") ?? "granite3.3:8b"; + + [CommandOption("--quiet")] + [Description("Hide raw generation segments in the trace.")] + public bool Quiet { get; init; } + + [CommandOption("--otel")] + [Description("Export OpenTelemetry traces/metrics to the console (or OTLP when OTEL_EXPORTER_OTLP_ENDPOINT is set).")] + public bool OpenTelemetry { get; init; } + + [CommandOption("--otel-log ")] + [Description("Append OpenTelemetry spans and metric snapshots to this file as JSON lines (composes with --otel).")] + public string? OtelLogPath { get; init; } + + [CommandOption("--mcp ")] + [Description("Bridge an MCP stdio server's tools as Universalis predicates — one command line per flag (e.g. --mcp \"dotnet tools/McpSampleServer/bin/Debug/net10.0/McpSampleServer.dll\"). Repeatable.")] + public string[]? McpServers { get; init; } + + [CommandOption("--mcp-tools ")] + [Description("Comma-separated allowlist of MCP tool names to bridge (default: all).")] + public string? McpTools { get; init; } + + public HostOptions ToOptions() => new(DataDirectory, Endpoint, Model, Verbose: !Quiet) + { + McpServers = McpServers, + McpToolFilter = McpTools, + }; + + /// Builds the OTel providers when requested; dispose to flush. + public IDisposable? StartTelemetry() + { + if (!OpenTelemetry && OtelLogPath is null) + { + return null; + } + + var traces = global::OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(Automind.Reaqtor.Telemetry.AutomindDiagnostics.SourceName); + var metrics = global::OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(Automind.Reaqtor.Telemetry.AutomindDiagnostics.SourceName); + + var fileLog = OtelLogPath is null ? null : new OtelFileLog(OtelLogPath); + + if (fileLog is not null) + { + // Simple (per-span, in-order) rather than batched: this is a diagnostic log — spans + // are low-volume and a crash must not swallow the tail that explains it. + traces = traces.AddProcessor(new global::OpenTelemetry.SimpleActivityExportProcessor(new FileActivityExporter(fileLog))); + metrics = metrics.AddReader(new global::OpenTelemetry.Metrics.PeriodicExportingMetricReader( + new FileMetricExporter(fileLog), exportIntervalMilliseconds: 5000)); + } + + if (OpenTelemetry) + { + var otlp = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT"); + + if (otlp is not null) + { + traces = traces.AddOtlpExporter(); + metrics = metrics.AddOtlpExporter(); + } + else + { + traces = traces.AddConsoleExporter(); + metrics = metrics.AddConsoleExporter(); + } + } + + var tracerProvider = traces.Build(); + var meterProvider = metrics.Build(); + + return new Telemetry(tracerProvider, meterProvider, fileLog); + } + + private sealed record Telemetry(IDisposable Traces, IDisposable Metrics, IDisposable? FileLog) : IDisposable + { + public void Dispose() + { + // Providers first — disposing them flushes the final metric snapshot into the sink. + Traces.Dispose(); + Metrics.Dispose(); + FileLog?.Dispose(); + } + } +} + +public sealed class AskSettings : HostSettings +{ + [CommandArgument(0, "")] + [Description("The question for the neural computer.")] + public string Question { get; init; } = ""; + + [CommandOption("--learn ")] + [Description("On success, save the derivation as a reusable rule with this name.")] + public string? Learn { get; init; } + + [CommandOption("--mode-b")] + [Description("Mode B: whole-program synthesis — one structured completion returns the complete program as JSON instead of streaming hedge by hedge (the conformance floor for weaker models).")] + public bool ModeB { get; init; } +} + +public sealed class AskCommand : AsyncCommand +{ + protected override async Task ExecuteAsync(CommandContext context, AskSettings settings, CancellationToken cancellationToken) + { + using var telemetry = settings.StartTelemetry(); + await using var host = await StartHostAsync(settings); + + var pending = await ResumeInFlightAsync(host); + + var id = await host.AskAsync(settings.Question, settings.Learn, settings.ModeB); + AnsiConsole.MarkupLineInterpolated($"[grey]derivation {id} started — kill this process at any point and re-run to resume[/]"); + + var result = await host.WaitForAsync(id); + + foreach (var resumed in pending) + { + await resumed; + } + + return result.Succeeded ? 0 : 1; + } + + internal static async Task StartHostAsync(HostSettings settings) + { + AutomindHost host = null!; + + await AnsiConsole.Status().StartAsync("starting engine…", async _ => + { + host = await AutomindHost.StartAsync(settings.ToOptions()); + }); + + if (host.Recovered) + { + AnsiConsole.MarkupLineInterpolated($"[green]engine recovered[/] from [grey]{settings.DataDirectory}[/] — {host.ResumedDerivations.Count} in-flight derivation(s)"); + } + else + { + AnsiConsole.MarkupLineInterpolated($"[green]engine created[/] — durable state in [grey]{settings.DataDirectory}[/]"); + } + + return host; + } + + internal static Task>> ResumeInFlightAsync(AutomindHost host) + { + var waits = new List>(); + + foreach (var id in host.ResumedDerivations) + { + AnsiConsole.MarkupLineInterpolated($"[yellow]⟲ resuming derivation {id} — replaying its trace[/]"); + waits.Add(host.WaitForAsync(id)); + } + + return Task.FromResult(waits); + } +} + +public sealed class ReplCommand : AsyncCommand +{ + protected override async Task ExecuteAsync(CommandContext context, HostSettings settings, CancellationToken cancellationToken) + { + using var telemetry = settings.StartTelemetry(); + await using var host = await AskCommand.StartHostAsync(settings); + + foreach (var wait in await AskCommand.ResumeInFlightAsync(host)) + { + await wait; // let recovered derivations finish (their traces stream live) + } + + AnsiConsole.MarkupLine("[grey]type a question, or 'exit' to quit (killing the process mid-derivation is encouraged — that's the demo)[/]"); + + while (true) + { + var question = AnsiConsole.Prompt(new TextPrompt("[bold cyan]?[/]").AllowEmpty()); + + if (string.IsNullOrWhiteSpace(question)) + { + continue; + } + + if (question.Trim().Equals("exit", StringComparison.OrdinalIgnoreCase)) + { + return 0; + } + + var id = await host.AskAsync(question.Trim()); + await host.WaitForAsync(id); + } + } +} + +public sealed class ResumeCommand : AsyncCommand +{ + protected override async Task ExecuteAsync(CommandContext context, HostSettings settings, CancellationToken cancellationToken) + { + using var telemetry = settings.StartTelemetry(); + await using var host = await AskCommand.StartHostAsync(settings); + + var waits = await AskCommand.ResumeInFlightAsync(host); + + if (waits.Count == 0) + { + AnsiConsole.MarkupLine("[grey]nothing to resume[/]"); + return 0; + } + + var results = await Task.WhenAll(waits); + return results.All(r => r.Succeeded) ? 0 : 1; + } +} + +public sealed class DemoSettings : HostSettings +{ + [CommandArgument(0, "")] + [Description("Demo name, or 'list' to enumerate the available demos.")] + public string Name { get; init; } = "list"; +} + +public sealed class DemoCommand : AsyncCommand +{ + protected override async Task ExecuteAsync(CommandContext context, DemoSettings settings, CancellationToken cancellationToken) + { + if (settings.Name.Equals("list", StringComparison.OrdinalIgnoreCase)) + { + var table = new Table().RoundedBorder() + .AddColumn("demo").AddColumn("shows").AddColumn("from"); + + foreach (var scenario in Demos.All) + { + table.AddRow(scenario.Name, scenario.Concept, scenario.Title); + } + + AnsiConsole.Write(table); + AnsiConsole.MarkupLine("[grey]run one with: automind demo [/]"); + return 0; + } + + var demo = Demos.Find(settings.Name); + + if (demo is null) + { + AnsiConsole.MarkupLineInterpolated($"[red]unknown demo '{settings.Name}' — try 'automind demo list'[/]"); + return 1; + } + + using var telemetry = settings.StartTelemetry(); + + AnsiConsole.Write(new Panel(string.Join("\n", demo.Blurb)) + .Header($"[bold] {demo.Title} [/]") + .BorderColor(Color.Blue) + .RoundedBorder()); + + var options = settings.ToOptions() with + { + ConfigureTools = demo.Tools, + SeedRules = demo.SeedRules, + }; + + AutomindHost host = null!; + await AnsiConsole.Status().StartAsync("starting engine…", async _ => + { + host = await AutomindHost.StartAsync(options); + }); + + await using (host) + { + var envelope = demo.CreateEnvelope(Path.GetFullPath(settings.DataDirectory)); + AnsiConsole.MarkupLineInterpolated($"[bold cyan]?[/] {envelope.Text}"); + + var id = await host.AskAsync(envelope); + var result = await host.WaitForAsync(id); + + foreach (var note in demo.AfterNotes?.Invoke(Path.GetFullPath(settings.DataDirectory)) ?? []) + { + AnsiConsole.MarkupLineInterpolated($"[grey]{note}[/]"); + } + + if (demo.ExpectFailure) + { + AnsiConsole.MarkupLine(result.Succeeded + ? "[red]→ this demo expected a refusal, but the derivation succeeded.[/]" + : "[green]→ the refusal above is this demo's expected outcome.[/]"); + + return result.Succeeded ? 1 : 0; + } + + return result.Succeeded ? 0 : 1; + } + } +} + +public sealed class LearnDocSettings : HostSettings +{ + [CommandArgument(0, "")] + [Description("Path to a text/markdown file to ingest.")] + public string File { get; init; } = ""; + + [CommandOption("--name ")] + [Description("Document name in the memory (defaults to the file name).")] + public string? Name { get; init; } +} + +public sealed class LearnDocCommand : AsyncCommand +{ + protected override async Task ExecuteAsync(CommandContext context, LearnDocSettings settings, CancellationToken cancellationToken) + { + if (!File.Exists(settings.File)) + { + AnsiConsole.MarkupLineInterpolated($"[red]file not found: {settings.File}[/]"); + return 1; + } + + using var telemetry = settings.StartTelemetry(); + await using var host = await AskCommand.StartHostAsync(settings); + + var name = settings.Name ?? Path.GetFileNameWithoutExtension(settings.File); + await host.LearnDocumentAsync(name, await File.ReadAllTextAsync(settings.File, cancellationToken)); + + AnsiConsole.MarkupLineInterpolated($"[green]ingested '{name}' into the durable virtual memory[/]"); + return 0; + } +} + +public sealed class RulesCommand : AsyncCommand +{ + protected override Task ExecuteAsync(CommandContext context, HostSettings settings, CancellationToken cancellationToken) + { + var store = Automind.Reaqtor.Store.FileQueryEngineStateStore.Open(settings.DataDirectory); + var rules = new Automind.Reaqtor.Catalog.RuleCatalogStore(store).All(); + + if (rules.Count == 0) + { + AnsiConsole.MarkupLine("[grey]no learned rules yet — try: automind ask --learn myRule \"…\"[/]"); + return Task.FromResult(0); + } + + var table = new Table().RoundedBorder().AddColumn("rule").AddColumn("signature").AddColumn("bonsai bytes"); + + foreach (var rule in rules) + { + var definition = rule.Definition; + var signature = string.Join(", ", definition.Signature.Params.Select(p => + $"{p.Name}: {(p.Mode == Universalis.Core.Ir.ParamMode.In ? "in" : "out")}")); + + table.AddRow(rule.Name, signature, rule.BonsaiJson.Length.ToString()); + } + + AnsiConsole.Write(table); + return Task.FromResult(0); + } +} + +public sealed class StoreCommand : AsyncCommand +{ + protected override Task ExecuteAsync(CommandContext context, HostSettings settings, CancellationToken cancellationToken) + { + var store = Automind.Reaqtor.Store.FileQueryEngineStateStore.Open(settings.DataDirectory); + + AnsiConsole.WriteLine(store.DebugView); + + return Task.FromResult(0); + } +} diff --git a/src/Automind.Cli/TraceRenderer.cs b/src/Automind.Cli/TraceRenderer.cs new file mode 100644 index 0000000..affb394 --- /dev/null +++ b/src/Automind.Cli/TraceRenderer.cs @@ -0,0 +1,248 @@ +using Automind.Kernel.Contract; +using Automind.Reaqtor.Reactive; + +using Spectre.Console; + +namespace Automind.Cli; + +/// +/// The live-programming view: renders derivation outputs as they stream from the engine — +/// the model's prose dimmed, hedges and bindings highlighted, ⇝ displays emphasized (values +/// the USER sees but the model never does), backtracks and repairs annotated. Generation wall +/// time is made visible (measured live: the same derivation swings 6–8× with Ollama's mood): +/// slow segments are annotated, a heartbeat line breaks long silences, and every answer gets +/// an LLM-time footer. +/// +public sealed class TraceRenderer : IDisposable +{ + private readonly Lock _gate = new(); + private readonly Timer _heartbeat; + + private DateTime _lastEventAt; + private bool _terminal; + private int _segments; + private double _generationTotalSeconds; + private double _generationMaxSeconds; + + public TraceRenderer() + { + _lastEventAt = DateTime.UtcNow; + _heartbeat = new Timer(_ => Heartbeat(), null, TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(15)); + } + + public bool Verbose { get; init; } = true; + + public void Dispose() => _heartbeat.Dispose(); + + public void Render(DerivationOutput output) + { + lock (_gate) + { + var now = DateTime.UtcNow; + var gap = now - _lastEventAt; + _lastEventAt = now; + + switch (output.Kind) + { + case DerivationOutput.TraceKind: + _terminal = false; + RenderTrace(output, gap); + break; + + case DerivationOutput.AnswerKind: + AnsiConsole.Write(new Panel(Markup.FromInterpolated($"[bold green]{output.PayloadJson}[/]")) + .Header($"[green] answer · {output.DerivationId} [/]") + .BorderColor(Color.Green) + .RoundedBorder()); + FinishDerivation(); + break; + + case DerivationOutput.FailedKind: + AnsiConsole.Write(new Panel(Markup.FromInterpolated($"[bold red]{output.PayloadJson}[/]")) + .Header($"[red] failed · {output.DerivationId} [/]") + .BorderColor(Color.Red) + .RoundedBorder()); + FinishDerivation(); + break; + + case DerivationOutput.RuleKind: + string ruleName; + try + { + ruleName = System.Text.Json.JsonDocument.Parse(output.PayloadJson) + .RootElement.GetProperty("name").GetString() ?? "?"; + } + catch (System.Text.Json.JsonException) + { + ruleName = "?"; + } + + AnsiConsole.MarkupLineInterpolated($"[blue]★ rule learned: {ruleName}[/]"); + break; + } + } + } + + /// Long silences get a visible pulse — a healthy slow generation must not read as a hang. + private void Heartbeat() + { + lock (_gate) + { + if (_terminal) + { + return; + } + + var quiet = DateTime.UtcNow - _lastEventAt; + + if (quiet > TimeSpan.FromSeconds(20)) + { + AnsiConsole.MarkupLineInterpolated($" [grey]⋯ still generating ({(int)quiet.TotalSeconds} s since the last step)[/]"); + } + } + } + + private void FinishDerivation() + { + if (_segments > 0) + { + AnsiConsole.MarkupLineInterpolated( + $"[grey]LLM: {_segments} segment(s), {_generationTotalSeconds:n0} s total, slowest {_generationMaxSeconds:n0} s[/]"); + } + + _terminal = true; + _segments = 0; + _generationTotalSeconds = 0; + _generationMaxSeconds = 0; + } + + private void RenderTrace(DerivationOutput output, TimeSpan gap) + { + TraceEvent trace; + try + { + trace = TraceEvent.FromJson(output.PayloadJson); + } + catch (System.Text.Json.JsonException) + { + AnsiConsole.MarkupLineInterpolated($" [grey]{output.PayloadJson}[/]"); + return; + } + + // A segment's arrival gap IS its generation wall time (the burst of execution traces + // that follows a segment lands within milliseconds of it). + if (trace is SegmentReceived) + { + _segments++; + _generationTotalSeconds += gap.TotalSeconds; + _generationMaxSeconds = Math.Max(_generationMaxSeconds, gap.TotalSeconds); + } + + switch (trace) + { + case SegmentReceived segment when Verbose: + var waited = gap.TotalSeconds >= 10 ? $" ({(int)gap.TotalSeconds} s)" : ""; + AnsiConsole.MarkupLineInterpolated($" [grey]⋯ {Truncate(segment.Text, 160)}{waited}[/]"); + break; + + case VariableBound bound: + AnsiConsole.MarkupLineInterpolated($" [cyan]@{bound.Name}[/] ← [white]{Truncate(bound.ValueJson, 100)}[/] [grey](via {bound.Source})[/]"); + break; + + case DisplayShown display: + AnsiConsole.MarkupLineInterpolated($" [bold yellow]⇝ {display.Formatted}[/]"); + break; + + case ToolInvoked tool: + var lifted = tool.Invocations > 1 ? $" ×{tool.Invocations}" : ""; + AnsiConsole.MarkupLineInterpolated($" [magenta]⚙ {tool.Tool}({Truncate(tool.ArgsJson, 80)}){lifted}[/]"); + break; + + case GuardEvaluated guard: + // Verdict styling lives in the literal template part — interpolated values are escaped. + if (guard.Result) + { + AnsiConsole.MarkupLineInterpolated($" [blue]? {guard.GuardText}[/] → [green]true[/]"); + } + else + { + AnsiConsole.MarkupLineInterpolated($" [blue]? {guard.GuardText}[/] → [red]false[/]"); + } + + break; + + case BranchTaken taken: + AnsiConsole.MarkupLineInterpolated($" [green]▶ branch {taken.Branch + 1} taken[/]"); + break; + + case BranchCrossedOut crossed: + AnsiConsole.MarkupLineInterpolated($" [grey strikethrough]✗ branch {crossed.Branch + 1}: {crossed.GuardText}[/]"); + break; + + case BacktrackStarted backtrack: + AnsiConsole.MarkupLineInterpolated($" [yellow]↩ backtrack (attempt {backtrack.Attempt + 1}): {Truncate(backtrack.Reason, 140)}[/]"); + break; + + case BranchAbandoned when !Verbose: + break; + + case BranchAbandoned abandoned: + AnsiConsole.MarkupLineInterpolated($" [grey strikethrough]{Truncate(abandoned.DiscardedText, 120)}[/]"); + break; + + case ProtocolRepaired repaired: + AnsiConsole.MarkupLineInterpolated($" [grey]✎ auto-repaired ({repaired.What})[/]"); + break; + + case QueryExecuted query: + AnsiConsole.MarkupLineInterpolated($" [blue]∑ query → @{query.IntoVar}: {query.RowsIn} rows in, {query.RowsOut} out[/]"); + break; + + case ContractChecked contract: + // NB: interpolated values are markup-escaped, so the verdict styling must live + // in the literal part of the template. + if (contract.Passed) + { + AnsiConsole.MarkupLineInterpolated($" [green]✓[/] {(contract.IsPre ? "pre" : "post")}-condition holds: {Truncate(contract.ClauseText, 100)}"); + } + else + { + AnsiConsole.MarkupLineInterpolated($" [red]✗[/] {(contract.IsPre ? "pre" : "post")}-condition violated: {Truncate(contract.ClauseText, 100)}"); + } + + break; + + case RuleInvoked invoked: + AnsiConsole.MarkupLineInterpolated($" [blue]▣ rule {invoked.Name} invoked (runs without the LLM)[/]"); + break; + + case RuleLearned learned: + AnsiConsole.MarkupLineInterpolated($" [blue]★ learned rule {learned.Name}: {learned.Signature}[/]"); + break; + + case DerivationFailed failed: + AnsiConsole.MarkupLineInterpolated($" [red]✖ {failed.Reason}[/]"); + break; + + case AnswerAssembled: + case CommentAdded: + case HedgeParsed: + case SegmentReceived: + break; + + default: + if (Verbose) + { + AnsiConsole.MarkupLineInterpolated($" [grey]{Truncate(output.PayloadJson, 120)}[/]"); + } + + break; + } + } + + private static string Truncate(string text, int max) + { + var flat = text.Replace('\n', ' ').Replace('\r', ' '); + return flat.Length <= max ? flat : flat[..max] + "…"; + } +} diff --git a/src/Automind.Kernel/Automind.Kernel.csproj b/src/Automind.Kernel/Automind.Kernel.csproj new file mode 100644 index 0000000..041a77a --- /dev/null +++ b/src/Automind.Kernel/Automind.Kernel.csproj @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/Automind.Kernel/Contract/DerivationEvents.cs b/src/Automind.Kernel/Contract/DerivationEvents.cs new file mode 100644 index 0000000..3130378 --- /dev/null +++ b/src/Automind.Kernel/Contract/DerivationEvents.cs @@ -0,0 +1,82 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; + +namespace Automind.Kernel.Contract; + +/// An input to the derivation state machine (delivered at-least-once by the substrate). +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$kind")] +[JsonDerivedType(typeof(QuestionReceived), "question")] +[JsonDerivedType(typeof(LlmCompleted), "llm")] +[JsonDerivedType(typeof(ToolSucceeded), "toolOk")] +[JsonDerivedType(typeof(ToolFailed), "toolFail")] +public abstract record DerivationEvent; + +public sealed record QuestionReceived(string QuestionJson) : DerivationEvent; + +/// +/// A generation segment finished. is true when the bridge cut the +/// stream at a balanced hedge close (the ] itself excluded — the engine owns that bracket); +/// false means the model stopped on its own (completion, or a truncated hedge). +/// +public sealed record LlmCompleted(string RequestId, string Text, bool StoppedAtHedge, bool Truncated = false) : DerivationEvent; + +/// One tool invocation succeeded. holds the observable's values (0..n). +public sealed record ToolSucceeded(string RequestId, ImmutableArray ResultsJson) : DerivationEvent; + +public sealed record ToolFailed(string RequestId, string Error) : DerivationEvent; + +/// An output of the state machine, realized by the substrate. +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$kind")] +[JsonDerivedType(typeof(RequestLlm), "requestLlm")] +[JsonDerivedType(typeof(InvokeTool), "invokeTool")] +[JsonDerivedType(typeof(EmitTrace), "trace")] +[JsonDerivedType(typeof(EmitAnswer), "answer")] +[JsonDerivedType(typeof(DefineRule), "defineRule")] +[JsonDerivedType(typeof(FailedEffect), "failed")] +public abstract record DerivationEffect; + +public sealed record RequestLlm(string RequestId, string PromptStateJson) : DerivationEffect; + +public sealed record InvokeTool(string RequestId, string ToolUri, string ArgsJson) : DerivationEffect; + +public sealed record EmitTrace(string TraceJson) : DerivationEffect; + +public sealed record EmitAnswer(string Text) : DerivationEffect; + +public sealed record DefineRule(string Name, string BonsaiJson) : DerivationEffect; + +public sealed record FailedEffect(string Reason) : DerivationEffect; + +/// A tool the model may call: its Universalis signature bound to an engine artifact URI. +public sealed record ToolBinding(PredicateSignature Signature, string ToolUri, bool IsIdempotent, string Description); + +/// +/// The injected context snapshot for one step: tool bindings and stored rules. NOT part of the +/// checkpointed state — the substrate rehydrates it from the catalog store on recovery. +/// +public sealed record StepContext( + ImmutableArray Tools, + ImmutableArray Rules) +{ + public ISignatureCatalog BuildCatalog() => new SignatureCatalog( + Tools.Select(t => t.Signature) + .Concat(Rules.Select(r => new PredicateSignature( + r.Signature.Name, + [.. r.Signature.Params.Select(p => new PredicateParam(p.Name, p.Mode))], + r.Signature.Description, + IsRule: true)))); + + public ToolBinding? FindTool(string name) => + Tools.FirstOrDefault(t => string.Equals(t.Signature.Name, name, StringComparison.OrdinalIgnoreCase)); +} + +public sealed record StepResult(DerivationState State, ImmutableArray Effects); + +/// The pure, deterministic derivation step: the papers' reasoning engine (control unit). +public interface IStepFunction +{ + StepResult Step(DerivationState state, DerivationEvent evt, StepContext context); +} diff --git a/src/Automind.Kernel/Contract/DerivationState.cs b/src/Automind.Kernel/Contract/DerivationState.cs new file mode 100644 index 0000000..5a241f9 --- /dev/null +++ b/src/Automind.Kernel/Contract/DerivationState.cs @@ -0,0 +1,145 @@ +using System.Collections.Immutable; +using System.Text.Json; +using System.Text.Json.Serialization; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; +using Universalis.Core.Parsing; + +namespace Automind.Kernel.Contract; + +/// The phase of a derivation — what external completion the machine is waiting for. +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$kind")] +[JsonDerivedType(typeof(Idle), "idle")] +[JsonDerivedType(typeof(Synthesizing), "synthesizing")] +[JsonDerivedType(typeof(AwaitingTools), "awaitingTools")] +[JsonDerivedType(typeof(Completed), "completed")] +[JsonDerivedType(typeof(FailedPhase), "failed")] +public abstract record Phase; + +public sealed record Idle : Phase; + +/// An LLM request is outstanding. +public sealed record Synthesizing : Phase; + +/// One or more tool invocations are outstanding. +public sealed record AwaitingTools : Phase; + +public sealed record Completed(string Answer) : Phase; + +public sealed record FailedPhase(string Reason) : Phase; + +/// +/// One stored-rule activation: the rule body being interpreted, its program counter, and its +/// local bindings. Rules run WITHOUT the LLM; a tool call inside a rule suspends the whole +/// machine exactly like a top-level call (checkpointable mid-rule — this is why v1 interprets +/// the IR rather than executing compiled trees). +/// +public sealed record RuleFrame( + string RuleName, + ImmutableArray Body, + int Pc, + ImmutableDictionary Locals, + ImmutableArray CallerOutArgs, + ImmutableArray FormalOuts); + +/// An in-flight (possibly zip-lifted) tool call: signatures, out-args, and collected results. +public sealed record PendingCall( + PredicateSignature Signature, + string ToolUri, + bool IsIdempotent, + ImmutableArray OutArgs, + ImmutableArray RequestIds, + ImmutableArray Results) +{ + public bool AllCollected => Results.All(r => r is not null); +} + +/// +/// A backtracking anchor: everything needed to rewind to just before an LLM generation and try an +/// alternative (the tree-of-thought search). Values are snapshots, not deltas — POC state is small. +/// +public sealed record ChoicePoint( + int ProgramLength, + ImmutableDictionary Sigma, + string AssistantPrefill, + RecognizerState Recognizer, + int Attempt); + +public sealed record RetryBudget( + int MaxAttemptsPerChoicePoint, + int MaxBacktrackDepth, + int MaxLlmRequests, + int BacktrackDepthUsed, + bool ContinuationRetryUsed) +{ + // 32 requests: successful live derivations use ≤ ~15, but the tree-of-thought RESTART is + // gated at ¾ of the budget — a 24-request budget closes that gate at request 18, exactly + // when a teach-heavy first derivation has accumulated the notes that make a fresh start + // succeed (observed live: the customers query exhausted choice points at ~request 20 with + // the restart locked out). The tail is restart room, not doomed-roll food. + public static RetryBudget Default { get; } = new(3, 4, 32, 0, false); +} + +/// +/// The complete, serializable state of one derivation — the checkpointed heart of the neural +/// computer. σ is the register set; is the model-visible trace +/// (names only, never values); is the intentional representation +/// growing as the model hallucinates the program. +/// +public sealed record DerivationState( + string ConversationId, + long NextSeq, + Phase Phase, + ImmutableArray PendingRequestIds, + QuestionEnvelope? Question, + ImmutableArray ProgramSoFar, + ImmutableDictionary Sigma, + ImmutableArray ChoicePoints, + RetryBudget Budget, + RecognizerState Recognizer, + string AssistantPrefill, + ImmutableArray EngineNotes, + PendingCall? Pending, + int LlmRequestCount, + ImmutableArray Frames) +{ + /// + /// Mode B (whole-program synthesis): the imported program still awaiting execution. + /// Empty when not mid-walk. Init-property (not positional) so old checkpoints deserialize. + /// + public ImmutableArray ModeBQueue { get; init; } = []; + + /// Mode B: index of the next queue item to execute (walk resumes here after a tool). + public int ModeBPc { get; init; } + + public static DerivationState New(string conversationId) => new( + conversationId, + NextSeq: 0, + new Idle(), + PendingRequestIds: [], + Question: null, + ProgramSoFar: [], + Sigma: ImmutableDictionary.Empty, + ChoicePoints: [], + RetryBudget.Default, + RecognizerState.Initial, + AssistantPrefill: "", + EngineNotes: [], + Pending: null, + LlmRequestCount: 0, + Frames: []); + + public bool IsTerminal => Phase is Completed or FailedPhase; + + private static readonly JsonSerializerOptions s_json = new() + { + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + public string ToJson() => JsonSerializer.Serialize(this, s_json); + + public static DerivationState FromJson(string json) => + JsonSerializer.Deserialize(json, s_json) + ?? throw new JsonException("null derivation state"); +} diff --git a/src/Automind.Kernel/Contract/QuestionEnvelope.cs b/src/Automind.Kernel/Contract/QuestionEnvelope.cs new file mode 100644 index 0000000..a441680 --- /dev/null +++ b/src/Automind.Kernel/Contract/QuestionEnvelope.cs @@ -0,0 +1,60 @@ +using System.Collections.Immutable; +using System.Text.Json; + +namespace Automind.Kernel.Contract; + +/// A contract clause as text: a Universalis condition plus its natural-language rationale. +public sealed record ContractClauseText(string Condition, string Rationale); + +/// A page of recalled knowledge, paged into the context by the memory pager (RAG as virtual memory). +public sealed record RecalledChunk(string Title, string Text); + +/// +/// The payload of a event: the user's question plus the +/// live-programming inputs (initial σ bindings, e.g. @B=10, @S=17), the declared outputs +/// to surface in the final answer, and optional pre/post-condition contracts — vanilla +/// Universalis conditions, per the paper's data-validation analogy. Pre-conditions gate the run +/// (violation = user error, no retry); post-conditions check the completed derivation +/// (violation = the model's plan was wrong → backtrack). +/// +public sealed record QuestionEnvelope( + string Text, + ImmutableDictionary InitialBindings, + ImmutableArray ExpectedOutputs, + bool LearnRuleOnSuccess, + ImmutableArray? Pre = null, + ImmutableArray? Post = null, + string? RuleName = null, + ImmutableArray? Context = null, + ImmutableArray? RecalledRules = null, + bool ModeB = false) +{ + public ImmutableArray PreClauses => Pre ?? []; + + public ImmutableArray PostClauses => Post ?? []; + + public ImmutableArray ContextChunks => Context ?? []; + + public static QuestionEnvelope ForText(string text) => new(text, ImmutableDictionary.Empty, [], false); + + public string ToJson() => JsonSerializer.Serialize(this); + + public static QuestionEnvelope FromJson(string json) + { + // Tolerance: a bare string is a question with no inputs. + try + { + var doc = JsonDocument.Parse(json); + if (doc.RootElement.ValueKind == JsonValueKind.String) + { + return ForText(doc.RootElement.GetString()!); + } + } + catch (JsonException) + { + return ForText(json); + } + + return JsonSerializer.Deserialize(json) ?? ForText(json); + } +} diff --git a/src/Automind.Kernel/Contract/TraceEvents.cs b/src/Automind.Kernel/Contract/TraceEvents.cs new file mode 100644 index 0000000..6a13ccb --- /dev/null +++ b/src/Automind.Kernel/Contract/TraceEvents.cs @@ -0,0 +1,77 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Automind.Kernel.Contract; + +/// +/// User-side trace vocabulary. Trace events MAY carry values — the paper's ⇝ is precisely +/// "visible to the user, not the model". The CLI renders these as the live-programming view. +/// +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$kind")] +[JsonDerivedType(typeof(SegmentReceived), "segment")] +[JsonDerivedType(typeof(CommentAdded), "comment")] +[JsonDerivedType(typeof(HedgeParsed), "hedge")] +[JsonDerivedType(typeof(ToolInvoked), "toolInvoked")] +[JsonDerivedType(typeof(VariableBound), "bound")] +[JsonDerivedType(typeof(DisplayShown), "display")] +[JsonDerivedType(typeof(GuardEvaluated), "guard")] +[JsonDerivedType(typeof(BranchTaken), "branchTaken")] +[JsonDerivedType(typeof(BranchCrossedOut), "branchCrossed")] +[JsonDerivedType(typeof(QueryExecuted), "query")] +[JsonDerivedType(typeof(ContractChecked), "contract")] +[JsonDerivedType(typeof(ProtocolRepaired), "repaired")] +[JsonDerivedType(typeof(BacktrackStarted), "backtrack")] +[JsonDerivedType(typeof(BranchAbandoned), "abandoned")] +[JsonDerivedType(typeof(RuleInvoked), "ruleInvoked")] +[JsonDerivedType(typeof(RuleLearned), "ruleLearned")] +[JsonDerivedType(typeof(AnswerAssembled), "answerAssembled")] +[JsonDerivedType(typeof(DerivationFailed), "derivationFailed")] +public abstract record TraceEvent(long Seq) +{ + private static readonly JsonSerializerOptions s_json = new() + { + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + public string ToJson() => JsonSerializer.Serialize(this, s_json); + + public static TraceEvent FromJson(string json) => + JsonSerializer.Deserialize(json, s_json) ?? throw new JsonException("null trace event"); +} + +public sealed record SegmentReceived(long Seq, string Text, bool EndedAtHedge) : TraceEvent(Seq); + +public sealed record CommentAdded(long Seq, string Text) : TraceEvent(Seq); + +public sealed record HedgeParsed(long Seq, string ConcreteText, string Kind) : TraceEvent(Seq); + +public sealed record ToolInvoked(long Seq, string Tool, string ArgsJson, int Invocations) : TraceEvent(Seq); + +public sealed record VariableBound(long Seq, string Name, string ValueJson, string Source) : TraceEvent(Seq); + +/// The ⇝ event: a display expression's value, shown to the user only. +public sealed record DisplayShown(long Seq, string ExprText, string Formatted) : TraceEvent(Seq); + +public sealed record GuardEvaluated(long Seq, int Branch, string GuardText, bool Result) : TraceEvent(Seq); + +public sealed record BranchTaken(long Seq, int Branch) : TraceEvent(Seq); + +public sealed record BranchCrossedOut(long Seq, int Branch, string GuardText) : TraceEvent(Seq); + +public sealed record QueryExecuted(long Seq, string IntoVar, int RowsIn, int RowsOut) : TraceEvent(Seq); + +public sealed record ContractChecked(long Seq, bool IsPre, string ClauseText, bool Passed) : TraceEvent(Seq); + +public sealed record ProtocolRepaired(long Seq, string What, string Detail) : TraceEvent(Seq); + +public sealed record BacktrackStarted(long Seq, string Reason, int ChoicePoint, int Attempt) : TraceEvent(Seq); + +public sealed record BranchAbandoned(long Seq, string DiscardedText) : TraceEvent(Seq); + +public sealed record RuleInvoked(long Seq, string Name) : TraceEvent(Seq); + +public sealed record RuleLearned(long Seq, string Name, string Signature) : TraceEvent(Seq); + +public sealed record AnswerAssembled(long Seq, string Text) : TraceEvent(Seq); + +public sealed record DerivationFailed(long Seq, string Reason) : TraceEvent(Seq); diff --git a/src/Automind.Kernel/DerivationStep.cs b/src/Automind.Kernel/DerivationStep.cs new file mode 100644 index 0000000..e85c006 --- /dev/null +++ b/src/Automind.Kernel/DerivationStep.cs @@ -0,0 +1,1809 @@ +using System.Collections.Immutable; + +using Automind.Kernel.Contract; +using Automind.Kernel.Prompting; + +using Universalis.Core.Compilation; +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; +using Universalis.Core.Parsing; +using Universalis.Core.Rendering; + +namespace Automind.Kernel; + +/// +/// The pure derivation step — the papers' reasoning engine (control unit). Consumes one event, +/// produces the successor state plus effects. Deterministic and I/O-free: request IDs come from +/// the state's sequence counter, retry temperature is a function of the attempt number, and time +/// enters only through tools. Events with unknown request IDs are no-ops, making the function +/// safe under the substrate's at-least-once redelivery. +/// +public sealed class DerivationStep : IStepFunction +{ + public StepResult Step(DerivationState state, DerivationEvent evt, StepContext context) + { + if (state.IsTerminal) + { + return new StepResult(state, []); + } + + return evt switch + { + QuestionReceived question when state.Phase is Idle => + OnQuestion(state, question, context), + + LlmCompleted llm when state.Phase is Synthesizing && state.PendingRequestIds.Contains(llm.RequestId) => + OnLlmCompleted(state, llm, context), + + ToolSucceeded ok when state.Phase is AwaitingTools && state.Pending is not null && state.Pending.RequestIds.Contains(ok.RequestId) => + OnToolSucceeded(state, ok, context), + + ToolFailed failed when state.Phase is AwaitingTools && state.Pending is not null && state.Pending.RequestIds.Contains(failed.RequestId) => + OnToolFailed(state, failed, context), + + _ => new StepResult(state, []), // stale or duplicate — idempotent no-op + }; + } + + // ================================================================ question + + private static StepResult OnQuestion(DerivationState state, QuestionReceived evt, StepContext context) + { + var envelope = QuestionEnvelope.FromJson(evt.QuestionJson); + + var b = new Builder(state with + { + Question = envelope, + Sigma = envelope.InitialBindings, + }); + + // Pre-conditions gate the run: a violation is a USER error — fail, don't retry. + foreach (var clause in envelope.PreClauses) + { + var verdict = b.CheckContract(context, clause, isPre: true); + + if (verdict is not null) + { + return verdict; + } + } + + b.IssueLlmRequest(context, newChoicePoint: true); + + return b.Freeze(); + } + + // ================================================================ LLM segments + + private static StepResult OnLlmCompleted(DerivationState state, LlmCompleted evt, StepContext context) + { + var b = new Builder(state with { PendingRequestIds = [] }); + + b.Trace(seq => new SegmentReceived(seq, evt.Text, evt.StoppedAtHedge)); + + if (b.IsModeB) + { + return OnWholeProgram(b, state, evt, context); + } + + var segments = HedgeScanner.Split(evt.Text); + + // Truncated hedge: the model stopped on its own mid-hedge → one continuation retry. + if (!evt.StoppedAtHedge && segments.Length > 0 && segments[^1].IsOpenHedge) + { + if (b.Budget.ContinuationRetryUsed) + { + return b.Backtrack(context, "the generation stopped in the middle of a [ ... ] hedge twice", null); + } + + b.Budget = b.Budget with { ContinuationRetryUsed = true }; + b.AppendPrefill(evt.Text); + b.IssueLlmRequest(context, newChoicePoint: false); // same choice point: continuation, not an alternative + return b.Freeze(); + } + + var prose = ""; + + foreach (var segment in segments) + { + if (!segment.IsHedge) + { + prose += segment.Text; + continue; + } + + // A closed hedge — or the final cut hedge (content complete; ']' is engine-owned). + var parsed = HedgeParser.Parse(segment.Text); + + if (!parsed.Success) + { + if (!segment.Text.Contains('@', StringComparison.Ordinal) && + !segment.Text.Contains('(', StringComparison.Ordinal)) + { + prose += "[" + segment.Text + "]"; // degenerate hedge → prose + continue; + } + + // A malformed hedge AFTER every declared output is bound is junk to skip, not + // a reason to unwind a finished derivation (observed live: @btcLeft bound and + // the branch taken, then [@totalCost (0.5)] — the backtrack destroyed the + // completed state and the request budget died before it could be rebuilt). + if (b.AnswerRequirementsMet) + { + b.Trace(seq => new ProtocolRepaired(seq, "junk-hedge-skipped", segment.Text)); + continue; + } + + return b.Backtrack(context, $"cannot parse [{segment.Text}]", parsed.Error); + } + + if (parsed.Healed) + { + b.Trace(seq => new ProtocolRepaired(seq, "surplus-paren", segment.Text)); + } + + var outcome = b.ProcessRecognizedContent(context, prose, new HedgeItem(parsed.Statement!, segment.Text)); + prose = ""; + + if (outcome is not null) + { + // A tool call started, or a failure backtracked — the segment's fate is decided. + // In either case the raw text (up to the cut) becomes part of the model trace. + return outcome; + } + } + + if (evt.StoppedAtHedge) + { + // The hedge executed inline (binding / display / guard): the engine appends the + // closing bracket and resumes generation — the papers' interception protocol. + b.AppendPrefill(evt.Text + "]"); + b.IssueLlmRequest(context, newChoicePoint: true); + return b.Freeze(); + } + + // Natural stop: trailing prose, no hedge → the derivation is complete. + var trailing = b.ProcessRecognizedContent(context, prose, hedge: null); + if (trailing is not null) + { + return trailing; + } + + var finish = b.ProcessRecognizerFinish(context); + if (finish is not null) + { + return finish; + } + + b.AppendPrefill(evt.Text); + + return FinalizeDerivation(b, state, context); + } + + /// + /// The completion gate shared by both modes: essay check, phantom-variable guard, the + /// declared-outputs completion contract, and post-conditions — then the answer. + /// + private static StepResult FinalizeDerivation(Builder b, DerivationState state, StepContext context) + { + // An answer with ZERO executed hedges is an essay, not a derivation (observed live: + // the model narrates what it WOULD do). Act, don't describe. + if (!b.HasExecutedAnything) + { + return b.Backtrack(context, + "the response contained no executable hedges — nothing was computed", + "act instead of describing: call the tools and bind values inside ⟨ ... ⟩ hedges, one per step"); + } + + // An answer claiming results in variables that were never computed is a hallucination + // (observed live: "the new paths are stored in @convertedDocs" with no such binding). + if (b.FindUnboundProseMention() is { } phantom) + { + return b.Backtrack(context, + $"the answer refers to '@{phantom}' but that variable was never computed", + $"compute '@{phantom}' with a real ⟨ ... ⟩ hedge before mentioning it — results only exist when a hedge produced them"); + } + + // Declared outputs are a completion CONTRACT: finishing without computing one is an + // incomplete answer, not a style choice (observed live: the model narrated success in + // prose while the required variable was never bound — and the quotation-aware phantom + // guard rightly forgave the narration, so nothing else would have caught it). + if (state.Question is not null && + state.Question.ExpectedOutputs.FirstOrDefault(name => !b.HasTopLevelBinding(name)) is { } missing) + { + // Observed live with checklists: the model DISPLAYS the right value in a branch but + // never binds it — the binding belongs inside every branch (cf. the ticket few-shot). + var hint = b.HasConditional + ? $"every '- If …, then …' and '- Otherwise …' branch must bind it: end the branch with ⟨@{missing} is …⟩ or ⟨@{missing} = …⟩" + : $"bind '@{missing}' before finishing — a tool or rule call with '@{missing}' as the final output argument, or a calculation '@{missing} is …'"; + + return b.Backtrack(context, $"the question requires '@{missing}' but it was never computed", hint); + } + + // Post-conditions check the COMPLETED derivation; a violation backtracks (the model's + // plan produced values that break the contract — a different derivation might not). + if (state.Question is not null) + { + foreach (var clause in state.Question.PostClauses) + { + var verdict = b.CheckContract(context, clause, isPre: false); + + if (verdict is not null) + { + return verdict; + } + } + } + + return b.Complete(); + } + + // ================================================================ Mode B: whole-program synthesis + + /// + /// Mode B: the entire program arrives as ONE structured completion in the papers' + /// {comment|expression}[] interchange form. Import re-parses every expression through the + /// same hedge grammar, the walk feeds (prose, hedge) pairs through the same literate + /// recognizer — so conditionals, queries, tools, rules, teachings, and contracts behave + /// exactly as in Mode A. Only the backtracking is coarse: ANY failure regenerates the whole + /// program with the failure carried as feedback. This is the conformance floor for models + /// that cannot hold the hedge-by-hedge interception protocol. + /// + private static StepResult OnWholeProgram(Builder b, DerivationState state, LlmCompleted evt, StepContext context) + { + var import = PaperShape.Import(ExtractProgramJson(evt.Text)); + + if (import.Program is null) + { + // Truncation is not malformation: the model already returned valid JSON and was cut + // at the token cap — telling it "return valid JSON" teaches the wrong lesson while + // the next attempt runs into the same ceiling (the cap grows with the attempt + // number in BuildWholeProgram; the teach asks for brevity as well). + if (evt.Truncated) + { + return b.Backtrack(context, + "the program was cut off at the generation limit before it ended", + "write a SHORTER program: fewer and terser comments, no repetition — only the steps the answer needs"); + } + + return b.Backtrack(context, + $"the program was rejected: {import.Error}", + "return ONLY the JSON object {\"program\": [ … ]} whose items are {\"comment\": \"…\"} or {\"expression\": \"…\"} — the COMPLETE program as valid JSON, nothing else"); + } + + foreach (var warning in import.Warnings) + { + b.Trace(seq => new ProtocolRepaired(seq, "import-healed", warning)); + } + + b.StartModeBWalk(import.Program.Items); + + var walk = b.ContinueModeBWalk(context); + return walk ?? FinalizeDerivation(b, state, context); + } + + /// Unwraps the schema root object and stray markdown fences down to the bare array. + private static string ExtractProgramJson(string text) + { + var trimmed = text.Trim(); + + if (trimmed.StartsWith("```", StringComparison.Ordinal)) + { + var firstLineEnd = trimmed.IndexOf('\n'); + var lastFence = trimmed.LastIndexOf("```", StringComparison.Ordinal); + + if (firstLineEnd >= 0 && lastFence > firstLineEnd) + { + trimmed = trimmed[(firstLineEnd + 1)..lastFence].Trim(); + } + } + + if (trimmed.StartsWith('{')) + { + try + { + using var doc = System.Text.Json.JsonDocument.Parse(trimmed); + + if (doc.RootElement.TryGetProperty("program", out var program)) + { + return program.GetRawText(); + } + } + catch (System.Text.Json.JsonException) + { + // Fall through — Import produces the teaching error. + } + } + + return trimmed; + } + + // ================================================================ tools + + private static StepResult OnToolSucceeded(DerivationState state, ToolSucceeded evt, StepContext context) + { + var pending = state.Pending!; + var index = pending.RequestIds.IndexOf(evt.RequestId); + + if (pending.Results[index] is not null) + { + return new StepResult(state, []); // duplicate delivery + } + + var b = new Builder(state); + + if (evt.ResultsJson.Length == 0) + { + return b.Backtrack(context, $"'{pending.Signature.Name}' returned no results", null); + } + + // Tools are relations (0..n results); fixed-mode execution takes the first alternative. + var updated = pending with { Results = pending.Results.SetItem(index, evt.ResultsJson[0]) }; + b.Pending = updated; + + if (!updated.AllCollected) + { + // Keep PendingRequestIds precise: only the still-outstanding invocations of the + // lifted batch. Recovery re-issues exactly these (a kill between partial results + // must not orphan the rest of the batch — observed live as a wedged resume). + b.SetPendingRequestIds([.. updated.RequestIds.Where((_, i) => updated.Results[i] is null)]); + return b.Freeze(); + } + + var env = b.Env(); + var outcome = Evaluator.BindToolResults( + updated.Signature, + updated.OutArgs, + [.. updated.Results.Select(r => r!)], + env); + + if (outcome is EvalFailure failure) + { + return b.Backtrack(context, failure.Message, failure.Hint); + } + + foreach (var binding in ((Bound)outcome).Bindings) + { + b.Bind(binding, source: updated.Signature.Name); + } + + b.Pending = null; + + // A tool that completed INSIDE a rule resumes the rule interpreter, not the LLM. + if (b.HasFrames) + { + var resumed = b.RunFrames(context); + + if (resumed is not null) + { + return resumed; + } + } + + // Mode B: the program is already fully known — resume the imported walk instead of + // asking the model for the next segment. + if (b.IsModeB) + { + var walk = b.ContinueModeBWalk(context); + return walk ?? FinalizeDerivation(b, state, context); + } + + // Close the bracket the model left open and resume generation. + b.AppendPrefill("]"); + b.IssueLlmRequest(context, newChoicePoint: true); + + return b.Freeze(); + } + + private static StepResult OnToolFailed(DerivationState state, ToolFailed evt, StepContext context) + { + var b = new Builder(state); + return b.Backtrack(context, $"'{state.Pending!.Signature.Name}' failed: {evt.Error}", "try different arguments or a different tool"); + } + + // ================================================================ builder + + /// Mutable working copy of the state during one step; frozen into the result. + private sealed class Builder + { + private readonly string _conversationId; + private long _seq; + private Phase _phase; + private ImmutableArray _pendingRequestIds; + private readonly QuestionEnvelope? _question; + private ImmutableArray.Builder _program; + private ImmutableDictionary _sigma; + private ImmutableArray _choicePoints; + private LiterateRecognizer _recognizer; + private string _prefill; + private ImmutableArray _engineNotes; + private int _llmRequestCount; + private ImmutableArray _frames; + private ImmutableArray _modeBQueue; + private int _modeBPc; + private readonly ImmutableArray.Builder _effects = ImmutableArray.CreateBuilder(); + + public RetryBudget Budget { get; set; } + + public PendingCall? Pending { get; set; } + + public Builder(DerivationState state) + { + _conversationId = state.ConversationId; + _seq = state.NextSeq; + _phase = state.Phase; + _pendingRequestIds = state.PendingRequestIds; + _question = state.Question; + _program = state.ProgramSoFar.ToBuilder(); + _sigma = state.Sigma; + _choicePoints = state.ChoicePoints; + Budget = state.Budget; + _recognizer = new LiterateRecognizer(state.Recognizer); + _prefill = state.AssistantPrefill; + _engineNotes = state.EngineNotes; + Pending = state.Pending; + _llmRequestCount = state.LlmRequestCount; + _frames = state.Frames; + _modeBQueue = state.ModeBQueue.IsDefault ? [] : state.ModeBQueue; + _modeBPc = state.ModeBPc; + } + + public bool HasFrames => _frames.Length > 0; + + public bool IsModeB => _question?.ModeB == true; + + /// True when the derivation ever executed a hedge or block (vs pure prose). + public bool HasExecutedAnything => _program.Any(i => i is HedgeItem or ConditionalBlock or ComprehensionBlock); + + /// First @variable mentioned in answer prose that σ never bound, if any. + public string? FindUnboundProseMention() + { + // Variables the ENGINE itself introduced in steering (teachings quote shapes like + // ⟨TOOL(@in, @newVar)⟩): a model repeating them in prose is quoting the teacher, + // not claiming a computed value. Vetoing quotations threw away three correct + // answers live ('@in' parroted from an affordance example). + var quoted = new HashSet(StringComparer.Ordinal); + + foreach (var item in _program) + { + if (item is Comment { Aside: true } aside) + { + foreach (System.Text.RegularExpressions.Match match in + System.Text.RegularExpressions.Regex.Matches(aside.Text, "@([A-Za-z_][A-Za-z0-9_]*)")) + { + quoted.Add(match.Groups[1].Value); + } + } + } + + foreach (var item in _program) + { + if (item is not Comment { Aside: false } comment) + { + continue; + } + + foreach (System.Text.RegularExpressions.Match match in + System.Text.RegularExpressions.Regex.Matches(comment.Text, "@([A-Za-z_][A-Za-z0-9_]*)")) + { + var name = match.Groups[1].Value; + + if (!_sigma.ContainsKey(name) && !quoted.Contains(name)) + { + return name; + } + } + } + + return null; + } + + public EvalEnv Env() + { + // A rule body is a Tennent abstraction: it sees ONLY its own frame's locals + // (in-params seeded at entry, plus its body bindings) — never the caller's σ. + // Layering the caller underneath was dynamic scoping: a body-local @lat collided + // with a caller @lat bound by an earlier direct tool call (observed live), and + // since rule execution is deterministic, no retry could ever succeed. + return _frames.Length > 0 + ? EvalEnv.FromSigma(_frames[^1].Locals) + : EvalEnv.FromSigma(_sigma); + } + + public void Trace(Func make) => _effects.Add(new EmitTrace(make(_seq++).ToJson())); + + public void Bind(Binding binding, string source) + { + if (_frames.Length > 0) + { + var top = _frames[^1]; + _frames = _frames.SetItem(_frames.Length - 1, + top with { Locals = top.Locals.SetItem(binding.Var, binding.Json) }); + } + else + { + _sigma = _sigma.SetItem(binding.Var, binding.Json); + } + + Trace(seq => new VariableBound(seq, binding.Var, binding.Json, source)); + } + + public bool HasTopLevelBinding(string name) => _sigma.ContainsKey(name); + + public bool HasConditional => _program.Any(item => item is ConditionalBlock); + + /// Every declared output is bound — the answer exists, whatever else happens. + public bool AnswerRequirementsMet => + _question is not null && + !_question.ExpectedOutputs.IsEmpty && + _question.ExpectedOutputs.All(_sigma.ContainsKey); + + public void AppendPrefill(string text) => _prefill += text; + + public void SetPendingRequestIds(ImmutableArray requestIds) => _pendingRequestIds = requestIds; + + private string NextRequestId() => $"{_conversationId}/{_seq++}"; + + // ---------------------------------------------------------- LLM issuance + + public void IssueLlmRequest(StepContext context, bool newChoicePoint) + { + if (IsModeB) + { + IssueWholeProgramRequest(context); + return; + } + + // The budget must bind on the SUCCESS paths too (hedge-cut resume, tool-completion + // resume, continuation retry) — enforcing it only in the backtracking machinery left + // a never-failing narration loop unbounded (review finding). + if (_llmRequestCount >= Budget.MaxLlmRequests) + { + FailInPlace("LLM request budget exhausted"); + return; + } + + _llmRequestCount++; + + if (newChoicePoint) + { + _choicePoints = _choicePoints.Add(new ChoicePoint( + _program.Count, _sigma, _prefill, _recognizer.State, Attempt: 0)); + } + + var attempt = _choicePoints.Length > 0 ? _choicePoints[^1].Attempt : 0; + var requestId = NextRequestId(); + + var prompt = PromptAssembler.Build( + _question!, context, _prefill, _engineNotes, attempt, _conversationId, requestId); + + _effects.Add(new RequestLlm(requestId, prompt.ToJson())); + _phase = new Synthesizing(); + _pendingRequestIds = [requestId]; + } + + /// Mode B: one request returns the WHOLE program — no choice points, no prefill. + private void IssueWholeProgramRequest(StepContext context) + { + if (_llmRequestCount >= Budget.MaxLlmRequests) + { + FailInPlace("LLM request budget exhausted"); + return; + } + + _llmRequestCount++; + + var requestId = NextRequestId(); + var prompt = PromptAssembler.BuildWholeProgram( + _question!, context, _engineNotes, attempt: _llmRequestCount - 1, _conversationId, requestId); + + _effects.Add(new RequestLlm(requestId, prompt.ToJson())); + _phase = new Synthesizing(); + _pendingRequestIds = [requestId]; + } + + // ---------------------------------------------------------- Mode B walk + + public void StartModeBWalk(ImmutableArray items) + { + _modeBQueue = items; + _modeBPc = 0; + } + + /// + /// Executes imported items until a tool suspends the walk, a failure regenerates, or + /// the program ends (null = walk finished; the caller runs the completion gate). + /// Consecutive comments join with newlines — the interchange form is line-granular and + /// the recognizer keys bullets ('- If …', '- Retain only …') on line starts, so the + /// joined prose reconstructs blocks exactly as Mode A streaming would. + /// + public StepResult? ContinueModeBWalk(StepContext context) + { + while (_modeBPc < _modeBQueue.Length) + { + var prose = ""; + + while (_modeBPc < _modeBQueue.Length && _modeBQueue[_modeBPc] is Comment comment) + { + prose += (prose.Length > 0 ? "\n" : "") + comment.Text; + _modeBPc++; + } + + HedgeItem? hedge = null; + + if (_modeBPc < _modeBQueue.Length && _modeBQueue[_modeBPc] is HedgeItem next) + { + hedge = next; + _modeBPc++; + } + + var outcome = ProcessRecognizedContent(context, prose, hedge); + + if (outcome is not null) + { + return outcome; + } + } + + return ProcessRecognizerFinish(context); + } + + // ---------------------------------------------------------- content processing + + /// + /// Routes (prose, hedge) through the literate recognizer and executes what it emits. + /// Returns a final StepResult when the step's fate is decided (tool call issued, + /// backtracked, failed); null to continue processing. + /// + public StepResult? ProcessRecognizedContent(StepContext context, string prose, HedgeItem? hedge) + { + foreach (var evt in _recognizer.Advance(prose, hedge)) + { + var result = ProcessRecognizerEvent(context, evt); + if (result is not null) + { + return result; + } + } + + return null; + } + + public StepResult? ProcessRecognizerFinish(StepContext context) + { + foreach (var evt in _recognizer.Finish()) + { + var result = ProcessRecognizerEvent(context, evt); + if (result is not null) + { + return result; + } + } + + return null; + } + + private StepResult? ProcessRecognizerEvent(StepContext context, RecognizerEvent evt) + { + switch (evt) + { + case ProseEmitted prose: + // Models sometimes echo the engine-appended ']' at the start of their next + // prose run — protocol debris, never content (observed live as answer text + // like "using ]. The engine…"). The prefill keeps the model's own tokens; + // only the assembled-answer view is cleaned. + _program.Add(new Comment( + System.Text.RegularExpressions.Regex.Replace(prose.Text, @"^\s*\]\.?\s*", " "))); + return null; + + case ExecuteHedge hedge: + return ExecuteHedgeNow(context, hedge.Hedge, repaired: false); + + case ConditionalCompleted conditional: + return ExecuteConditional(context, conditional.Block); + + case ComprehensionCompleted comprehension: + return ExecuteComprehension(context, comprehension.Draft); + + default: + return null; + } + } + + private StepResult? ExecuteHedgeNow(StepContext context, HedgeItem hedge, bool repaired) + { + // A σ-mutating hedge inside an "If …" SENTENCE (not a checklist bullet) executes + // unconditionally — the engine cannot evaluate a prose condition. Observed live: + // "If Sam has enough money, he … [$@remaining is @price - @cash]. If not, he keeps + // [$@remaining is @cash]" bound the then-arm's value with the guard FALSE, and the + // poisoned σ made every honest checklist retry fail its assertion to depth death. + if (!repaired && hedge.Statement is not DisplayStmt && InConditionalSentence()) + { + return Backtrack(context, + $"[{hedge.ConcreteText}] is guarded by an 'If …' sentence the engine cannot evaluate", + "prose never guards a computation — write the decision as checklist bullets " + + "'- If ⟨condition⟩, then …' and '- Otherwise, …', binding each branch's value inside its own bullet"); + } + + var outcome = Evaluator.Evaluate(hedge.Statement, Env(), context.BuildCatalog()); + + switch (outcome) + { + case Bound bound: + _program.Add(hedge); + foreach (var binding in bound.Bindings) + { + Bind(binding, SourceOf(hedge.Statement)); + } + + return null; + + case DisplayValue display: + _program.Add(hedge); + Trace(seq => new DisplayShown(seq, hedge.ConcreteText, display.Formatted)); + return null; + + case GuardResult guard: + _program.Add(hedge); + Trace(seq => new GuardEvaluated(seq, -1, hedge.ConcreteText, guard.Value)); + return guard.Value + ? null + : Backtrack(context, $"the assertion [{hedge.ConcreteText}] is false", + "both sides are already bound to different values — earlier prose computed one of them; either drop the hedge (say it in plain prose) or bind a FRESH variable if you meant a new value"); + + case NeedTool need: + return StartToolCall(context, hedge, need); + + case NeedRule need: + return InvokeRule(context, hedge, need); + + case EvalFailure { Code: EvalFailureCodes.LiteralInOutPosition } when !repaired: + return AutoRepairOutArgs(context, hedge); + + case EvalFailure { Code: EvalFailureCodes.Rebind } when !repaired && hedge.Statement is PredicateCall: + return HandleRepeatedCall(context, hedge); + + // [@x is TOOL(@a, @b)] where the call was already complete: the is-rewrite + // appended @x as an extra arg. Drop it and run the call as written. + // An invented extraction predicate over a bound object and a pattern IS a + // pattern match — the intent is unambiguous (observed live: [EXTRACT(@stockData, + // { "close": @price })] dead-ending as unknown). Run the match the model meant, + // opened, since extraction never implies an exact key set. + case EvalFailure { Code: EvalFailureCodes.UnknownPredicate } when !repaired && + hedge.Statement is PredicateCall { Args: [VarTerm source, ObjectPatternTerm extractPattern] } && + Env().IsBound(source.Name): + { + Trace(seq => new ProtocolRepaired(seq, "extract-as-match", hedge.ConcreteText)); + var match = new BindStmt(new VarTerm(source.Name), extractPattern with { IsOpen = true }); + + return ExecuteHedgeNow(context, + new HedgeItem(match, ConcreteRenderer.RenderStatement(match, RenderMode.Formulas)), + repaired: true); + } + + case EvalFailure { Code: EvalFailureCodes.Arity } when !repaired && + hedge.Statement is PredicateCall { Args: [.., var beforeExtra, VarTerm extra] } overfull && + context.BuildCatalog().TryGet(overfull.Name, out var sig) && + overfull.Args.Length == sig.Params.Length + 1 && + // A BOUND extra is still trimmable when a pattern carries the real + // out-binding: [@p is STOCK("MSFT", { "close": @close })] with @p already + // bound (observed live — the guard used to refuse and the roll died). + (!Env().IsBound(extra.Name) || beforeExtra is ObjectPatternTerm): + { + var trimmed = overfull with { Args = overfull.Args.RemoveAt(overfull.Args.Length - 1) }; + Trace(seq => new ProtocolRepaired(seq, "extra-out-arg", hedge.ConcreteText)); + return ExecuteHedgeNow(context, + new HedgeItem(trimmed, ConcreteRenderer.RenderStatement(trimmed, RenderMode.Formulas)), + repaired: true); + } + + // Once every declared output is bound the answer EXISTS — a failing trailing + // hedge is post-answer noise to skip, never a reason to unwind the finished + // state (observed live twice in one roll: @btcLeft bound, branch taken, then a + // junk MATH re-call backtracked the completed derivation into budget death). + case EvalFailure when AnswerRequirementsMet: + Trace(seq => new ProtocolRepaired(seq, "post-answer-noise", hedge.ConcreteText)); + return null; + + case EvalFailure { Code: EvalFailureCodes.Unbound } failure: + return Backtrack(context, $"[{hedge.ConcreteText}] failed: {failure.Message}", UnboundHint()); + + case EvalFailure failure: + return Backtrack(context, $"[{hedge.ConcreteText}] failed: {failure.Message}", failure.Hint); + + default: + return Backtrack(context, $"unexpected evaluation outcome {outcome.GetType().Name}", null); + } + } + + /// + /// The repeat-after-success pathology (observed live): after a successful call and resume, + /// small models re-narrate the same call with the already-bound output variable. If every + /// output is already bound, the narration is redundant — the call HAPPENED; close the + /// bracket and move on (never re-execute: the tool may not be idempotent). If only some + /// outputs are rebound, substitute fresh variables for those and execute. + /// + private StepResult? HandleRepeatedCall(StepContext context, HedgeItem hedge) + { + var call = (PredicateCall)hedge.Statement; + + if (!context.BuildCatalog().TryGet(call.Name, out var signature) || + !Evaluator.TryNormalizeNamedArgs(call, signature, out call) || + call.Args.Length != signature.Params.Length) + { + return Backtrack(context, $"[{hedge.ConcreteText}] reuses a bound variable", "write a fresh variable name for every tool output"); + } + + var env = Env(); + var outArgs = signature.Params + .Select((p, i) => (Param: p, Arg: call.Args[i])) + .Where(x => x.Param.Mode == ParamMode.Out) + .ToList(); + + var allOutsBound = outArgs.All(x => x.Arg is VarTerm v && env.IsBound(v.Name)); + + // Narration ⇔ the same call (by IN-argument VALUES) already ran. Different inputs + // with a reused output variable is a genuinely new call — repair, don't skip. + var currentKey = InArgsKey(call, signature, env); + var isNarration = allOutsBound && currentKey is not null && _program + .OfType() + .Select(h => h.Statement) + .OfType() + .Any(prior => + string.Equals(prior.Name, call.Name, StringComparison.OrdinalIgnoreCase) && + InArgsKey(prior, signature, env) == currentKey); + + if (isNarration) + { + Trace(seq => new ProtocolRepaired(seq, "duplicate-call", hedge.ConcreteText)); + + // The prose leading INTO redundant re-narration is itself re-narration — keep + // it out of the assembled answer (observed live: "First, list the files…" ×3). + if (_program.Count > 0 && _program[^1] is Comment { Aside: false } lead) + { + _program[_program.Count - 1] = lead with { Aside = true }; + } + + return null; // redundant narration — resume generation, no re-execution + } + + var args = call.Args.ToBuilder(); + + for (var i = 0; i < signature.Params.Length; i++) + { + if (signature.Params[i].Mode == ParamMode.Out && args[i] is VarTerm v && env.IsBound(v.Name)) + { + args[i] = new VarTerm($"auto{_seq++}"); + } + } + + var repairedCall = call with { Args = args.ToImmutable() }; + var repairedHedge = new HedgeItem(repairedCall, ConcreteRenderer.RenderStatement(repairedCall, RenderMode.Formulas)); + + Trace(seq => new ProtocolRepaired(seq, "rebound-output", hedge.ConcreteText)); + + return ExecuteHedgeNow(context, repairedHedge, repaired: true); + } + + /// + /// Context-aware teaching for unbound-variable failures: what IS bound, and — when a + /// collection is bound — the comprehension form to process it with. (Observed live: + /// without this the model free-forms counting logic and dies referencing phantom vars.) + /// + private string UnboundHint() + { + // No call-shaped placeholders here: models INVOKE them literally (observed live: + // 'unknown predicate TOOL' burned whole budgets). Wordy descriptions are imitated + // correctly; example calls are imitated verbatim. + var bound = _sigma.Keys.OrderBy(k => k, StringComparer.Ordinal).Take(8).ToList(); + var hint = bound.Count == 0 + ? "no variables are bound yet; compute values first with ⟨@newVar is …⟩ or by calling one of the TOOLS" + : $"bound so far: {string.Join(", ", bound.Select(n => "@" + n))}; a new variable is bound ONLY by a computation that outputs it — call one of the TOOLS with the new variable as its last argument, or compute ⟨@newVar is …⟩"; + + var collection = bound.FirstOrDefault(n => _sigma[n].TrimStart().StartsWith('[')); + + if (collection is not null) + { + hint += $". To count or filter @{collection}, write exactly: Consider each ⟨@item = {{ … \"field\": @field … }}⟩ from ⟨@{collection}⟩: then bullets '- Retain only … ⟨condition⟩.' and '- Subsequently, increment ⟨@total⟩ by one …'"; + } + + return hint; + } + + /// In-argument identity by VALUE (bound variables substituted), for duplicate detection. + private static string? InArgsKey(PredicateCall call, PredicateSignature signature, EvalEnv env) + { + if (call.Args.Length != signature.Params.Length) + { + return null; + } + + var parts = new List(); + + for (var i = 0; i < signature.Params.Length; i++) + { + if (signature.Params[i].Mode != ParamMode.In) + { + continue; + } + + if (!Evaluator.IsGround(call.Args[i], env)) + { + return null; + } + + parts.Add(ConcreteRenderer.RenderTerm(call.Args[i], RenderMode.Values, env)); + } + + // A visible escape, deliberately: this used to be a literal U+0001 BETWEEN the quotes, // which rendered as Join("", parts) to every reader and got code-reviewed as a // key-collision bug. The separator is load-bearing: without one, MATH(12, 3) and // MATH(1, 23) would share an identity and a genuinely new call would be skipped as // duplicate narration, silently keeping a stale output. + return string.Join("\u001F", parts); + } + + /// + /// Paper 1's discard-and-replace rule: the model hallucinated a value where an output + /// belongs; substitute a fresh variable and proceed — the engine's value wins anyway. + /// + private StepResult? AutoRepairOutArgs(StepContext context, HedgeItem hedge) + { + if (hedge.Statement is not PredicateCall call || + !context.BuildCatalog().TryGet(call.Name, out var signature) || + !Evaluator.TryNormalizeNamedArgs(call, signature, out call)) + { + return Backtrack(context, $"[{hedge.ConcreteText}] misuses an output position", null); + } + + var args = call.Args.ToBuilder(); + + for (var i = 0; i < signature.Params.Length && i < args.Count; i++) + { + if (signature.Params[i].Mode == ParamMode.Out && args[i] is not VarTerm and not ObjectPatternTerm and not ArrayPatternTerm) + { + args[i] = new VarTerm($"auto{_seq++}"); + } + } + + var repairedCall = call with { Args = args.ToImmutable() }; + var repairedHedge = new HedgeItem(repairedCall, ConcreteRenderer.RenderStatement(repairedCall, RenderMode.Formulas)); + + Trace(seq => new ProtocolRepaired(seq, "literal-in-out-position", hedge.ConcreteText)); + + return ExecuteHedgeNow(context, repairedHedge, repaired: true); + } + + // ---------------------------------------------------------- stored rules (frames) + + /// Activates a stored rule: push a frame and interpret its body — zero LLM calls inside. + private StepResult? InvokeRule(StepContext context, HedgeItem? hedge, NeedRule need) + { + // Last-wins, matching SignatureCatalog.BuildCatalog and the host's "seeds come last + // so they shadow" contract — FirstOrDefault executed a stale stored body against the + // shadowing seed's signature (review finding). + var rule = context.Rules.LastOrDefault(r => + string.Equals(r.Signature.Name, need.Signature.Name, StringComparison.OrdinalIgnoreCase)); + + if (rule is null) + { + return Backtrack(context, $"rule '{need.Signature.Name}' has no stored definition", null); + } + + if (_frames.Length >= 8) + { + return Backtrack(context, "rule recursion is too deep", null); + } + + if (hedge is not null) + { + _program.Add(hedge); + } + + Trace(seq => new RuleInvoked(seq, rule.Signature.Name)); + + _frames = _frames.Add(new RuleFrame( + rule.Signature.Name, + rule.Body.Items, + Pc: 0, + Locals: need.InBindings.ToImmutableDictionary(b => b.Var, b => b.Json), + need.OutArgs, + [.. rule.Signature.Params.Where(p => p.Mode == ParamMode.Out).Select(p => p.Name)])); + + return RunFrames(context); + } + + /// + /// Interprets active rule frames until they all complete (null) or the machine suspends + /// on a tool / backtracks / fails (a StepResult). Resumable: Pc and Locals are part of + /// the checkpointed state, so a kill mid-rule recovers mid-rule. + /// + public StepResult? RunFrames(StepContext context) + { + while (_frames.Length > 0) + { + var frame = _frames[^1]; + + if (frame.Pc >= frame.Body.Length) + { + var popped = PopFrame(context, frame); + + if (popped is not null) + { + return popped; + } + + continue; + } + + var item = frame.Body[frame.Pc]; + _frames = _frames.SetItem(_frames.Length - 1, frame with { Pc = frame.Pc + 1 }); + + switch (item) + { + case Comment: + continue; + + case HedgeItem hedgeItem: + { + var result = ExecuteRuleStatement(context, frame.RuleName, hedgeItem); + + if (result is not null) + { + return result; + } + + continue; + } + + case ConditionalBlock conditional: + { + var result = ExecuteConditional(context, conditional, addToProgram: false); + + if (result is not null) + { + return result; + } + + continue; + } + + case ComprehensionBlock comprehension: + { + var outcome = QueryPipeline.Execute(comprehension, Env(), out _, out _); + + if (outcome is EvalFailure failure) + { + return Backtrack(context, $"inside rule '{frame.RuleName}': the query failed: {failure.Message}", failure.Hint); + } + + foreach (var binding in ((Bound)outcome).Bindings) + { + Bind(binding, source: "query"); + } + + continue; + } + } + } + + return null; + } + + private StepResult? ExecuteRuleStatement(StepContext context, string ruleName, HedgeItem hedge) + { + var outcome = Evaluator.Evaluate(hedge.Statement, Env(), context.BuildCatalog()); + + switch (outcome) + { + case Bound bound: + foreach (var binding in bound.Bindings) + { + Bind(binding, SourceOf(hedge.Statement)); + } + + return null; + + case DisplayValue display: + Trace(seq => new DisplayShown(seq, hedge.ConcreteText, display.Formatted)); + return null; + + case GuardResult { Value: true }: + return null; + + case GuardResult: + return Backtrack(context, $"inside rule '{ruleName}': the assertion [{hedge.ConcreteText}] is false", null); + + case NeedTool need: + return StartToolCall(context, hedge: null, need); + + case NeedRule nested: + return InvokeRule(context, hedge: null, nested); + + case EvalFailure failure: + return Backtrack(context, $"inside rule '{ruleName}': [{hedge.ConcreteText}] failed: {failure.Message}", failure.Hint); + + default: + return Backtrack(context, $"inside rule '{ruleName}': unexpected outcome {outcome.GetType().Name}", null); + } + } + + /// Copies the rule's formal outputs back to the caller's out-terms, then pops. + private StepResult? PopFrame(StepContext context, RuleFrame frame) + { + _frames = _frames.RemoveAt(_frames.Length - 1); + + var outerEnv = Env(); + + for (var i = 0; i < frame.FormalOuts.Length && i < frame.CallerOutArgs.Length; i++) + { + if (!frame.Locals.TryGetValue(frame.FormalOuts[i], out var value)) + { + return Backtrack(context, + $"rule '{frame.RuleName}' completed without binding its output '@{frame.FormalOuts[i]}'", null); + } + + switch (frame.CallerOutArgs[i]) + { + case VarTerm v: + Bind(new Binding(v.Name, value), source: frame.RuleName); + break; + + default: + { + var match = PatternMatcher.Match( + frame.CallerOutArgs[i], + System.Text.Json.Nodes.JsonNode.Parse(value), + outerEnv); + + if (!match.Success) + { + // Observed live: a destructuring pattern in a rule's output + // position when the rule returns a list. Teach the plain form. + return Backtrack(context, + $"the output pattern for rule '{frame.RuleName}' did not match: {match.Reason}", + "bind the rule's output to ONE plain fresh variable — no pattern — then display or query it"); + } + + foreach (var binding in match.Bindings) + { + Bind(binding, source: frame.RuleName); + } + + break; + } + } + } + + return null; + } + + private StepResult StartToolCall(StepContext context, HedgeItem? hedge, NeedTool need) + { + var binding = context.FindTool(need.Signature.Name); + + if (binding is null) + { + return Backtrack(context, $"tool '{need.Signature.Name}' has no engine binding", null)!; + } + + if (hedge is not null) + { + _program.Add(hedge); + } + + var requestIds = ImmutableArray.CreateBuilder(need.Plan.Count); + + for (var k = 0; k < need.Plan.Count; k++) + { + requestIds.Add(NextRequestId()); + } + + Pending = new PendingCall( + need.Signature, + binding.ToolUri, + binding.IsIdempotent, + need.OutArgs, + requestIds.ToImmutable(), + [.. Enumerable.Repeat(null, need.Plan.Count)]); + + Trace(seq => new ToolInvoked(seq, need.Signature.Name, need.Plan.InvocationArgsJson[0], need.Plan.Count)); + + for (var k = 0; k < need.Plan.Count; k++) + { + _effects.Add(new InvokeTool(Pending.RequestIds[k], binding.ToolUri, need.Plan.InvocationArgsJson[k])); + } + + _phase = new AwaitingTools(); + _pendingRequestIds = Pending.RequestIds; + + return Freeze(); + } + + /// Query comprehension: compile the recognized draft, run the LINQ pipeline, bind results. + private StepResult? ExecuteComprehension(StepContext context, ComprehensionDraft draft) + { + var (block, error) = ComprehensionCompiler.Compile(draft); + + if (block is null) + { + // The rewind erases the WHOLE block including its opener, so the steering must ask + // for the whole query back — a model told to fix one bullet re-emits just that + // bullet, which then executes as a top-level hedge against an empty row scope. + return Backtrack(context, $"could not compile the query: {error}", + "rewrite the WHOLE query from its first line — 'Consider each … from …:' — followed by bullets such as 'Retain only … ⟨condition⟩', 'Determine the ⟨average/min/max⟩ … as ⟨{\"field\": @var}⟩', 'Collect … as …', or 'Subsequently, increment ⟨@total⟩ by one'; never 'If' or 'For' bullets", + beforeBlock: true); + } + + var outcome = QueryPipeline.Execute(block, Env(), out var rowsIn, out var rowsOut); + + if (outcome is EvalFailure failure) + { + return Backtrack(context, $"the query failed: {failure.Message}", failure.Hint, beforeBlock: true); + } + + _program.Add(block); + + var bound = (Bound)outcome; + + foreach (var binding in bound.Bindings) + { + Bind(binding, source: "query"); + } + + var into = bound.Bindings.Length > 0 ? bound.Bindings[0].Var : "queryResult"; + Trace(seq => new QueryExecuted(seq, into, rowsIn, rowsOut)); + + return null; + } + + /// Checklist execution: first true branch runs; all guards trace; untaken branches cross out. + private StepResult? ExecuteConditional(StepContext context, ConditionalBlock block, bool addToProgram = true) + { + if (addToProgram) + { + _program.Add(block); + } + + var takenIndex = -1; + + for (var i = 0; i < block.Branches.Length; i++) + { + var branch = block.Branches[i]; + bool guardValue; + + if (branch.Guard is null) + { + guardValue = takenIndex < 0; // else-branch: taken iff nothing before it was + } + else + { + var outcome = Evaluator.Evaluate(branch.Guard, Env(), context.BuildCatalog()); + + switch (outcome) + { + case GuardResult g: + guardValue = g.Value; + break; + case EvalFailure failure: + // The underlying failure's teaching (e.g. object-extraction with the + // real field names) beats the generic advice — dropping it starved + // four identical guard failures of the fix (observed live). + return Backtrack(context, $"guard failed: {failure.Message}", + failure.Hint ?? "bind every value with ⟨@newVar is …⟩ BEFORE the first '- If' bullet", beforeBlock: true); + default: + return Backtrack(context, "a branch guard must be a comparison", null, beforeBlock: true); + } + + var index = i; + var text = ConcreteRenderer.RenderStatement(branch.Guard, RenderMode.Formulas); + var value = guardValue; + Trace(seq => new GuardEvaluated(seq, index, text, value)); + } + + if (guardValue && takenIndex < 0) + { + takenIndex = i; + } + } + + for (var i = 0; i < block.Branches.Length; i++) + { + if (i == takenIndex) + { + var index = i; + Trace(seq => new BranchTaken(seq, index)); + continue; + } + + var crossed = i; + var guardText = block.Branches[i].Guard is { } g2 + ? ConcreteRenderer.RenderStatement(g2, RenderMode.Formulas) + : "otherwise"; + Trace(seq => new BranchCrossedOut(seq, crossed, guardText)); + } + + if (takenIndex < 0) + { + return null; // no branch taken: legal — downstream reads of branch vars will fail → backtrack there + } + + foreach (var item in block.Branches[takenIndex].Body) + { + if (item is not HedgeItem hedge) + { + continue; + } + + var outcome = Evaluator.Evaluate(hedge.Statement, Env(), context.BuildCatalog()); + + switch (outcome) + { + case Bound bound: + foreach (var binding in bound.Bindings) + { + Bind(binding, SourceOf(hedge.Statement)); + } + + break; + + case DisplayValue display: + Trace(seq => new DisplayShown(seq, hedge.ConcreteText, display.Formatted)); + break; + + case GuardResult { Value: true }: + break; + + case GuardResult: + return Backtrack(context, $"the assertion [{hedge.ConcreteText}] is false", + "both sides are already bound to different values — earlier prose computed one of them; either drop the hedge (say it in plain prose) or bind a FRESH variable if you meant a new value"); + + case NeedTool: + return Backtrack(context, + "tool calls inside conditional branches are not supported yet", + "call the tool before the checklist and use its result in the branches"); + + case NeedRule: + return Backtrack(context, + "rule calls inside conditional branches are not supported yet", + "call the rule before the checklist and use its output in the branches"); + + // Same principle as the top-level skip: once the declared outputs are bound + // the answer exists — a failing later body hedge is noise, not a reason to + // unwind (observed live: a second, conflicting @btcLeft bind in the branch). + case EvalFailure when AnswerRequirementsMet: + Trace(seq => new ProtocolRepaired(seq, "post-answer-noise", hedge.ConcreteText)); + break; + + case EvalFailure failure: + return Backtrack(context, $"[{hedge.ConcreteText}] failed: {failure.Message}", failure.Hint); + + // A silently-fallen-through outcome was exactly how NeedRule vanished here + // (review finding) — an unhandled kind must be a visible failure, never a drop. + default: + return Backtrack(context, + $"[{hedge.ConcreteText}] cannot run inside a conditional branch", + "compute it before the checklist and use the result in the branches"); + } + } + + return null; + } + + private static string SourceOf(Statement statement) => statement switch + { + IsBinding => "is", + BindStmt => "bind", + _ => "match", + }; + + /// + /// True when the prose sentence containing the current hedge opens with a conditional word + /// ("If Sam has enough money, he … [hedge]"). The sentence is read from the program's + /// trailing comments — durable and correctly rewound by backtracking. It may flow across + /// inline DISPLAY hedges; a preceding binding hedge ends it (had that hedge shared the + /// sentence, it would have been caught itself). Checklist bullets never reach this check — + /// the recognizer routes '- If [guard]' into conditional blocks before hedges execute. + /// + private bool InConditionalSentence() + { + var text = ""; + + for (var i = _program.Count - 1; i >= 0 && _program.Count - i <= 4; i--) + { + if (_program[i] is Comment { Aside: false } comment) + { + text = comment.Text + text; + } + else if (_program[i] is not HedgeItem { Statement: DisplayStmt }) + { + break; + } + + if (text.IndexOfAny(SentenceBoundaries) >= 0) + { + break; + } + } + + var cut = text.LastIndexOfAny(SentenceBoundaries); + var sentence = cut >= 0 ? text[(cut + 1)..] : text; + + return System.Text.RegularExpressions.Regex.IsMatch( + sentence, @"^\s*(?:if|otherwise|unless|when)\b", + System.Text.RegularExpressions.RegexOptions.IgnoreCase); + } + + private static readonly char[] SentenceBoundaries = ['.', '!', '?', ':', ';', '\n']; + + // ---------------------------------------------------------- backtracking + + public StepResult Backtrack(StepContext context, string reason, string? hint, bool fromRoot = false, bool beforeBlock = false) + { + var note = Sanitize(hint is null ? reason : $"{reason} — {hint}"); + + // Consecutive-identical-failure escalation. A fresh choice point is planted after + // every executed hedge, so a model looping "narrate, then fail the same way" retries + // at attempt 1 forever and burns the whole LLM budget (observed live: display @files, + // fail on unbound @pdfs, repeat). Identical repeats consume extra attempts on the + // CURRENT point only, exhausting it fast and forcing the rewind deeper — a genuinely + // different resume context — while the note tells the model it is repeating itself. + var reasonKey = Sanitize(reason); + reasonKey = reasonKey.Length > 200 ? reasonKey[..200] : reasonKey; + + // Count matches across the whole (capped) note window, not just the tail: models also + // PING-PONG between two failure modes, and strictly-consecutive counting lets each one + // reset the other's escalation forever (observed live). + var repeats = _engineNotes.Count(n => n.StartsWith(reasonKey, StringComparison.Ordinal)); + + if (repeats > 0) + { + note = Sanitize( + $"{reason} — this exact failure has now happened {repeats + 1} times; the same approach will keep failing. " + + (hint ?? "take a genuinely different route")); + } + + // Mode B has no choice points: the tree-of-thought degenerates to regenerating the + // WHOLE program with the (escalated) failure carried as feedback. + if (IsModeB) + { + return RegenerateWholeProgram(context, reason, note, repeats); + } + + Trace(seq => new BacktrackStarted(seq, reason, _choicePoints.Length - 1, + _choicePoints.Length > 0 ? _choicePoints[^1].Attempt : 0)); + + if (fromRoot && _choicePoints.Length > 1) + { + // Whole-derivation properties (post-conditions) restart the derivation: keeping + // intermediate bindings would make every local retry fail identically. + _choicePoints = [_choicePoints[0]]; + } + + if (beforeBlock) + { + // A failed checklist/comprehension must rewind to BEFORE the block opened: + // retrying from a mid-block snapshot replays the buffered broken block before the + // model's correction can take effect (observed live — the fix got eaten). + while (_choicePoints.Length > 1 && + _choicePoints[^1].Recognizer.Context != RecognizerContext.Top) + { + _choicePoints = _choicePoints.RemoveAt(_choicePoints.Length - 1); + } + } + + var boost = repeats; + + while (_choicePoints.Length > 0) + { + var cp = _choicePoints[^1]; + + if (cp.Attempt + 1 + boost < Budget.MaxAttemptsPerChoicePoint) + { + return RetryFrom(context, cp with { Attempt = cp.Attempt + 1 + boost }, note); + } + + boost = 0; // acceleration applies to the repeating point only, not the pop cascade + + _choicePoints = _choicePoints.RemoveAt(_choicePoints.Length - 1); + Budget = Budget with { BacktrackDepthUsed = Budget.BacktrackDepthUsed + 1 }; + + if (Budget.BacktrackDepthUsed > Budget.MaxBacktrackDepth) + { + return Restart(context, reason, note) ?? Fail($"backtracking depth exhausted: {reason}"); + } + } + + return Restart(context, reason, note) ?? Fail($"no alternatives left: {reason}"); + } + + /// + /// Mode B's coarse backtracking: reset everything to the question and ask for a fresh + /// whole program, the failure note riding in ENGINE NOTES. Bounded twice: the request + /// budget, and — because a whole-program generation costs minutes on slow models — the + /// SAME failure repeating MaxAttemptsPerChoicePoint times kills the derivation early + /// instead of grinding all 32 generations against an immovable wall (review finding). + /// + private StepResult RegenerateWholeProgram(StepContext context, string reason, string note, int repeats) + { + Trace(seq => new BacktrackStarted(seq, $"regenerate: {reason}", -1, _llmRequestCount)); + + if (repeats + 1 >= Budget.MaxAttemptsPerChoicePoint) + { + return Fail($"the same failure repeated {repeats + 1} times: {reason}"); + } + + if (_llmRequestCount >= Budget.MaxLlmRequests) + { + return Fail("LLM request budget exhausted"); + } + + _program = ImmutableArray.CreateBuilder(); + _sigma = _question!.InitialBindings; + _prefill = ""; + _recognizer = new LiterateRecognizer(RecognizerState.Initial); + _frames = []; + _choicePoints = []; + _modeBQueue = []; + _modeBPc = 0; + Pending = null; + _engineNotes = Append(_engineNotes, note); + + IssueLlmRequest(context, newChoicePoint: false); + + return Freeze(); + } + + /// + /// Tree-of-thought RESTART: the search is exhausted (choice points OR depth — observed + /// live for both: early query-compile failures killed a shallow stack with most of the + /// budget unspent, and a σ poisoned early made every honest checklist retry fail its + /// assertion until the depth ran out) but most of the request budget remains. Begin a + /// fresh derivation from the question — the accumulated engine notes carry the teachings + /// forward, so the new attempt starts smarter than the first did. Null when gated. + /// + private StepResult? Restart(StepContext context, string reason, string note) + { + if (_question is null || _llmRequestCount >= Budget.MaxLlmRequests * 3 / 4) + { + return null; + } + + Trace(seq => new BacktrackStarted(seq, $"fresh start: {reason}", -1, 0)); + + _program = ImmutableArray.CreateBuilder(); + _sigma = _question.InitialBindings; + _prefill = ""; + _recognizer = new LiterateRecognizer(RecognizerState.Initial); + _frames = []; + _choicePoints = []; + Pending = null; + Budget = Budget with { BacktrackDepthUsed = 0 }; + _engineNotes = Append(_engineNotes, note); + + IssueLlmRequest(context, newChoicePoint: true); + + return Freeze(); + } + + private StepResult RetryFrom(StepContext context, ChoicePoint cp, string note) + { + if (_llmRequestCount >= Budget.MaxLlmRequests) + { + return Fail("LLM request budget exhausted"); + } + + var discarded = _prefill.Length > cp.AssistantPrefill.Length ? _prefill[cp.AssistantPrefill.Length..] : ""; + if (discarded.Length > 0) + { + Trace(seq => new BranchAbandoned(seq, discarded)); + } + + // Rewind to the snapshot. + var kept = ImmutableArray.CreateBuilder(cp.ProgramLength); + for (var i = 0; i < cp.ProgramLength; i++) + { + kept.Add(_program[i]); + } + + _program = kept; + _sigma = cp.Sigma; + _recognizer = new LiterateRecognizer(cp.Recognizer); + _choicePoints = _choicePoints.SetItem(_choicePoints.Length - 1, cp); + _engineNotes = Append(_engineNotes, note); + Pending = null; + _frames = []; // choice points are only taken at top level; abandon any rule activation + + // Inline steering, in the model's own voice (recency matters for small models). It + // becomes part of the trace/program like any other prose. + var steering = $" That didn't work ({note}). Let me try a different approach. "; + _prefill = cp.AssistantPrefill + steering; + _program.Add(new Comment(steering, Aside: true)); + + IssueLlmRequest(context, newChoicePoint: false); + + return Freeze(); + } + + /// + /// Engine notes carry names, codes, and schema keys — never σ values — and NEVER square + /// brackets: bracketed hedges inside a steering sentence get parroted by small models and + /// then executed by the scanner, creating a failure loop (observed live with granite3.3). + /// + private static string Sanitize(string note) + { + var bracketFree = note + .Replace('[', '⟨') + .Replace(']', '⟩'); + + return bracketFree.Length > 400 ? bracketFree[..400] + "…" : bracketFree; + } + + /// Keeps the most recent notes only — old notes stop being useful and bloat the prompt. + private static ImmutableArray Append(ImmutableArray notes, string note) + { + var appended = notes.Add(note); + return appended.Length > 4 ? appended.RemoveRange(0, appended.Length - 4) : appended; + } + + private StepResult Fail(string reason) + { + FailInPlace(reason); + return Freeze(); + } + + /// Marks the builder failed without freezing — for guards inside void paths. + private void FailInPlace(string reason) + { + _phase = new FailedPhase(reason); + _pendingRequestIds = []; + Pending = null; + _frames = []; + Trace(seq => new DerivationFailed(seq, reason)); + _effects.Add(new FailedEffect(reason)); + } + + // ---------------------------------------------------------- contracts + + /// + /// Evaluates one contract clause. Returns null when it passes; a final result otherwise + /// (pre-violations fail the derivation, post-violations backtrack — the model's plan was + /// wrong, so an alternative derivation may still satisfy the contract). + /// + public StepResult? CheckContract(StepContext context, ContractClauseText clause, bool isPre) + { + var parsed = HedgeParser.Parse(clause.Condition); + + if (!parsed.Success) + { + return Fail($"cannot parse the {(isPre ? "pre" : "post")}-condition [{clause.Condition}]: {parsed.Error}"); + } + + var outcome = Evaluator.Evaluate(parsed.Statement!, Env(), context.BuildCatalog()); + var passed = outcome is GuardResult { Value: true }; + + Trace(seq => new ContractChecked(seq, isPre, clause.Condition, passed)); + + if (passed) + { + return null; + } + + var explanation = outcome is EvalFailure failure + ? $"[{clause.Condition}] could not be evaluated: {failure.Message}" + : $"[{clause.Condition}] is violated — {clause.Rationale}"; + + return isPre + ? Fail($"pre-condition failed: {explanation}") + : Backtrack(context, $"post-condition failed: {explanation}", + "derive the answer a different way so the contract holds", fromRoot: true); + } + + // ---------------------------------------------------------- completion + + public StepResult Complete() + { + var env = Env(); + var answer = AnswerAssembler.Assemble(_program, env); + + if (_question is not null && !_question.ExpectedOutputs.IsEmpty) + { + var lines = _question.ExpectedOutputs + .Where(name => _sigma.ContainsKey(name)) + .Select(name => $"@{name} = {Evaluator.FormatForDisplay(env.GetNode(name))}") + .ToList(); + + if (lines.Count > 0) + { + answer = answer + "\n" + string.Join("\n", lines); + } + } + + if (_question is { LearnRuleOnSuccess: true }) + { + // Learning is best-effort: a compiler surprise on a live program shape must + // never block the answer. + try + { + LearnRule(); + } + catch (Exception ex) + { + Trace(seq => new ProtocolRepaired(seq, "learning-skipped", Sanitize(ex.Message))); + } + } + + _phase = new Completed(answer); + _pendingRequestIds = []; + Trace(seq => new AnswerAssembled(seq, answer)); + _effects.Add(new EmitAnswer(answer)); + + return Freeze(); + } + + /// + /// Tennent abstraction, mechanically: the successful derivation (taken paths, engine + /// asides stripped) becomes a rule whose In-parameters are the question's initial + /// bindings and whose Out-parameters are its declared outputs. The DefineRule payload + /// carries BOTH forms — the executable IR and the Bonsai expression tree (the durable, + /// language-agnostic intentional representation). + /// + private void LearnRule() + { + var question = _question!; + var name = question.RuleName ?? "learned_rule"; + + var parameters = ImmutableArray.CreateBuilder(); + + foreach (var input in question.InitialBindings.Keys.OrderBy(k => k, StringComparer.Ordinal)) + { + parameters.Add(new RuleParam(input, ParamMode.In)); + } + + foreach (var output in question.ExpectedOutputs) + { + parameters.Add(new RuleParam(output, ParamMode.Out)); + } + + var description = question.Text.Length > 140 ? question.Text[..140] + "…" : question.Text; + var body = _program.Where(item => item is not Comment { Aside: true }).ToImmutableArray(); + + var rule = new RuleDefinition( + new RuleSignature(name, parameters.ToImmutable(), description), + question.Text, + new UniversalisProgram(body, [], [])); + + var bonsai = BonsaiSerialization.ToBonsaiJson(BonsaiCompiler.Compile(rule)); + + var payload = System.Text.Json.JsonSerializer.Serialize(new Dictionary + { + ["ir"] = IrJson.Serialize(rule), + ["bonsai"] = bonsai, + }); + + Trace(seq => new RuleLearned(seq, name, + $"{name}({string.Join(", ", parameters.Select(p => $"{p.Name}: {(p.Mode == ParamMode.In ? "in" : "out")}"))})")); + + _effects.Add(new DefineRule(name, payload)); + } + + public StepResult Freeze() => new( + new DerivationState( + _conversationId, + _seq, + _phase, + _pendingRequestIds, + _question, + _program.ToImmutable(), + _sigma, + _choicePoints, + Budget, + _recognizer.State, + _prefill, + _engineNotes, + Pending, + _llmRequestCount, + _frames) + { + ModeBQueue = _modeBQueue, + ModeBPc = _modeBPc, + }, + _effects.ToImmutable()); + } +} \ No newline at end of file diff --git a/src/Automind.Kernel/Prompting/PromptAssembler.cs b/src/Automind.Kernel/Prompting/PromptAssembler.cs new file mode 100644 index 0000000..83be5ae --- /dev/null +++ b/src/Automind.Kernel/Prompting/PromptAssembler.cs @@ -0,0 +1,330 @@ +using System.Collections.Immutable; +using System.Text; + +using Automind.Kernel.Contract; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; + +namespace Automind.Kernel.Prompting; + +/// +/// Builds the LAWS-first system prompt for an 8B model: terse rules up front, byte-exact few-shot +/// transcripts in the middle (exemplars dominate small-model behavior), and a recency reminder — +/// plus value-sanitized engine notes accumulated by backtracking — at the very end. +/// +public static class PromptAssembler +{ + public static PromptState Build( + QuestionEnvelope question, + StepContext context, + string assistantPrefill, + ImmutableArray engineNotes, + int attempt, + string conversationId, + string requestId) + { + return new PromptState( + BuildSystemPrompt(question, context, engineNotes), + BuildQuestionText(question), + assistantPrefill, + HardStopSequences: ["\nQuestion:", "\nuser:", "\nUser:"], + MaxSegmentTokens: 512, + Temperature: Math.Min(0.2 + 0.3 * attempt, 0.8), + Seed: StableHash(conversationId + "|" + requestId)); + } + + private static string BuildQuestionText(QuestionEnvelope question) + { + if (question.InitialBindings.IsEmpty) + { + return question.Text; + } + + var sb = new StringBuilder(question.Text); + sb.Append("\n\nGiven values:"); + + foreach (var (name, json) in question.InitialBindings.OrderBy(b => b.Key, StringComparer.Ordinal)) + { + sb.Append("\n- @").Append(name).Append(" = ").Append(json); + } + + return sb.ToString(); + } + + private static string BuildSystemPrompt(QuestionEnvelope question, StepContext context, ImmutableArray engineNotes) + { + var sb = new StringBuilder(); + + sb.Append( + """ + You are Automind, a reasoning engine that answers questions by writing literate Universalis: + short natural-language sentences with executable code only inside [ ... ] hedges. + + LAWS — follow these exactly: + 1. Code appears ONLY inside [...] hedges. Never use [ or ] in ordinary prose. + 2. You NEVER know or write tool results. For every tool OUTPUT argument write a FRESH + @variable, e.g. [WEATHER("Palo Alto", @weatherPaloAlto)]. Never write a value where + an output belongs. + 3. Never invent the value of any @variable. To show a value to the user, write the + variable alone in a hedge: [@profitPct]. + 4. Never reuse an @variable name for a different value; invent a new lowerCamelCase name. + 5. Each hedge contains exactly ONE thing: one tool call, one calculation, or one variable. + Never join two calls with a comma — write each call in its own [ ... ] hedge, one at a + time, continuing the sentence naturally after each ]. + 6. Arithmetic: [@x is ]. Comparisons/conditions as bullets: "- If [@a >= @b], then ...". + Destructure JSON tool results with patterns: [TOOL(@in, { ... "field": @out ... })]. + 7. There are NO loops and none are needed: if a tool takes ONE item but your variable + holds a LIST, call the tool ONCE with the list variable — the engine automatically + applies it to every element and returns a list of results. + 8. ACT, never describe: every step must be an executable [ ... ] hedge. When the answer + has been fully shown via [@...] display hedges, finish with one short closing + sentence and STOP. + + """); + + AppendCatalogSections(sb, question, context); + + sb.Append( + """ + + EXAMPLES of perfect answers: + + Question: What is the weather between Mountain View and Menlo Park? + Answer: The city between Mountain View and Menlo Park is Palo Alto. Let's find the weather + there [WEATHER("Palo Alto", @weatherPaloAlto)]. The current weather in Palo Alto is + [@weatherPaloAlto]. That answers the question. + + Question: Alice bought a kilo of apples for $10. She sold them for $17. How much percent + profit or loss did Alice make? + Answer: The apples cost $[@buyPrice is 10] and sold for $[@sellPrice is 17], so the profit + is [@profit is @sellPrice - @buyPrice] dollars. The profit percentage is therefore + [@profitPct is (@profit / @buyPrice) * 100]. Alice made a profit of [@profitPct] percent. + + Question: Kim has $250 and a ticket costs $180. Can Kim buy it, and how much remains? + Answer: Kim has [@cash is 250] dollars and the ticket costs [@price is 180] dollars. Now decide: + - If [@cash >= @price], then Kim buys the ticket and keeps [@left is @cash - @price] dollars. + - Otherwise, Kim skips the ticket and keeps [@left = @cash] dollars. + Kim ends up with [@left] dollars. + + Question: What did IBM stock close at? + Answer: Let's get a quote [STOCK("IBM", { ... "close": @closePrice ... })]. IBM closed + at [@closePrice] dollars. + + Question: How many of the customers in @customers live in Palo Alto? (given values: @customers) + Answer: Consider each customer [@c = { ... "city": @city ... }] from [@customers]: + - Retain only customers [@c] where [@city = "Palo Alto"]. + - Subsequently, increment [@total] by one for each retained customer [@c]. + There are [@total] such customers. + + Question: Convert every file in the folder @folder to PDF. (given values: @folder) + Answer: First, list the files [LIST_FILES(@folder, @files)]. The conversion tool takes + ONE file, so I call it once with the whole list [TO_PDF(@files, @pdfs)] and the engine + converts every file automatically. The new files are [@pdfs]. All done. + + REMINDER: outputs are FRESH @variables, never values (law 2). Show values only via + [@variable] display hedges (law 3). After each ] the call is DONE and its @variable + holds the result — NEVER repeat a call; continue with the next step or display the + result. Stop after a short closing sentence (law 7). + """); + + AppendEngineNotes(sb, engineNotes); + + return sb.ToString(); + } + + // ================================================================ Mode B: whole-program synthesis + + /// + /// The Mode B prompt: ONE structured completion returns the papers' {comment|expression}[] + /// interchange form (wrapped in a {"program": …} root for schema-constrained decoding). + /// Same laws, same tool/rule catalog, same engine-note feedback — only the delivery differs: + /// the model writes the complete program up front instead of being intercepted per hedge. + /// + public static PromptState BuildWholeProgram( + QuestionEnvelope question, + StepContext context, + ImmutableArray engineNotes, + int attempt, + string conversationId, + string requestId) + { + var sb = new StringBuilder(); + + sb.Append( + """ + You are Automind, a reasoning engine. You answer by writing ONE complete literate + Universalis program as JSON. You never execute anything and never know any result — + the engine runs your program afterwards and fills in every value. + + OUTPUT FORMAT — exactly one JSON object, nothing else: + {"program": [ {"comment": "prose"}, {"expression": "code"}, … ]} + The items read top-to-bottom as one literate document: prose in "comment" items, + executable code in "expression" items. + + LAWS — follow these exactly: + 1. Code appears ONLY in "expression" items — never write brackets [ ] anywhere. + 2. You NEVER know or write tool results. Every tool OUTPUT argument is a FRESH + @variable. Never write a value where an output belongs. + 3. Never invent the value of any @variable. To show a value in the answer, add an + expression holding the variable alone: {"expression": "@profitPct"}. + 4. Each expression contains exactly ONE thing: one tool call, one calculation + "@x is ", one condition, or one display variable. Never join two. + 5. Decisions are checklists: a comment starting "- If" then the condition expression, + the branch prose and its binding expression; then "- Otherwise, …" with its + binding expression. EVERY branch must bind the same variable. + 6. There are NO loops: a tool that takes ONE item accepts a LIST variable — the + engine applies it to every element automatically. + 7. End by displaying each required output variable, then one short closing comment. + + """); + + AppendCatalogSections(sb, question, context); + + sb.Append( + """ + + EXAMPLE — question: "Kim has $250 and a ticket costs $180. Can Kim buy it, and how + much remains? Bind the remainder as @left." Perfect answer: + {"program": [ + {"comment": "Kim has"}, + {"expression": "@cash is 250"}, + {"comment": "dollars and the ticket costs"}, + {"expression": "@price is 180"}, + {"comment": "dollars. Now decide:"}, + {"comment": "- If"}, + {"expression": "@cash >= @price"}, + {"comment": ", then Kim buys the ticket and keeps"}, + {"expression": "@left is @cash - @price"}, + {"comment": "dollars."}, + {"comment": "- Otherwise, Kim skips the ticket and keeps"}, + {"expression": "@left = @cash"}, + {"comment": "dollars."}, + {"comment": "Kim ends up with"}, + {"expression": "@left"}, + {"comment": "dollars."} + ]} + + EXAMPLE — question: "Alice bought apples for $10 and sold them for $17. What is the + profit percentage? Bind it as @profitPct." Perfect answer: + {"program": [ + {"comment": "The apples cost"}, + {"expression": "@buyPrice is 10"}, + {"comment": "dollars and sold for"}, + {"expression": "@sellPrice is 17"}, + {"comment": "dollars, so the profit is"}, + {"expression": "@profit is @sellPrice - @buyPrice"}, + {"comment": "dollars. The profit percentage is"}, + {"expression": "@profitPct is (@profit / @buyPrice) * 100"}, + {"comment": ". Alice made a profit of"}, + {"expression": "@profitPct"}, + {"comment": "percent."} + ]} + + REMINDER: outputs are FRESH @variables, never values (law 2). The program must be + COMPLETE — the engine executes it exactly once, top to bottom. + """); + + AppendEngineNotes(sb, engineNotes); + + return new PromptState( + sb.ToString(), + BuildQuestionText(question), + AssistantPrefill: "", + HardStopSequences: [], + // Grows across attempts: a program truncated at a fixed cap regenerates into the + // SAME ceiling forever (review finding) — and thinking models spend part of the + // budget before the first JSON byte. + MaxSegmentTokens: 1024 + 512 * Math.Min(attempt, 2), + Temperature: Math.Min(0.2 + 0.3 * attempt, 0.8), + Seed: StableHash(conversationId + "|" + requestId), + WholeProgram: true); + } + + // ================================================================ shared sections + + private static void AppendCatalogSections(StringBuilder sb, QuestionEnvelope question, StepContext context) + { + sb.Append("\nTOOLS — the only predicates you may call:\n"); + + foreach (var tool in context.Tools) + { + sb.Append("- ").Append(Describe(tool.Signature)).Append(" — ").Append(tool.Description).Append('\n'); + } + + // RAG as virtual memory: when the pager recalled a subset, advertise ONLY that subset — + // the rule library scales beyond what always-in-context could hold. (Execution can still + // dispatch ANY stored rule; this only shapes what the model sees.) + ImmutableArray advertisedRules = question.RecalledRules is { } recalled + ? [.. context.Rules.Where(r => recalled.Contains(r.Signature.Name, StringComparer.OrdinalIgnoreCase))] + : context.Rules; + + if (!advertisedRules.IsEmpty) + { + sb.Append("\nLEARNED RULES — reusable like tools:\n"); + + foreach (var rule in advertisedRules) + { + sb.Append("- ").Append(Describe(ToSignature(rule))).Append(" — ").Append(rule.Signature.Description).Append('\n'); + } + + // The invocation affordance: without it, small models re-derive a rule's BODY from + // its description instead of calling it (observed live with a stored query rule). + // NO @-tokens in this sentence: models parrot them into answer prose and the + // phantom-variable guard then vetoes an otherwise-correct answer (observed live + // with '@in' — three good completions rejected, budget exhausted). + sb.Append("Invoke a learned rule exactly like a tool, in ONE hedge — inputs first, a FRESH variable as the final output — and never re-derive what a rule already does.\n"); + } + + if (!question.ContextChunks.IsEmpty) + { + sb.Append("\nCONTEXT — background knowledge that may help answer this question:\n"); + + foreach (var chunk in question.ContextChunks) + { + var text = chunk.Text.Length > 900 ? chunk.Text[..900] + "…" : chunk.Text; + sb.Append("• ").Append(chunk.Title).Append(": ").Append(text.ReplaceLineEndings(" ")).Append('\n'); + } + } + } + + private static void AppendEngineNotes(StringBuilder sb, ImmutableArray engineNotes) + { + if (engineNotes.IsEmpty) + { + return; + } + + sb.Append("\n\nENGINE NOTES from earlier attempts:\n"); + + foreach (var note in engineNotes) + { + sb.Append("- ").Append(note).Append('\n'); + } + } + + private static string Describe(PredicateSignature signature) => + $"{signature.Name}({string.Join(", ", signature.Params.Select(p => $"{p.Name}: {(p.Mode == ParamMode.In ? "in" : "out")}"))})"; + + private static PredicateSignature ToSignature(RuleDefinition rule) => new( + rule.Signature.Name, + [.. rule.Signature.Params.Select(p => new PredicateParam(p.Name, p.Mode))], + rule.Signature.Description, + IsRule: true); + + /// Deterministic FNV-1a hash — reproducible seeds without . + public static int StableHash(string text) + { + unchecked + { + var hash = 2166136261u; + + foreach (var c in text) + { + hash = (hash ^ c) * 16777619u; + } + + return (int)(hash & 0x7FFFFFFF); + } + } +} diff --git a/src/Automind.Kernel/Prompting/PromptState.cs b/src/Automind.Kernel/Prompting/PromptState.cs new file mode 100644 index 0000000..5d93c46 --- /dev/null +++ b/src/Automind.Kernel/Prompting/PromptState.cs @@ -0,0 +1,30 @@ +using System.Collections.Immutable; +using System.Text.Json; + +namespace Automind.Kernel.Prompting; + +/// +/// Everything the LLM bridge needs to issue one generation segment. Serialized into the +/// RequestLlm effect; the bridge renders it to chat messages, streams with the hedge +/// scanner, and cuts at the balanced hedge close. +/// +public sealed record PromptState( + string SystemPrompt, + string QuestionText, + string AssistantPrefill, + ImmutableArray HardStopSequences, + int MaxSegmentTokens, + double Temperature, + int Seed, + bool WholeProgram = false) +{ + private static readonly JsonSerializerOptions s_json = new() + { + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + public string ToJson() => JsonSerializer.Serialize(this, s_json); + + public static PromptState FromJson(string json) => + JsonSerializer.Deserialize(json, s_json) ?? throw new JsonException("null prompt state"); +} diff --git a/src/Automind.Mcp/Automind.Mcp.csproj b/src/Automind.Mcp/Automind.Mcp.csproj new file mode 100644 index 0000000..f0d09db --- /dev/null +++ b/src/Automind.Mcp/Automind.Mcp.csproj @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/src/Automind.Mcp/McpBridgedTool.cs b/src/Automind.Mcp/McpBridgedTool.cs new file mode 100644 index 0000000..1dc9982 --- /dev/null +++ b/src/Automind.Mcp/McpBridgedTool.cs @@ -0,0 +1,69 @@ +using System.Text.Json; + +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol; + +using Automind.Tools; + +using Universalis.Core.Evaluation; + +namespace Automind.Mcp; + +/// +/// One MCP tool as a Universalis predicate. Invocation flows through the engine's durable +/// InvokeTool effect like every other tool — the hedge protocol, NOT the chat client's native +/// function calling, is the tool-calling mechanism (register discipline, durability, and +/// backtracking all depend on it). An MCP error result throws, which the driver converts to +/// ToolFailed and the kernel to a backtrack. +/// +public sealed class McpBridgedTool : ITool +{ + // Same source name as the substrate's — listeners subscribe by NAME; this project + // deliberately has no Automind.Reaqtor reference. + private static readonly System.Diagnostics.ActivitySource s_activity = new("Automind.Substrate"); + + private readonly McpClientTool _tool; + private readonly string _serverName; + + public McpBridgedTool(McpClientTool tool, string serverName, PredicateSignature signature, bool isIdempotent) + { + _tool = tool; + _serverName = serverName; + Signature = signature; + IsIdempotent = isIdempotent; + } + + public PredicateSignature Signature { get; } + + public string Description => _tool.Description is { Length: > 0 } d ? d : _tool.Name; + + public bool IsIdempotent { get; } + + public async Task> InvokeAsync(string argsJson, CancellationToken cancellationToken) + { + using var activity = s_activity.StartActivity("mcp.call"); + activity?.SetTag("automind.mcp.server", _serverName); + activity?.SetTag("automind.mcp.tool", _tool.Name); + + var arguments = new Dictionary(StringComparer.Ordinal); + + using (var doc = JsonDocument.Parse(argsJson)) + { + foreach (var property in doc.RootElement.EnumerateObject()) + { + arguments[property.Name] = property.Value.Clone(); + } + } + + var result = await _tool.CallAsync(arguments, cancellationToken: cancellationToken).ConfigureAwait(false); + + if (result.IsError == true) + { + var reason = string.Join("\n", result.Content.OfType().Select(block => block.Text)); + activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, reason); + throw new InvalidOperationException(reason.Length > 0 ? reason : $"MCP tool '{_tool.Name}' reported an error"); + } + + return [McpPredicateMapper.MapResult(result)]; + } +} diff --git a/src/Automind.Mcp/McpPredicateMapper.cs b/src/Automind.Mcp/McpPredicateMapper.cs new file mode 100644 index 0000000..eb3c19d --- /dev/null +++ b/src/Automind.Mcp/McpPredicateMapper.cs @@ -0,0 +1,179 @@ +using System.Collections.Immutable; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; + +using ModelContextProtocol.Protocol; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; + +namespace Automind.Mcp; + +/// +/// Pure mapping between the MCP tool shape and the Universalis predicate shape. The predicate +/// NAME is uppercased to the catalog convention (search_files → SEARCH_FILES), +/// but parameter names stay VERBATIM — the engine serializes call arguments keyed by parameter +/// name, and those keys must match the MCP input schema exactly. +/// +public static class McpPredicateMapper +{ + /// MCP tool name → predicate name (UPPER_SNAKE, leading letter guaranteed). + public static string ToPredicateName(string mcpName) + { + var sb = new StringBuilder(mcpName.Length); + + foreach (var c in mcpName) + { + sb.Append(char.IsLetterOrDigit(c) ? char.ToUpperInvariant(c) : '_'); + } + + var name = sb.ToString().Trim('_'); + + return name.Length == 0 || !char.IsLetter(name[0]) ? "MCP_" + name : name; + } + + /// + /// Input schema → fixed-mode signature: the REQUIRED properties (in schema document order) + /// become In-parameters, plus one trailing Out-parameter for the result. Optional MCP + /// parameters are dropped — Universalis is fixed-arity, and a narrower tool confuses a + /// small model less than optional arity would (documented v1 limitation). + /// + public static PredicateSignature ToSignature(string mcpName, string? description, JsonElement inputSchema) + { + var required = new HashSet(StringComparer.Ordinal); + + if (inputSchema.ValueKind == JsonValueKind.Object && + inputSchema.TryGetProperty("required", out var requiredArray) && + requiredArray.ValueKind == JsonValueKind.Array) + { + foreach (var name in requiredArray.EnumerateArray()) + { + if (name.ValueKind == JsonValueKind.String) + { + required.Add(name.GetString()!); + } + } + } + + var parameters = ImmutableArray.CreateBuilder(); + + if (inputSchema.ValueKind == JsonValueKind.Object && + inputSchema.TryGetProperty("properties", out var properties) && + properties.ValueKind == JsonValueKind.Object) + { + foreach (var property in properties.EnumerateObject()) + { + if (required.Contains(property.Name)) + { + // An array-typed parameter must receive the WHOLE list: without + // AcceptsCollection the engine zip-lifts a list argument into one + // invocation per element and the server rejects every scalar + // (review finding). + parameters.Add(new PredicateParam(property.Name, ParamMode.In, IsArrayTyped(property.Value))); + } + } + } + + var outName = parameters.Any(p => p.Name == "result") ? "outValue" : "result"; + parameters.Add(new PredicateParam(outName, ParamMode.Out)); + + return new PredicateSignature( + ToPredicateName(mcpName), + parameters.ToImmutable(), + description ?? mcpName); + } + + /// "type": "array" — as a plain string or inside a type union (["array","null"]). + private static bool IsArrayTyped(JsonElement propertySchema) + { + if (propertySchema.ValueKind != JsonValueKind.Object || + !propertySchema.TryGetProperty("type", out var type)) + { + return false; + } + + return type.ValueKind switch + { + JsonValueKind.String => type.GetString() == "array", + JsonValueKind.Array => type.EnumerateArray().Any(t => t.ValueKind == JsonValueKind.String && t.GetString() == "array"), + _ => false, + }; + } + + /// + /// True when the input schema uses composition the flat mapper cannot see: allOf/anyOf/ + /// oneOf/$ref at the top level, or required names with no matching top-level property. + /// Bridging such a tool would advertise a silently WRONG arity — better to skip it with a + /// warning at connect time than fail every invocation (review finding). + /// + public static bool HasUnmappedComposition(JsonElement inputSchema) + { + if (inputSchema.ValueKind != JsonValueKind.Object) + { + return false; + } + + foreach (var keyword in (string[])["allOf", "anyOf", "oneOf", "$ref"]) + { + if (inputSchema.TryGetProperty(keyword, out _)) + { + return true; + } + } + + if (inputSchema.TryGetProperty("required", out var required) && required.ValueKind == JsonValueKind.Array) + { + var hasProperties = inputSchema.TryGetProperty("properties", out var properties) && + properties.ValueKind == JsonValueKind.Object; + + foreach (var name in required.EnumerateArray()) + { + if (name.ValueKind == JsonValueKind.String && + (!hasProperties || !properties.TryGetProperty(name.GetString()!, out _))) + { + return true; + } + } + } + + return false; + } + + /// + /// Recovery re-issues in-flight idempotent calls with the same request id; an MCP tool with + /// unknown side effects must NOT double-fire, so only the server's own annotations opt in. + /// + public static bool IsIdempotent(ToolAnnotations? annotations) => + annotations?.ReadOnlyHint == true || annotations?.IdempotentHint == true; + + /// + /// Result → one JSON value: prefer the server's structured content (it IS JSON); otherwise + /// join the text blocks — passed through RAW when the text parses as JSON (so pattern + /// destructuring works on JSON-returning tools), string-encoded when it is plain prose. + /// + public static string MapResult(CallToolResult result) + { + if (result.StructuredContent is { } structured) + { + return structured.GetRawText(); + } + + var text = string.Join("\n", result.Content.OfType().Select(block => block.Text)); + + if (text.Length == 0) + { + return "null"; + } + + try + { + _ = JsonNode.Parse(text); + return text; + } + catch (JsonException) + { + return JsonSerializer.Serialize(text); + } + } +} diff --git a/src/Automind.Mcp/McpToolBridge.cs b/src/Automind.Mcp/McpToolBridge.cs new file mode 100644 index 0000000..e47660a --- /dev/null +++ b/src/Automind.Mcp/McpToolBridge.cs @@ -0,0 +1,147 @@ +using ModelContextProtocol.Client; + +using Automind.Tools; + +namespace Automind.Mcp; + +/// +/// Connects MCP servers and surfaces their tools as Universalis predicates. One bridge owns +/// every client session (a stdio session owns its child process); dispose the bridge to shut +/// them down. Connection is fail-soft per server: a server that will not start costs a warning, +/// never the derivation host. +/// +public sealed class McpToolBridge : IAsyncDisposable +{ + private readonly List _clients = []; + private readonly List _tools = []; + + private McpToolBridge() + { + } + + public IReadOnlyList Tools => _tools; + + /// + /// : one command line per stdio server ("npx -y @scope/server", + /// "dotnet path/to/Server.dll"). filters by MCP or predicate + /// name, case-insensitive; null bridges everything. + /// + public static async Task ConnectAsync( + IEnumerable serverCommands, + IReadOnlySet? allowlist, + Action log, + CancellationToken cancellationToken = default) + { + var bridge = new McpToolBridge(); + + foreach (var commandLine in serverCommands) + { + var parts = SplitCommandLine(commandLine); + + if (parts.Length == 0) + { + continue; + } + + try + { + var client = await McpClient.CreateAsync( + new StdioClientTransport(new StdioClientTransportOptions + { + Command = parts[0], + Arguments = [.. parts.Skip(1)], + Name = Path.GetFileNameWithoutExtension(parts[0]), + }), + cancellationToken: cancellationToken).ConfigureAwait(false); + + bridge._clients.Add(client); + + var serverName = client.ServerInfo?.Name ?? parts[0]; + + foreach (var tool in await client.ListToolsAsync(cancellationToken: cancellationToken).ConfigureAwait(false)) + { + // A schema this mapper cannot see through (allOf/$ref composition) would + // bridge with a silently wrong arity and fail every invocation — skip loudly. + if (McpPredicateMapper.HasUnmappedComposition(tool.JsonSchema)) + { + log($"MCP tool '{tool.Name}' uses schema composition (allOf/$ref) the bridge cannot map — skipped"); + continue; + } + + var signature = McpPredicateMapper.ToSignature(tool.Name, tool.Description, tool.JsonSchema); + + if (allowlist is not null && + !allowlist.Contains(tool.Name) && + !allowlist.Contains(signature.Name)) + { + continue; + } + + bridge._tools.Add(new McpBridgedTool( + tool, + serverName, + signature, + McpPredicateMapper.IsIdempotent(tool.ProtocolTool.Annotations))); + } + } + catch (Exception ex) + { + log($"MCP server '{commandLine}' skipped: {ex.Message}"); + } + } + + return bridge; + } + + /// Splits a command line on whitespace, honoring double quotes. + public static string[] SplitCommandLine(string commandLine) + { + var parts = new List(); + var current = new System.Text.StringBuilder(); + var quoted = false; + + foreach (var c in commandLine) + { + if (c == '"') + { + quoted = !quoted; + } + else if (char.IsWhiteSpace(c) && !quoted) + { + if (current.Length > 0) + { + parts.Add(current.ToString()); + current.Clear(); + } + } + else + { + current.Append(c); + } + } + + if (current.Length > 0) + { + parts.Add(current.ToString()); + } + + return [.. parts]; + } + + public async ValueTask DisposeAsync() + { + foreach (var client in _clients) + { + try + { + await client.DisposeAsync().ConfigureAwait(false); + } + catch (Exception) + { + // Shutdown is best-effort; a dead child process is already the goal state. + } + } + + _clients.Clear(); + } +} diff --git a/src/Automind.Memory/Automind.Memory.csproj b/src/Automind.Memory/Automind.Memory.csproj new file mode 100644 index 0000000..fbd87e6 --- /dev/null +++ b/src/Automind.Memory/Automind.Memory.csproj @@ -0,0 +1,18 @@ + + + + + $(NoWarn);SKEXP0070;SKEXP0001 + + + + + + + + + + + + diff --git a/src/Automind.Memory/OnnxMemoryPager.cs b/src/Automind.Memory/OnnxMemoryPager.cs new file mode 100644 index 0000000..c9d845e --- /dev/null +++ b/src/Automind.Memory/OnnxMemoryPager.cs @@ -0,0 +1,164 @@ +using Microsoft.Extensions.AI; +using Microsoft.SemanticKernel.Connectors.Onnx; + +namespace Automind.Memory; + +/// One page of the neural computer's virtual memory: a rule summary or a document chunk. +public sealed record MemoryChunk(string Id, string Kind, string Title, string Text) +{ + public const string RuleKind = "rule"; + public const string DocKind = "doc"; +} + +public interface IMemoryPager : IDisposable +{ + Task IndexAsync(MemoryChunk chunk, CancellationToken cancellationToken = default); + + Task> RecallAsync(string query, int top, CancellationToken cancellationToken = default); +} + +/// +/// RAG as virtual memory, per the papers: the ENGINE (not the model) pages relevant knowledge +/// into the context. Embeddings run fully IN-PROCESS via the Semantic Kernel BERT-ONNX connector +/// (bge-micro-v2, ~23 MB, CPU, milliseconds) — no Ollama, no server, no generative model. +/// Retrieval is a plain cosine top-k over an in-memory index: at POC scale (hundreds of chunks) +/// this is exact, dependency-light, and trivially swappable for a vector database later. +/// +public sealed class OnnxMemoryPager : IMemoryPager +{ + // Same source name as Automind.Reaqtor's — listeners subscribe by NAME, and this project + // deliberately has no substrate reference. + private static readonly System.Diagnostics.ActivitySource s_activity = new("Automind.Substrate"); + + private readonly IEmbeddingGenerator> _embeddings; + private readonly Lock _gate = new(); + private readonly List<(MemoryChunk Chunk, float[] Vector)> _index = []; + + private OnnxMemoryPager(IEmbeddingGenerator> embeddings) => _embeddings = embeddings; + + /// Probes for model.onnx + vocab.txt; null when absent. + public static async Task TryCreateAsync(string modelDirectory, CancellationToken cancellationToken = default) + { + var model = Path.Combine(modelDirectory, "model.onnx"); + var vocab = Path.Combine(modelDirectory, "vocab.txt"); + + if (!File.Exists(model) || !File.Exists(vocab)) + { + return null; + } + + // NB: the M.E.AI-native BertOnnxEmbeddingGenerator is INTERNAL in 1.78.0-alpha (only the + // DI registration reaches it); the service type is the public construction surface, so we + // adapt it to IEmbeddingGenerator ourselves. +#pragma warning disable CS0618 + var service = await BertOnnxTextEmbeddingGenerationService.CreateAsync( + model, vocab, new BertOnnxOptions { NormalizeEmbeddings = true }, cancellationToken).ConfigureAwait(false); +#pragma warning restore CS0618 + + return new OnnxMemoryPager(new ServiceAdapter(service)); + } + + public async Task IndexAsync(MemoryChunk chunk, CancellationToken cancellationToken = default) + { + using var activity = s_activity.StartActivity("memory.index"); + activity?.SetTag("automind.memory.chunk", chunk.Id); + activity?.SetTag("automind.memory.kind", chunk.Kind); + + var embedding = await _embeddings.GenerateAsync(chunk.Title + "\n" + chunk.Text, cancellationToken: cancellationToken).ConfigureAwait(false); + var vector = embedding.Vector.ToArray(); + + lock (_gate) + { + _index.RemoveAll(entry => entry.Chunk.Id == chunk.Id); + _index.Add((chunk, vector)); + activity?.SetTag("automind.memory.index_size", _index.Count); + } + } + + public async Task> RecallAsync(string query, int top, CancellationToken cancellationToken = default) + { + using var activity = s_activity.StartActivity("memory.recall"); + activity?.SetTag("automind.memory.top", top); + + var embedding = await _embeddings.GenerateAsync(query, cancellationToken: cancellationToken).ConfigureAwait(false); + var vector = embedding.Vector.ToArray(); + + lock (_gate) + { + var hits = _index + .Select(entry => (entry.Chunk, Score: Cosine(vector, entry.Vector))) + .OrderByDescending(x => x.Score) + .Take(top) + .ToList(); + + activity?.SetTag("automind.memory.hits", hits.Count); + activity?.SetTag("automind.memory.best_score", hits.Count > 0 ? hits[0].Score : 0f); + + return [.. hits.Select(x => x.Chunk)]; + } + } + + private static float Cosine(float[] a, float[] b) + { + // Vectors are normalized at embedding time; the dot product IS the cosine. + var sum = 0f; + var length = Math.Min(a.Length, b.Length); + + for (var i = 0; i < length; i++) + { + sum += a[i] * b[i]; + } + + return sum; + } + + /// Paragraph-aware document chunking (~ per page). + public static IEnumerable ChunkDocument(string name, string text, int maxChars = 1500) + { + var paragraphs = text.Split("\n\n", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var buffer = new System.Text.StringBuilder(); + var page = 0; + + foreach (var paragraph in paragraphs) + { + if (buffer.Length > 0 && buffer.Length + paragraph.Length > maxChars) + { + yield return new MemoryChunk($"{name}#{page}", MemoryChunk.DocKind, name, buffer.ToString()); + buffer.Clear(); + page++; + } + + buffer.AppendLine(paragraph).AppendLine(); + } + + if (buffer.Length > 0) + { + yield return new MemoryChunk($"{name}#{page}", MemoryChunk.DocKind, name, buffer.ToString()); + } + } + + public void Dispose() => _embeddings.Dispose(); + +#pragma warning disable CS0618 // adapting the public (obsolete-flagged) service to M.E.AI + private sealed class ServiceAdapter : IEmbeddingGenerator> + { + private readonly BertOnnxTextEmbeddingGenerationService _service; + + public ServiceAdapter(BertOnnxTextEmbeddingGenerationService service) => _service = service; + + public async Task>> GenerateAsync( + IEnumerable values, + EmbeddingGenerationOptions? options = null, + CancellationToken cancellationToken = default) + { + var vectors = await _service.GenerateEmbeddingsAsync([.. values], kernel: null, cancellationToken).ConfigureAwait(false); + + return new GeneratedEmbeddings>([.. vectors.Select(v => new Embedding(v))]); + } + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() => _service.Dispose(); + } +#pragma warning restore CS0618 +} diff --git a/src/Automind.Reaqtor/Automind.Reaqtor.csproj b/src/Automind.Reaqtor/Automind.Reaqtor.csproj new file mode 100644 index 0000000..da1b4b7 --- /dev/null +++ b/src/Automind.Reaqtor/Automind.Reaqtor.csproj @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Automind.Reaqtor/Catalog/ConversationCatalog.cs b/src/Automind.Reaqtor/Catalog/ConversationCatalog.cs new file mode 100644 index 0000000..77ddec7 --- /dev/null +++ b/src/Automind.Reaqtor/Catalog/ConversationCatalog.cs @@ -0,0 +1,80 @@ +using System.Text; +using System.Text.Json; + +using Reaqtor.QueryEngine; + +namespace Automind.Reaqtor.Catalog; + +public sealed record ConversationRecord(string DerivationId, string Topic, string QuestionJson, string Status) +{ + public const string PendingStatus = "pending"; + public const string CompletedStatus = "completed"; + public const string FailedStatus = "failed"; +} + +/// +/// Durable conversation index on the engine's own key-value store (separate table). The host +/// reads it BEFORE RecoverAsync to pre-create egress topics — a recovered egress observer +/// resolves its topic inside SetContext, so topics must exist before engine recovery. +/// +public sealed class ConversationCatalog +{ + private const string TableName = "automind-conversations"; + + private readonly IKeyValueStore _store; + + public ConversationCatalog(IKeyValueStore store) => _store = store; + + public async Task UpsertAsync(ConversationRecord record, CancellationToken token = default) + { + using var tx = _store.CreateTransaction(); + var table = _store.GetTable(TableName).Enter(tx); + var bytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(record)); + + if (table.Contains(record.DerivationId)) + { + table.Update(record.DerivationId, bytes); + } + else + { + table.Add(record.DerivationId, bytes); + } + + await tx.CommitAsync(token).ConfigureAwait(false); + } + + public async Task SetStatusAsync(string derivationId, string status, CancellationToken token = default) + { + var existing = All().FirstOrDefault(r => r.DerivationId == derivationId); + + if (existing is not null) + { + await UpsertAsync(existing with { Status = status }, token).ConfigureAwait(false); + } + } + + public IReadOnlyList All() + { + var records = new List(); + + using var tx = _store.CreateTransaction(); + var table = _store.GetTable(TableName).Enter(tx); + + using var rows = table.GetEnumerator(); + + while (rows.MoveNext()) + { + var record = JsonSerializer.Deserialize(Encoding.UTF8.GetString(rows.Current.Value)); + + if (record is not null) + { + records.Add(record); + } + } + + return records; + } + + public IReadOnlyList Active() => + [.. All().Where(r => r.Status == ConversationRecord.PendingStatus)]; +} diff --git a/src/Automind.Reaqtor/Catalog/DocCatalogStore.cs b/src/Automind.Reaqtor/Catalog/DocCatalogStore.cs new file mode 100644 index 0000000..4a77bf4 --- /dev/null +++ b/src/Automind.Reaqtor/Catalog/DocCatalogStore.cs @@ -0,0 +1,62 @@ +using System.Text; +using System.Text.Json; + +using Reaqtor.QueryEngine; + +namespace Automind.Reaqtor.Catalog; + +/// +/// Durable raw-document store (table automind-docs). Documents persist as text; the memory +/// pager re-embeds them at startup — embeddings are local and take milliseconds, so re-indexing +/// beats persisting vectors. +/// +public sealed class DocCatalogStore +{ + private const string TableName = "automind-docs"; + + private readonly IKeyValueStore _store; + + public DocCatalogStore(IKeyValueStore store) => _store = store; + + public sealed record StoredDoc(string Name, string Text); + + public async Task SaveAsync(string name, string text, CancellationToken token = default) + { + using var tx = _store.CreateTransaction(); + var table = _store.GetTable(TableName).Enter(tx); + var bytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(new StoredDoc(name, text))); + + if (table.Contains(name)) + { + table.Update(name, bytes); + } + else + { + table.Add(name, bytes); + } + + await tx.CommitAsync(token).ConfigureAwait(false); + } + + public IReadOnlyList All() + { + var docs = new List(); + + using var tx = _store.CreateTransaction(); + var table = _store.GetTable(TableName).Enter(tx); + + using var rows = table.GetEnumerator(); + + while (rows.MoveNext()) + { + var doc = JsonSerializer.Deserialize(Encoding.UTF8.GetString(rows.Current.Value)); + + if (doc is not null) + { + docs.Add(doc); + } + } + + return docs; + } +} diff --git a/src/Automind.Reaqtor/Catalog/McpCatalogStore.cs b/src/Automind.Reaqtor/Catalog/McpCatalogStore.cs new file mode 100644 index 0000000..0964de1 --- /dev/null +++ b/src/Automind.Reaqtor/Catalog/McpCatalogStore.cs @@ -0,0 +1,51 @@ +using System.Text; +using System.Text.Json; + +using Reaqtor.QueryEngine; + +namespace Automind.Reaqtor.Catalog; + +/// +/// Durable record of the MCP server commands bridged into this data directory (table +/// automind-mcp). Recovery re-issues in-flight tool calls by URI, so a resumed process +/// must reconnect the same servers or those calls fail "no tool is registered" — the CLI +/// promises "kill and re-run to resume" without requiring the flags to be repeated +/// (review finding). +/// +public sealed class McpCatalogStore +{ + private const string TableName = "automind-mcp"; + private const string Key = "servers"; + + private readonly IKeyValueStore _store; + + public McpCatalogStore(IKeyValueStore store) => _store = store; + + public async Task SaveAsync(IReadOnlyList serverCommands, CancellationToken token = default) + { + using var tx = _store.CreateTransaction(); + var table = _store.GetTable(TableName).Enter(tx); + var bytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(serverCommands)); + + if (table.Contains(Key)) + { + table.Update(Key, bytes); + } + else + { + table.Add(Key, bytes); + } + + await tx.CommitAsync(token).ConfigureAwait(false); + } + + public IReadOnlyList Servers() + { + using var tx = _store.CreateTransaction(); + var table = _store.GetTable(TableName).Enter(tx); + + return table.Contains(Key) + ? JsonSerializer.Deserialize>(Encoding.UTF8.GetString(table[Key])) ?? [] + : []; + } +} diff --git a/src/Automind.Reaqtor/Catalog/RuleCatalogStore.cs b/src/Automind.Reaqtor/Catalog/RuleCatalogStore.cs new file mode 100644 index 0000000..cc29625 --- /dev/null +++ b/src/Automind.Reaqtor/Catalog/RuleCatalogStore.cs @@ -0,0 +1,68 @@ +using System.Text; +using System.Text.Json; + +using Reaqtor.QueryEngine; + +using Universalis.Core.Ir; + +namespace Automind.Reaqtor.Catalog; + +/// +/// Durable rule library on the engine's own key-value store. Each row carries BOTH forms of a +/// learned rule: the executable IR (interpreted at call sites) and the Bonsai expression tree +/// (the durable, language-agnostic intentional representation). Rows commit through the same +/// write-through path as everything else — a learned rule survives a process kill. +/// +public sealed class RuleCatalogStore +{ + private const string TableName = "automind-rules"; + + private readonly IKeyValueStore _store; + + public RuleCatalogStore(IKeyValueStore store) => _store = store; + + public sealed record StoredRule(string Name, string IrJson, string BonsaiJson) + { + public RuleDefinition Definition => Universalis.Core.Ir.IrJson.DeserializeRule(IrJson); + } + + public async Task SaveAsync(string name, string irJson, string bonsaiJson, CancellationToken token = default) + { + using var tx = _store.CreateTransaction(); + var table = _store.GetTable(TableName).Enter(tx); + var bytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(new StoredRule(name, irJson, bonsaiJson))); + + if (table.Contains(name)) + { + table.Update(name, bytes); + } + else + { + table.Add(name, bytes); + } + + await tx.CommitAsync(token).ConfigureAwait(false); + } + + public IReadOnlyList All() + { + var rules = new List(); + + using var tx = _store.CreateTransaction(); + var table = _store.GetTable(TableName).Enter(tx); + + using var rows = table.GetEnumerator(); + + while (rows.MoveNext()) + { + var rule = JsonSerializer.Deserialize(Encoding.UTF8.GetString(rows.Current.Value)); + + if (rule is not null) + { + rules.Add(rule); + } + } + + return rules; + } +} diff --git a/src/Automind.Reaqtor/Catalog/ToolRegistryStepContextProvider.cs b/src/Automind.Reaqtor/Catalog/ToolRegistryStepContextProvider.cs new file mode 100644 index 0000000..37e0b74 --- /dev/null +++ b/src/Automind.Reaqtor/Catalog/ToolRegistryStepContextProvider.cs @@ -0,0 +1,27 @@ +using System.Collections.Immutable; + +using Automind.Kernel.Contract; +using Automind.Reaqtor.Reactive; +using Automind.Tools; + +using Universalis.Core.Ir; + +namespace Automind.Reaqtor.Catalog; + +/// +/// Builds the step-context snapshot from the live tool registry (and, from P7, the rule catalog). +/// The snapshot is rebuilt per access so runtime-defined rules become visible to the next step. +/// +public sealed class ToolRegistryStepContextProvider : IStepContextProvider +{ + private readonly IToolRegistry _tools; + private ImmutableArray _rules = []; + + public ToolRegistryStepContextProvider(IToolRegistry tools) => _tools = tools; + + public void AddRule(RuleDefinition rule) => _rules = _rules.Add(rule); + + public StepContext Current => new( + [.. _tools.All.Select(t => new ToolBinding(t.Tool.Signature, t.Uri, t.Tool.IsIdempotent, t.Tool.Description))], + _rules); +} diff --git a/src/Automind.Reaqtor/Client/AutomindClientContext.cs b/src/Automind.Reaqtor/Client/AutomindClientContext.cs new file mode 100644 index 0000000..53f1296 --- /dev/null +++ b/src/Automind.Reaqtor/Client/AutomindClientContext.cs @@ -0,0 +1,47 @@ +using Automind.Reaqtor.Engine; + +using Reaqtor; +using Reaqtor.Shebang.Client; +using Reaqtor.Shebang.Service; + +namespace Automind.Reaqtor.Client; + +/// +/// The strongly-typed development surface over the Automind engine — the Reaqtor analog of a +/// LINQ-to-SQL DataContext. Each accessor binds a URI-identified engine artifact; the +/// makes the expression rewriter turn calls into invocations +/// of the unbound artifact parameter. +/// +public sealed class AutomindClientContext : ClientContext +{ + public AutomindClientContext(IReactiveServiceProvider provider) : base(provider) + { + } + + public static AutomindClientContext For(SimplerCheckpointingQueryEngine engine) => + new(new LocalReactiveServiceProvider(engine.ServiceProvider)); + + [KnownResource(AutomindUris.TimerId)] + public IAsyncReactiveQbservable Heartbeat(TimeSpan period) => + GetObservable(AutomindUris.Timer)(period); + + [KnownResource(AutomindUris.IngressId)] + public IAsyncReactiveQbservable Ingress(string topic) => + GetObservable(AutomindUris.Ingress)(topic); + + [KnownResource(AutomindUris.EgressId)] + public IAsyncReactiveQbserver Egress(string topic) => + GetObserver(AutomindUris.Egress)(topic); + + [KnownResource(AutomindUris.ConsoleId)] + public IAsyncReactiveQbserver ConsoleSink() => + GetObserver(AutomindUris.Console); + + [KnownResource(AutomindUris.DerivationId)] + public IAsyncReactiveQbservable Derivation(string derivationId, string questionJson) => + GetObservable(AutomindUris.Derivation)(derivationId, questionJson); + + [KnownResource(AutomindUris.ToolInvokeId)] + public IAsyncReactiveQbservable ToolInvoke(string toolUri, string argsJson) => + GetObservable(AutomindUris.ToolInvoke)(toolUri, argsJson); +} diff --git a/src/Automind.Reaqtor/Engine/AutomindArtifacts.cs b/src/Automind.Reaqtor/Engine/AutomindArtifacts.cs new file mode 100644 index 0000000..93b776d --- /dev/null +++ b/src/Automind.Reaqtor/Engine/AutomindArtifacts.cs @@ -0,0 +1,55 @@ +using System.Linq.CompilerServices.TypeSystem; +using System.Linq.Expressions; + +using Automind.Reaqtor.Client; +using Automind.Reaqtor.Reactive; + +using Reaqtor.Shebang.Linq; +using Reaqtor.Shebang.Service; + +namespace Automind.Reaqtor.Engine; + +/// +/// Defines the Automind artifact catalog in a (fresh) engine. All definitions go through the +/// async ServiceProvider path, which is write-ahead-logged; the factory checkpoints +/// immediately afterwards so definitions can never be lost. +/// +public static class AutomindArtifacts +{ + public static async Task DefineAsync(SimplerCheckpointingQueryEngine engine, CancellationToken token = default) + { + var ctx = AutomindClientContext.For(engine); + + await ctx.DefineObservableAsync( + AutomindUris.Timer, + period => new TimerObservable(period).AsAsyncQbservable(), + null, token).ConfigureAwait(false); + + await ctx.DefineObservableAsync( + AutomindUris.Ingress, + topic => new IngressObservable(topic).AsAsyncQbservable(), + null, token).ConfigureAwait(false); + + await ctx.DefineObserverAsync( + AutomindUris.Egress, + topic => new EgressObserver(topic).AsAsyncQbserver(), + null, token).ConfigureAwait(false); + + await ctx.DefineObserverAsync( + AutomindUris.Console, + ctx.Provider.CreateQbserver(Expression.New(typeof(ConsoleObserver))), + null, token).ConfigureAwait(false); + + // The neural computer's entry point: one derivation per question, as a standing query. + await ctx.DefineObservableAsync( + AutomindUris.Derivation, + (derivationId, questionJson) => new DerivationObservable(derivationId, questionJson).AsAsyncQbservable(), + null, token).ConfigureAwait(false); + + // The tool router: dynamic dispatch over the static artifact space (tool URI as data). + await ctx.DefineObservableAsync( + AutomindUris.ToolInvoke, + (toolUri, argsJson) => new ToolInvokeObservable(toolUri, argsJson).AsAsyncQbservable(), + null, token).ConfigureAwait(false); + } +} diff --git a/src/Automind.Reaqtor/Engine/AutomindEngineFactory.cs b/src/Automind.Reaqtor/Engine/AutomindEngineFactory.cs new file mode 100644 index 0000000..bab5d81 --- /dev/null +++ b/src/Automind.Reaqtor/Engine/AutomindEngineFactory.cs @@ -0,0 +1,69 @@ +using System.Diagnostics; + +using Reaqtive.Scheduler; + +using Reaqtor.Shebang.Service; + +namespace Automind.Reaqtor.Engine; + +/// +/// Stands up (or recovers) an Automind query engine over a state store. The caller owns the +/// (its worker threads are foreground by design — a live scheduler +/// means in-flight work — so hosts and tests must dispose it to let the process exit); each engine +/// gets its own , whose ownership transfers to the engine. +/// +public static class AutomindEngineFactory +{ + public static async Task CreateNewAsync( + IQueryEngineStateStore store, + PhysicalScheduler scheduler, + IReadOnlyDictionary? services = null, + IIngressEgressManager? ingressEgressManager = null, + TraceSource? traceSource = null) + { + var engine = Create(store, scheduler, services, ingressEgressManager, traceSource); + + await AutomindArtifacts.DefineAsync(engine).ConfigureAwait(false); + + // Persist the definitions before accepting any traffic (KB: sync/unlogged defines that + // aren't followed by a checkpoint can be lost forever; ours are WAL-logged, but the + // bootstrap checkpoint also makes recovery of a freshly created store well-defined). + await engine.CheckpointAsync().ConfigureAwait(false); + + return engine; + } + + public static async Task RecoverAsync( + IQueryEngineStateStore store, + PhysicalScheduler scheduler, + IReadOnlyDictionary? services = null, + IIngressEgressManager? ingressEgressManager = null, + TraceSource? traceSource = null) + { + var engine = Create(store, scheduler, services, ingressEgressManager, traceSource); + + await engine.RecoverAsync().ConfigureAwait(false); + + return engine; + } + + private static SimplerCheckpointingQueryEngine Create( + IQueryEngineStateStore store, + PhysicalScheduler scheduler, + IReadOnlyDictionary? services, + IIngressEgressManager? ingressEgressManager, + TraceSource? traceSource) + { +#pragma warning disable CA2000 // Dispose objects before losing scope. (Engine takes ownership.) + var logicalScheduler = new LogicalScheduler(scheduler); +#pragma warning restore CA2000 + + return new SimplerCheckpointingQueryEngine( + new Uri("automind://engine/" + Guid.NewGuid().ToString("D")), + logicalScheduler, + store, + services, + ingressEgressManager, + traceSource); + } +} diff --git a/src/Automind.Reaqtor/Engine/AutomindServices.cs b/src/Automind.Reaqtor/Engine/AutomindServices.cs new file mode 100644 index 0000000..30c6b9d --- /dev/null +++ b/src/Automind.Reaqtor/Engine/AutomindServices.cs @@ -0,0 +1,25 @@ +using Automind.Kernel.Contract; +using Automind.Reaqtor.Llm; +using Automind.Reaqtor.Reactive; +using Automind.Tools; + +namespace Automind.Reaqtor.Engine; + +/// +/// The service set injected into every operator context — how in-engine operators reach the +/// world outside (LLM bridge, tool registry) and the pure kernel (step function, catalogs). +/// +public sealed record AutomindServices( + IStepFunction StepFunction, + IStepContextProvider StepContext, + ILlmService LlmService, + IToolRegistry ToolRegistry) +{ + public IReadOnlyDictionary ToDictionary() => new Dictionary + { + [AutomindServiceKeys.StepFunction] = StepFunction, + [AutomindServiceKeys.StepContext] = StepContext, + [AutomindServiceKeys.LlmService] = LlmService, + [AutomindServiceKeys.ToolRegistry] = ToolRegistry, + }; +} diff --git a/src/Automind.Reaqtor/Engine/AutomindUris.cs b/src/Automind.Reaqtor/Engine/AutomindUris.cs new file mode 100644 index 0000000..34a9acd --- /dev/null +++ b/src/Automind.Reaqtor/Engine/AutomindUris.cs @@ -0,0 +1,23 @@ +namespace Automind.Reaqtor.Engine; + +/// +/// The URI catalog of Automind-owned engine artifacts. All Automind artifacts live under the +/// automind:// scheme; the (borrowed) generic operator space keeps its rx:// URIs +/// (rx://builtin/id is load-bearing and well-known to all Reaqtor components). +/// +public static class AutomindUris +{ + public const string TimerId = "automind://observables/timer"; + public const string IngressId = "automind://observables/ingress"; + public const string EgressId = "automind://observers/egress"; + public const string ConsoleId = "automind://observers/console"; + public const string DerivationId = "automind://derivation"; + public const string ToolInvokeId = "automind://tools/invoke"; + + public static readonly Uri Timer = new(TimerId); + public static readonly Uri Ingress = new(IngressId); + public static readonly Uri Egress = new(EgressId); + public static readonly Uri Console = new(ConsoleId); + public static readonly Uri Derivation = new(DerivationId); + public static readonly Uri ToolInvoke = new(ToolInvokeId); +} diff --git a/src/Automind.Reaqtor/IO/AutomindIngressEgressManager.cs b/src/Automind.Reaqtor/IO/AutomindIngressEgressManager.cs new file mode 100644 index 0000000..d4fe9e1 --- /dev/null +++ b/src/Automind.Reaqtor/IO/AutomindIngressEgressManager.cs @@ -0,0 +1,109 @@ +// Adapted from the Reaqtor Shebang/IoT samples (MIT, .NET Foundation), with one deliberate +// behavioral change: topics are get-or-create. The sample manager throws for unknown topics, +// which is a recovery-order trap — a recovered EgressObserver resolves its topic inside +// SetContext, *before* the host has a chance to re-create it. + +using Reaqtive; + +using Reaqtor.Reliable; +using Reaqtor.Shebang.Service; + +namespace Automind.Reaqtor.IO; + +/// +/// The world outside the query engine: named reliable topics (compare to Event Hub topics). +/// In-engine proxies ( / ) +/// discover this manager through the operator context under the key "IngressEgressManager". +/// +public sealed class AutomindIngressEgressManager : IIngressEgressManager +{ + private readonly Dictionary _subjects = []; + + public IReliableSubject GetOrCreateSubject(string name) + { + lock (_subjects) + { + if (_subjects.TryGetValue(name, out var existing)) + { + return existing as ReliableSubject + ?? throw new InvalidOperationException($"Topic '{name}' already exists with element type '{existing.GetType().GenericTypeArguments[0]}', not '{typeof(T)}'."); + } + + var subject = new ReliableSubject(); + _subjects.Add(name, subject); + return subject; + } + } + + public IReadOnlyList ListTopics() + { + lock (_subjects) + { + return [.. _subjects.Keys]; + } + } + + IReliableObserver IIngressEgressManager.GetObserver(string name) + { + // Used by EgressObserver instances inside the query engine. + return new ReliableObserver((ReliableSubject)GetOrCreateSubject(name)); + } + + IReliableObservable IIngressEgressManager.GetObservable(string name) + { + // Used by IngressObservable instances inside the query engine. + return new ReliableObservable((ReliableSubject)GetOrCreateSubject(name)); + } + + private sealed class ReliableObserver : IReliableObserver + { + private readonly ReliableSubject _subject; + + public ReliableObserver(ReliableSubject subject) => _subject = subject; + + public Uri ResubscribeUri => throw new NotSupportedException("Used for engine-to-engine communication; N/A here."); + + public void OnCompleted() => _subject.OnCompleted(); + + public void OnError(Exception error) => _subject.OnError(error); + + public void OnNext(T item, long sequenceId) => _subject.OnNext((sequenceId, item)); + + public void OnStarted() { } + } + + private sealed class ReliableObservable : IReliableObservable + { + private readonly ReliableSubject _subject; + + public ReliableObservable(ReliableSubject subject) => _subject = subject; + + public IReliableSubscription Subscribe(IReliableObserver observer) => new Subscription(_subject, observer); + + private sealed class Subscription : IReliableSubscription + { + private readonly ReliableSubject _subject; + private readonly IReliableObserver _observer; + private IDisposable? _subscription; + + public Subscription(ReliableSubject subject, IReliableObserver observer) + { + _subject = subject; + _observer = observer; + } + + public Uri ResubscribeUri => throw new NotSupportedException("Used for engine-to-engine communication; N/A here."); + + public void Accept(ISubscriptionVisitor visitor) => throw new NotSupportedException("Used for engine-to-engine communication; N/A here."); + + public void AcknowledgeRange(long sequenceId) + { + // NB: Could wire up to prune history. + } + + public void Dispose() => _subscription?.Dispose(); + + public void Start(long sequenceId) => _subscription = _subject.Subscribe(_observer, sequenceId); + } + } +} diff --git a/src/Automind.Reaqtor/IO/IReliableSubject.cs b/src/Automind.Reaqtor/IO/IReliableSubject.cs new file mode 100644 index 0000000..0e73ffe --- /dev/null +++ b/src/Automind.Reaqtor/IO/IReliableSubject.cs @@ -0,0 +1,11 @@ +// Adapted from the Reaqtor Shebang sample (MIT, .NET Foundation). + +namespace Automind.Reaqtor.IO; + +/// +/// A subject speaking the reliable protocol: every event carries a sequence ID, enabling +/// replay (Start(sequenceId)) and acknowledgement-based pruning after checkpoints. +/// +public interface IReliableSubject : IObservable<(long sequenceId, T item)>, IObserver<(long sequenceId, T item)> +{ +} diff --git a/src/Automind.Reaqtor/IO/ReliableSubject.cs b/src/Automind.Reaqtor/IO/ReliableSubject.cs new file mode 100644 index 0000000..6d439fa --- /dev/null +++ b/src/Automind.Reaqtor/IO/ReliableSubject.cs @@ -0,0 +1,142 @@ +// Adapted from the Reaqtor Shebang sample (MIT, .NET Foundation). + +using Reaqtor.Reliable; + +namespace Automind.Reaqtor.IO; + +/// +/// In-memory reliable subject: retains history keyed by sequence ID, deduplicates by sequence ID +/// (which is what makes at-least-once redelivery after recovery safe), and supports replay from +/// a given sequence ID for late/recovering subscribers. +/// +public sealed class ReliableSubject : IReliableSubject +{ + private readonly Lock _gate = new(); + private readonly SortedDictionary _values = []; + private readonly List> _observers = []; + private Exception? _error; + private bool _done; + + public void OnCompleted() + { + lock (_gate) + { + _done = true; + + foreach (var observer in _observers) + { + observer.OnCompleted(); + } + } + } + + public void OnError(Exception error) + { + lock (_gate) + { + _error = error; + + foreach (var observer in _observers) + { + observer.OnError(error); + } + } + } + + public void OnNext((long sequenceId, T item) value) + { + if (value.sequenceId < 0) + throw new ArgumentOutOfRangeException(nameof(value)); + + lock (_gate) + { + // NB: Deduplication of single producer. + + if (!_values.ContainsKey(value.sequenceId)) + { + _values.Add(value.sequenceId, value.item); + + foreach (var observer in _observers) + { + observer.OnNext(value); + } + } + } + } + + public IDisposable Subscribe(IObserver<(long sequenceId, T item)> observer) => Subscribe(observer, null); + + internal IDisposable Subscribe(IReliableObserver observer, long sequenceId) => Subscribe(new Observer(observer), sequenceId); + + private IDisposable Subscribe(IObserver<(long sequenceId, T item)> observer, long? sequenceId) + { + lock (_gate) + { + if (_done) + { + observer.OnCompleted(); + return new Subscription(this, null); + } + + if (_error != null) + { + observer.OnError(_error); + return new Subscription(this, null); + } + + if (sequenceId >= 0) + { + foreach (var item in _values.SkipWhile(x => x.Key < sequenceId)) + { + observer.OnNext((item.Key, item.Value)); + } + } + + _observers.Add(observer); + return new Subscription(this, observer); + } + } + + private sealed class Observer : IObserver<(long sequenceId, T item)> + { + private readonly IReliableObserver _observer; + + public Observer(IReliableObserver observer) => _observer = observer; + + public void OnCompleted() => _observer.OnCompleted(); + + public void OnError(Exception error) => _observer.OnError(error); + + public void OnNext((long sequenceId, T item) value) => _observer.OnNext(value.item, value.sequenceId); + } + + private sealed class Subscription : IDisposable + { + private readonly ReliableSubject _parent; + private IObserver<(long sequenceId, T item)>? _observer; + + public Subscription(ReliableSubject parent, IObserver<(long sequenceId, T item)>? observer) + { + _parent = parent; + _observer = observer; + } + + public void Dispose() + { + var observer = Interlocked.Exchange(ref _observer, null); + + if (observer != null) + { + _parent.Unsubscribe(observer); + } + } + } + + private void Unsubscribe(IObserver<(long sequenceId, T item)> observer) + { + lock (_gate) + { + _observers.Remove(observer); + } + } +} diff --git a/src/Automind.Reaqtor/Llm/ILlmService.cs b/src/Automind.Reaqtor/Llm/ILlmService.cs new file mode 100644 index 0000000..5fb73a2 --- /dev/null +++ b/src/Automind.Reaqtor/Llm/ILlmService.cs @@ -0,0 +1,16 @@ +using Automind.Kernel.Prompting; + +namespace Automind.Reaqtor.Llm; + +/// One generation segment: the text up to the cut (or natural stop) and how it ended. +public sealed record LlmSegmentResult(string Text, bool StoppedAtHedge, bool Truncated = false); + +/// +/// The LLM bridge: renders a to chat messages, streams the completion, +/// and cuts at the balanced hedge close. Lives OUTSIDE the engine (reached via operator context); +/// implementations own transport concerns including transient-error retries. +/// +public interface ILlmService +{ + Task CompleteAsync(PromptState prompt, CancellationToken cancellationToken); +} diff --git a/src/Automind.Reaqtor/Llm/OllamaLlmService.cs b/src/Automind.Reaqtor/Llm/OllamaLlmService.cs new file mode 100644 index 0000000..e0e6874 --- /dev/null +++ b/src/Automind.Reaqtor/Llm/OllamaLlmService.cs @@ -0,0 +1,59 @@ +using Automind.Kernel.Prompting; + +using Microsoft.Extensions.AI; + +namespace Automind.Reaqtor.Llm; + +/// +/// The production LLM bridge: the segment streamer plus transient-transport retries. If the +/// transport ultimately fails, the exception propagates and the derivation simply stays in its +/// awaiting phase — the pending request re-issues on the next recovery, which is the durable +/// behavior we want when the model server is down. +/// +public sealed class OllamaLlmService : ILlmService +{ + private readonly OllamaSegmentStreamer _streamer; + private readonly string _modelId; + private readonly int _maxAttempts; + + public OllamaLlmService(IChatClient chat, string modelId, int maxTransportAttempts = 3) + { + _streamer = new OllamaSegmentStreamer(chat, modelId); + _modelId = modelId; + _maxAttempts = maxTransportAttempts; + } + + public async Task CompleteAsync(PromptState prompt, CancellationToken cancellationToken) + { + using var activity = Telemetry.AutomindDiagnostics.ActivitySource.StartActivity("llm.complete"); + activity?.SetTag("gen_ai.request.model", _modelId); + activity?.SetTag("gen_ai.request.temperature", prompt.Temperature); + activity?.SetTag("gen_ai.request.seed", prompt.Seed); + activity?.SetTag("automind.prefill.chars", prompt.AssistantPrefill.Length); + Telemetry.AutomindDiagnostics.LlmRequests.Add(1); + + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + + for (var attempt = 1; ; attempt++) + { + try + { + var result = await _streamer.StreamSegmentAsync(prompt, cancellationToken).ConfigureAwait(false); + Telemetry.AutomindDiagnostics.LlmSegmentDuration.Record(stopwatch.Elapsed.TotalMilliseconds); + activity?.SetTag("automind.stopped_at_hedge", result.StoppedAtHedge); + activity?.SetTag("automind.segment.chars", result.Text.Length); + activity?.SetTag("automind.transport.attempts", attempt); + return result; + } + catch (Exception ex) when (attempt < _maxAttempts && IsTransient(ex) && !cancellationToken.IsCancellationRequested) + { + await Task.Delay(TimeSpan.FromMilliseconds(500 * attempt), cancellationToken).ConfigureAwait(false); + } + } + } + + private static bool IsTransient(Exception ex) => ex + is HttpRequestException + or IOException + or TaskCanceledException; // HttpClient timeout surfaces as TaskCanceledException +} diff --git a/src/Automind.Reaqtor/Llm/OllamaSegmentStreamer.cs b/src/Automind.Reaqtor/Llm/OllamaSegmentStreamer.cs new file mode 100644 index 0000000..a195857 --- /dev/null +++ b/src/Automind.Reaqtor/Llm/OllamaSegmentStreamer.cs @@ -0,0 +1,176 @@ +using System.Text; + +using Automind.Kernel.Prompting; + +using Microsoft.Extensions.AI; + +using Universalis.Core.Parsing; + +namespace Automind.Reaqtor.Llm; + +/// +/// Streams one generation segment and cuts it at the balanced hedge close — the papers' +/// interception point. Client-side abort (feed every streamed character through the +/// ; cancel the HTTP stream the instant the closing ] arrives) +/// rather than a server-side ] stop sequence, which would false-cut inside JSON array +/// patterns. The ] itself and any tail the model raced ahead with are discarded — the +/// engine owns that bracket. Resumption rides on trailing-assistant-message prefill +/// (verified against Ollama + granite3.3). +/// +public sealed class OllamaSegmentStreamer +{ + // Mode B structured decoding: the paper-shape program under a {"program": …} root (object + // roots decode more reliably than bare arrays across models). The JsonDocument is + // deliberately never disposed — its RootElement backs every request. + private static readonly System.Text.Json.JsonElement s_programSchema = System.Text.Json.JsonDocument.Parse( + """ + { + "type": "object", + "properties": { + "program": { + "type": "array", + "items": { + "type": "object", + "properties": { + "comment": { "type": "string" }, + "expression": { "type": "string" } + } + } + } + }, + "required": ["program"] + } + """).RootElement; + + private readonly IChatClient _chat; + private readonly string _modelId; + + public OllamaSegmentStreamer(IChatClient chat, string modelId) + { + _chat = chat; + _modelId = modelId; + } + + public async Task StreamSegmentAsync(PromptState prompt, CancellationToken cancellationToken) + { + var messages = new List + { + new(ChatRole.System, prompt.SystemPrompt), + new(ChatRole.User, prompt.QuestionText), + }; + + if (prompt.AssistantPrefill.Length > 0) + { + messages.Add(new ChatMessage(ChatRole.Assistant, prompt.AssistantPrefill)); + } + + var options = new ChatOptions + { + ModelId = _modelId, + Temperature = (float)prompt.Temperature, + Seed = prompt.Seed, + MaxOutputTokens = prompt.MaxSegmentTokens, + StopSequences = [.. prompt.HardStopSequences], + }; + + // Mode B: one schema-constrained completion carries the WHOLE program — no hedge + // cutting, no prefill resumption, no client-side abort. STREAMED and accumulated: + // a non-streaming call holds one HTTP request open for the entire generation, and + // grammar-constrained decoding on a cold model blew straight through the client + // timeout (observed live: llm.complete died at ~302 s with no result). + // Reasoning-tuned models (qwen3, deepseek-r1) interleave … chains of + // thought with the answer — suppressed BEFORE the scanner, or hedges the model merely + // considered would execute. + var think = new ThinkFilter(); + + if (prompt.WholeProgram) + { + options.ResponseFormat = ChatResponseFormat.ForJsonSchema( + s_programSchema, "universalis_program", "A literate Universalis program as comment/expression items."); + + var whole = new StringBuilder(); + ChatFinishReason? finish = null; + + await foreach (var update in _chat.GetStreamingResponseAsync(messages, options, cancellationToken).ConfigureAwait(false)) + { + whole.Append(think.Push(update.Text)); + finish = update.FinishReason ?? finish; + } + + whole.Append(think.Flush()); + + // A generation cut at the token cap is NOT malformed JSON — the kernel must know + // the difference to teach truthfully and raise the cap (review finding). + return new LlmSegmentResult(whole.ToString(), StoppedAtHedge: false, Truncated: finish == ChatFinishReason.Length); + } + + var scanner = new HedgeScanner(); + var text = new StringBuilder(); + var stoppedAtHedge = false; + + // One character through scanner + accumulator; true = the hedge closed (stop). + bool ProcessChar(char c) + { + if (scanner.Push(c) == ScanEvent.HedgeClosed) + { + return true; + } + + text.Append(c); + return false; + } + + using var abort = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + + try + { + await foreach (var update in _chat.GetStreamingResponseAsync(messages, options, abort.Token).ConfigureAwait(false)) + { + foreach (var raw in update.Text) + { + foreach (var c in think.Push(raw)) + { + if (ProcessChar(c)) + { + stoppedAtHedge = true; + break; + } + } + + if (stoppedAtHedge) + { + break; + } + } + + if (stoppedAtHedge) + { + abort.Cancel(); // stop the generation; Ollama runs a few tokens past this, harmlessly + break; + } + } + } + catch (OperationCanceledException) when (stoppedAtHedge && !cancellationToken.IsCancellationRequested) + { + // Our own abort-at-hedge racing the stream teardown — expected. Profiler note: this + // teardown also surfaces ~6–8 CAUGHT IOExceptions per segment inside the + // HttpClient/socket stack (measured via dotnet-counters: 190 across a 32-segment + // derivation, CPU cost immaterial). A nonzero IOException counter during generation + // is this mechanism, not a fault. + } + + if (!stoppedAtHedge) + { + foreach (var c in think.Flush()) + { + if (ProcessChar(c)) + { + stoppedAtHedge = true; + break; + } + } + } + + return new LlmSegmentResult(text.ToString(), stoppedAtHedge); + } +} diff --git a/src/Automind.Reaqtor/Llm/ThinkFilter.cs b/src/Automind.Reaqtor/Llm/ThinkFilter.cs new file mode 100644 index 0000000..bd9d798 --- /dev/null +++ b/src/Automind.Reaqtor/Llm/ThinkFilter.cs @@ -0,0 +1,99 @@ +using System.Text; + +namespace Automind.Reaqtor.Llm; + +/// +/// Streaming <think>…</think> suppressor: reasoning-tuned models (qwen3, +/// deepseek-r1) emit their chain of thought BEFORE the answer, and letting it reach the hedge +/// scanner would EXECUTE hedges the model was only thinking about. Suppression is armed only in +/// the LEADING region of the stream (before any non-whitespace content has been emitted): once +/// the answer has begun, a literal '<think>' is ordinary prose — an unmatched mid-answer +/// echo must never swallow the rest of the segment (review finding). Tags can split across +/// streamed chunks, so a holdback buffer carries partial matches; a mismatch flushes it as +/// content. +/// +public sealed class ThinkFilter +{ + private const string OpenTag = ""; + private const string CloseTag = ""; + + private readonly StringBuilder _held = new(); + private bool _suppressing; + private bool _answered; + + public string Push(string chunk) + { + var sb = new StringBuilder(chunk.Length); + + foreach (var c in chunk) + { + sb.Append(Push(c)); + } + + return sb.ToString(); + } + + public string Push(char c) + { + var target = _suppressing ? CloseTag : OpenTag; + + if (_held.Length == 0) + { + if (c == target[0] && (_suppressing || !_answered)) + { + _held.Append(c); + return ""; + } + + return _suppressing ? "" : Emit(c.ToString()); + } + + if (c == target[_held.Length]) + { + _held.Append(c); + + if (_held.Length == target.Length) + { + // Complete tag: flip the mode; the tag itself is never content. + _suppressing = !_suppressing; + _held.Clear(); + } + + return ""; + } + + // Mismatch: what was held is ordinary content (or discarded thought), and the current + // character starts over — it may itself open a new potential tag. + var flushed = _suppressing ? "" : Emit(_held.ToString()); + _held.Clear(); + + return flushed + Push(c); + } + + /// End of stream: a partial OPEN tag was real content; a partial CLOSE was thought. + public string Flush() + { + var tail = _suppressing ? "" : Emit(_held.ToString()); + _held.Clear(); + + return tail; + } + + /// The first non-whitespace content ends the leading region for good. + private string Emit(string s) + { + if (!_answered) + { + foreach (var c in s) + { + if (!char.IsWhiteSpace(c)) + { + _answered = true; + break; + } + } + } + + return s; + } +} diff --git a/src/Automind.Reaqtor/Reactive/ConsoleObserver.cs b/src/Automind.Reaqtor/Reactive/ConsoleObserver.cs new file mode 100644 index 0000000..330cada --- /dev/null +++ b/src/Automind.Reaqtor/Reactive/ConsoleObserver.cs @@ -0,0 +1,16 @@ +// Adapted from the Reaqtor Shebang sample (MIT, .NET Foundation). + +namespace Automind.Reaqtor.Reactive; + +/// +/// Trivial stateless sink writing to the console; handy for diagnostics. Constructed inside +/// expression trees via Expression.New, hence the public parameterless constructor. +/// +public sealed class ConsoleObserver : IObserver +{ + public void OnCompleted() => Console.WriteLine("OnCompleted()"); + + public void OnError(Exception error) => Console.WriteLine($"OnError({error.Message})"); + + public void OnNext(T value) => Console.WriteLine($"OnNext({value})"); +} diff --git a/src/Automind.Reaqtor/Reactive/DerivationObservable.cs b/src/Automind.Reaqtor/Reactive/DerivationObservable.cs new file mode 100644 index 0000000..714ccbb --- /dev/null +++ b/src/Automind.Reaqtor/Reactive/DerivationObservable.cs @@ -0,0 +1,473 @@ +using System.Text.Json; + +using Automind.Kernel.Contract; +using Automind.Kernel.Prompting; +using Automind.Reaqtor.Llm; +using Automind.Tools; + +using Reaqtive; +using Reaqtive.Tasks; + +namespace Automind.Reaqtor.Reactive; + +/// +/// The derivation driver — one standing query per question. The subscription hosts the pure +/// as checkpointed operator state and realizes its effects: +/// +/// +/// LLM/tool effects run as cancellable async work OFF the engine scheduler (a blocked +/// OnNext stalls checkpointing); completions post back through the scheduler as events. +/// Trace/answer effects flow downstream through the ContextSwitchOperator's checkpointed +/// output queue to an egress observer. +/// +/// +/// The recovery invariant: every in-flight external call is re-derivable from state. +/// In-flight work is deliberately not checkpointed — after recovery, +/// re-issues the persisted pending effects with the same request IDs, and the deterministic +/// step function ignores stale completions (an interrupted HTTP call is unrecoverable regardless, +/// so state-directed re-issue is the ceiling of achievable recovery semantics). +/// +public sealed class DerivationObservable : SubscribableBase +{ + private readonly string _derivationId; + private readonly string _questionJson; + + public DerivationObservable(string derivationId, string questionJson) + { + _derivationId = derivationId; + _questionJson = questionJson; + } + + protected override ISubscription SubscribeCore(IObserver observer) => new Subscription(this, observer); + + private sealed class Subscription : ContextSwitchOperator, IUnloadableOperator + { + private static readonly JsonSerializerOptions s_json = new() + { + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + // ---- checkpointed state (Save/Load order matched) ---- + private string _stateJson = ""; + private string _pendingEffectsJson = ""; + private int _outputSeq; + + // ---- runtime-only ---- + private IStepFunction _step = null!; + private IStepContextProvider _stepContext = null!; + private ILlmService _llm = null!; + private IToolRegistry _tools = null!; + private IOperatorContext _context = null!; + private readonly Lock _gate = new(); + private readonly List _inFlight = []; + private bool _finished; + + public Subscription(DerivationObservable parent, IObserver observer) + : base(parent, observer) + { + } + + public override string Name => "am:Derivation"; + + public override Version Version => new(1, 0, 0, 0); + + public override void SetContext(IOperatorContext context) + { + base.SetContext(context); + _context = context; + + _step = Require(context, AutomindServiceKeys.StepFunction); + _stepContext = Require(context, AutomindServiceKeys.StepContext); + _llm = Require(context, AutomindServiceKeys.LlmService); + _tools = Require(context, AutomindServiceKeys.ToolRegistry); + } + + private static T Require(IOperatorContext context, string key) => + context.TryGetElement(key, out var value) + ? value + : throw new InvalidOperationException($"'{key}' service not found in the operator context."); + + protected override void SaveStateCore(IOperatorStateWriter writer) + { + base.SaveStateCore(writer); + + writer.Write(_stateJson); + writer.Write(_pendingEffectsJson); + writer.Write(_outputSeq); + } + + protected override void LoadStateCore(IOperatorStateReader reader) + { + base.LoadStateCore(reader); + + _stateJson = reader.Read(); + _pendingEffectsJson = reader.Read(); + _outputSeq = reader.Read(); + } + + protected override void OnStart() + { + base.OnStart(); // drains the checkpointed output queue (recovered undelivered outputs) + + if (_stateJson.Length == 0) + { + // Fresh subscription: the question itself is durably recorded in the + // subscription's expression parameters. + Post(new QuestionReceived(Params._questionJson)); + return; + } + + var state = DerivationState.FromJson(_stateJson); + var pending = DeserializeEffects(_pendingEffectsJson); + + using var activity = Telemetry.AutomindDiagnostics.ActivitySource.StartActivity("derivation.recover"); + activity?.SetTag("automind.derivation.id", Params._derivationId); + activity?.SetTag("automind.recover.state_bytes", _stateJson.Length); + activity?.SetTag("automind.recover.pending_effects", pending.Count); + activity?.SetTag("automind.recover.phase", state.Phase.GetType().Name); + + if (state.IsTerminal) + { + Post(null); // completion sentinel: finish once the queue has drained + return; + } + + // Recovery: re-issue in-flight external work with the SAME request ids — EXCEPT a + // non-idempotent tool call, whose side effect may already have happened before the + // kill. Re-firing it would double the side effect (review finding: the IsIdempotent + // flag was threaded through the whole chain but consulted nowhere). Synthesizing a + // failure lets the kernel backtrack and the model decide again, per the P3 design. + foreach (var effect in pending) + { + if (effect is InvokeTool invocation && + _tools.Resolve(invocation.ToolUri) is { IsIdempotent: false } sideEffecting) + { + Post(new ToolFailed(invocation.RequestId, + $"'{sideEffecting.Signature.Name}' was in flight when the process died and has side effects, so it was NOT re-issued — check whether it already happened, then decide the next step")); + continue; + } + + Telemetry.AutomindDiagnostics.LlmReissues.Add(1); + ExecuteExternal(effect); + } + } + + // ------------------------------------------------------------ event pump + + /// + /// Marshals an event to the event pump. NB: the scheduler does NOT serialize independent + /// scheduled tasks (Reaqtive's serialization comes from event-flow discipline, not from + /// Schedule itself), so the pump lock below is what makes ProcessEvent the single + /// mutation path — two tool completions landing simultaneously after recovery raced and + /// lost an update without it (observed live). + /// + private void Post(DerivationEvent? evt) => + _context.Scheduler.Schedule(new ActionTask(() => + { + lock (_pump) + { + if (IsDisposed || _finished) + { + return; + } + + if (evt is null) + { + Finish(); + return; + } + + ProcessEvent(evt); + } + })); + + private readonly Lock _pump = new(); + + private void ProcessEvent(DerivationEvent evt) + { + using var activity = Telemetry.AutomindDiagnostics.ActivitySource.StartActivity("derivation.step"); + activity?.SetTag("automind.derivation.id", Params._derivationId); + activity?.SetTag("automind.event", evt.GetType().Name); + Telemetry.AutomindDiagnostics.DerivationSteps.Add(1); + + try + { + var state = _stateJson.Length == 0 + ? DerivationState.New(Params._derivationId) + : DerivationState.FromJson(_stateJson); + + activity?.SetTag("automind.phase.before", state.Phase.GetType().Name); + + var result = _step.Step(state, evt, _stepContext.Current); + + activity?.SetTag("automind.phase.after", result.State.Phase.GetType().Name); + activity?.SetTag("automind.effects", string.Join(",", result.Effects.Select(e => e.GetType().Name))); + + // Pending = the OUTSTANDING external requests, not just this step's new ones: a + // partially-collected lifted batch emits no new effects, but its remaining + // invocations must survive a kill (previous pending ∪ new, filtered by the + // state's outstanding request ids). + var outstanding = result.State.PendingRequestIds; + var pendingEffects = DeserializeEffects(_pendingEffectsJson) + .Concat(result.Effects.Where(e => e is RequestLlm or InvokeTool)) + .Where(e => e switch + { + RequestLlm llm => outstanding.Contains(llm.RequestId), + InvokeTool tool => outstanding.Contains(tool.RequestId), + _ => false, + }) + .DistinctBy(e => e switch + { + RequestLlm llm => llm.RequestId, + InvokeTool tool => tool.RequestId, + _ => "", + }) + .ToList(); + + _stateJson = result.State.ToJson(); + _pendingEffectsJson = SerializeEffects(pendingEffects); + StateChanged = true; + + foreach (var effect in result.Effects) + { + Execute(effect); + } + + if (result.State.IsTerminal) + { + Finish(); + } + } + catch (Exception ex) + { + // A kernel/effect exception must NEVER wedge the standing query silently — + // surface it as a visible failure and terminate the derivation. + _context.TraceSource?.TraceEvent(System.Diagnostics.TraceEventType.Error, 0, + "Automind derivation step failed: {0}", ex); + activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, ex.Message); + Telemetry.AutomindDiagnostics.DerivationFailures.Add(1); + Emit(DerivationOutput.FailedKind, $"internal error while reasoning: {ex.Message}"); + Finish(); + } + } + + private void Finish() + { + _finished = true; + OnCompleted(); + } + + // ------------------------------------------------------------ effects + + private void Execute(DerivationEffect effect) + { + switch (effect) + { + case RequestLlm or InvokeTool: + ExecuteExternal(effect); + break; + + case EmitTrace trace: + // The step span is Current here (Execute runs synchronously inside + // ProcessEvent) — the kernel's logic flow rides on it as span events. + Telemetry.AutomindDiagnostics.RecordTrace(System.Diagnostics.Activity.Current, trace.TraceJson); + Emit(DerivationOutput.TraceKind, trace.TraceJson); + break; + + case EmitAnswer answer: + Telemetry.AutomindDiagnostics.DerivationAnswers.Add(1); + Emit(DerivationOutput.AnswerKind, answer.Text); + break; + + case DefineRule rule: + Emit(DerivationOutput.RuleKind, JsonSerializer.Serialize( + new Dictionary { ["name"] = rule.Name, ["bonsai"] = rule.BonsaiJson }, s_json)); + break; + + case FailedEffect failed: + Telemetry.AutomindDiagnostics.DerivationFailures.Add(1); + Emit(DerivationOutput.FailedKind, failed.Reason); + break; + } + } + + private void Emit(string kind, string payload) + { + StateChanged = true; + OnNext(new DerivationOutput(Params._derivationId, _outputSeq++, kind, payload)); + } + + /// Runs LLM/tool work off the engine scheduler; results post back as events. + private void ExecuteExternal(DerivationEffect effect, int attempt = 0) + { + var cts = new CancellationTokenSource(); + + lock (_gate) + { + _inFlight.Add(cts); + } + + // NB: no `_ =` discard here — inside a ContextSwitchOperator-derived class the + // identifier `_` resolves to the base class's private nested type of that name. + Task.Run(async () => + { + try + { + switch (effect) + { + case RequestLlm request: + { + // Transport retries live inside ILlmService. If it ultimately + // throws, the derivation stays Synthesizing and the pending + // request re-issues on the next recovery — durable by design. + var segment = await _llm.CompleteAsync( + PromptState.FromJson(request.PromptStateJson), cts.Token).ConfigureAwait(false); + + Post(new LlmCompleted(request.RequestId, segment.Text, segment.StoppedAtHedge, segment.Truncated)); + break; + } + + case InvokeTool invocation: + { + // Root span by design: the step span that requested this call is + // long gone by the time the Task.Run body executes. + using var activity = Telemetry.AutomindDiagnostics.ActivitySource.StartActivity("tool.invoke"); + activity?.SetTag("automind.derivation.id", Params._derivationId); + activity?.SetTag("automind.tool.uri", invocation.ToolUri); + activity?.SetTag("automind.tool.route", "derivation"); + activity?.SetTag("automind.request.id", invocation.RequestId); + Telemetry.AutomindDiagnostics.ToolInvocations.Add(1); + + var tool = _tools.Resolve(invocation.ToolUri); + + if (tool is null) + { + activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, "unregistered tool"); + Post(new ToolFailed(invocation.RequestId, $"no tool is registered at '{invocation.ToolUri}'")); + break; + } + + try + { + var results = await tool.InvokeAsync(invocation.ArgsJson, cts.Token).ConfigureAwait(false); + Post(new ToolSucceeded(invocation.RequestId, [.. results])); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, ex.Message); + Post(new ToolFailed(invocation.RequestId, ex.Message)); + } + + break; + } + } + } + catch (OperationCanceledException) + { + // Unload/dispose cancellation — recovery will re-issue if still pending. + } + catch (Exception ex) + { + _context.TraceSource?.TraceEvent(System.Diagnostics.TraceEventType.Error, 0, + "Automind external effect failed: {0}", ex); + + // An LLM request that ultimately failed must not wedge the derivation until + // a process restart (review finding: Ollama down past the transport retries + // froze `ask` forever). The request is still pending in state with its id + // unchanged, so re-issuing here after a capped backoff is exactly what + // recovery would do — the derivation self-heals when the server returns. + if (effect is RequestLlm && !cts.IsCancellationRequested) + { + Telemetry.AutomindDiagnostics.LlmTransportRetries.Add(1); + RetryLater(effect, attempt + 1, TimeSpan.FromSeconds(Math.Min(60, 1 << Math.Min(attempt, 6)))); + } + } + finally + { + lock (_gate) + { + _inFlight.Remove(cts); + } + + cts.Dispose(); + } + }); + } + + private void RetryLater(DerivationEffect effect, int attempt, TimeSpan delay) + { + Task.Run(async () => + { + await Task.Delay(delay).ConfigureAwait(false); + + if (!IsDisposed && !_finished) + { + ExecuteExternal(effect, attempt); + } + }); + } + + // ------------------------------------------------------------ lifecycle + + public void Unload() => CancelInFlight(); + + protected override void OnDispose() + { + base.OnDispose(); + CancelInFlight(); + } + + private void CancelInFlight() + { + List pending; + + lock (_gate) + { + pending = [.. _inFlight]; + _inFlight.Clear(); + } + + foreach (var cts in pending) + { + try + { + cts.Cancel(); + } + catch (ObjectDisposedException) + { + // Raced with normal completion. + } + } + } + + // ------------------------------------------------------------ effect serialization + + private static string SerializeEffects(IEnumerable effects) + { + var list = effects.ToList(); + return list.Count == 0 ? "" : JsonSerializer.Serialize(list, s_json); + } + + private static List DeserializeEffects(string json) => + json.Length == 0 ? [] : JsonSerializer.Deserialize>(json, s_json) ?? []; + } +} + +/// Operator-context keys for the services the derivation driver needs. +public static class AutomindServiceKeys +{ + public const string StepFunction = "Automind.StepFunction"; + public const string StepContext = "Automind.StepContext"; + public const string LlmService = "Automind.LlmService"; + public const string ToolRegistry = "Automind.ToolRegistry"; +} + +/// Provides the current tool/rule snapshot to steps (rehydrated by the host, not checkpointed). +public interface IStepContextProvider +{ + StepContext Current { get; } +} diff --git a/src/Automind.Reaqtor/Reactive/DerivationOutput.cs b/src/Automind.Reaqtor/Reactive/DerivationOutput.cs new file mode 100644 index 0000000..86b5afd --- /dev/null +++ b/src/Automind.Reaqtor/Reactive/DerivationOutput.cs @@ -0,0 +1,43 @@ +using Nuqleon.DataModel; + +namespace Automind.Reaqtor.Reactive; + +/// +/// The single DTO crossing the client↔engine expression boundary: one derivation output event +/// (trace, answer, rule definition, or failure). Deliberately string-shaped — payloads ride as +/// JSON — to keep the Nuqleon data-model surface trivial. Consumers deduplicate redelivered +/// events by (topic, egress sequence id); orders events within a derivation. +/// +public sealed class DerivationOutput +{ + public const string TraceKind = "trace"; + public const string AnswerKind = "answer"; + public const string RuleKind = "rule"; + public const string FailedKind = "failed"; + + public DerivationOutput( + [Mapping("automind://v1/derivationOutput/id")] string derivationId, + [Mapping("automind://v1/derivationOutput/seq")] int seq, + [Mapping("automind://v1/derivationOutput/kind")] string kind, + [Mapping("automind://v1/derivationOutput/payload")] string payloadJson) + { + DerivationId = derivationId; + Seq = seq; + Kind = kind; + PayloadJson = payloadJson; + } + + [Mapping("automind://v1/derivationOutput/id")] + public string DerivationId { get; } + + [Mapping("automind://v1/derivationOutput/seq")] + public int Seq { get; } + + [Mapping("automind://v1/derivationOutput/kind")] + public string Kind { get; } + + [Mapping("automind://v1/derivationOutput/payload")] + public string PayloadJson { get; } + + public override string ToString() => $"{DerivationId}#{Seq} {Kind}"; +} diff --git a/src/Automind.Reaqtor/Reactive/EgressObserver.cs b/src/Automind.Reaqtor/Reactive/EgressObserver.cs new file mode 100644 index 0000000..bb22197 --- /dev/null +++ b/src/Automind.Reaqtor/Reactive/EgressObserver.cs @@ -0,0 +1,70 @@ +// Adapted from the Reaqtor Shebang sample (MIT, .NET Foundation). + +using Reaqtive; + +using Reaqtor.Reliable; +using Reaqtor.Shebang.Service; + +namespace Automind.Reaqtor.Reactive; + +/// +/// Stateful in-engine sink sending events to the outside world through the ingress/egress manager. +/// The persisted sequence counter guarantees stable event numbering across recovery, so downstream +/// consumers can deduplicate redelivered events by (topic, sequenceId). +/// +public sealed class EgressObserver : StatefulObserver +{ + private readonly string _name; + private IReliableObserver? _observer; + private long _sequenceId; + + public EgressObserver(string name) => _name = name; + + public override string Name => "am:Egress"; + + public override Version Version => new(1, 0, 0, 0); + + public override void SetContext(IOperatorContext context) + { + base.SetContext(context); + + if (!context.TryGetElement("IngressEgressManager", out var iemgr)) + { + throw new InvalidOperationException("Ingress/egress manager not found"); + } + + _observer = iemgr.GetObserver(_name); + } + + protected override void OnNextCore(T value) + { + _observer!.OnNext(value, _sequenceId++); + + StateChanged = true; // Mark dirty for differential checkpointing. + } + + protected override void OnErrorCore(Exception error) => _observer!.OnError(error); + + protected override void OnCompletedCore() => _observer!.OnCompleted(); + + protected override void OnStart() + { + base.OnStart(); + + _observer!.OnStarted(); + } + + protected override void SaveStateCore(IOperatorStateWriter writer) + { + base.SaveStateCore(writer); + + writer.Write(_sequenceId); + } + + protected override void LoadStateCore(IOperatorStateReader reader) + { + base.LoadStateCore(reader); + + _sequenceId = reader.Read(); + } +} diff --git a/src/Automind.Reaqtor/Reactive/IngressObservable.cs b/src/Automind.Reaqtor/Reactive/IngressObservable.cs new file mode 100644 index 0000000..821acec --- /dev/null +++ b/src/Automind.Reaqtor/Reactive/IngressObservable.cs @@ -0,0 +1,124 @@ +// Adapted from the Reaqtor Shebang sample (MIT, .NET Foundation). + +using Reaqtive; + +using Reaqtor.Reliable; +using Reaqtor.Shebang.Service; + +namespace Automind.Reaqtor.Reactive; + +/// +/// In-engine source receiving events from the outside world through the ingress/egress manager. +/// Persists the last observed sequence ID; on recovery it calls Start(sequenceId) to replay +/// unacknowledged events, and acknowledges the watermark after each successful checkpoint. +/// +public sealed class IngressObservable : ISubscribable +{ + private readonly string _name; + + public IngressObservable(string name) => _name = name; + + public ISubscription Subscribe(IObserver observer) => new Subscription(this, observer); + + IDisposable IObservable.Subscribe(IObserver observer) => throw new NotSupportedException(); + + private sealed class Subscription : ContextSwitchOperator, T>, IReliableObserver, IUnloadableOperator + { +#pragma warning disable CA2213 // "never disposed." Analyzer hasn't understood OnDispose. + private IReliableSubscription? _subscription; +#pragma warning restore CA2213 + private long _sequenceId; + private long _watermark; + + public Subscription(IngressObservable parent, IObserver observer) + : base(parent, observer) + { + _sequenceId = -1; // NB: Start from the next published event rather than replaying all history (0). + } + + public override string Name => "am:Ingress"; + + public override Version Version => new(1, 0, 0, 0); + + Uri IReliableObserver.ResubscribeUri => throw new NotSupportedException(); + + public override void SetContext(IOperatorContext context) + { + base.SetContext(context); + + if (!context.TryGetElement("IngressEgressManager", out var iemgr)) + { + throw new InvalidOperationException("Ingress/egress manager not found"); + } + + _subscription = iemgr.GetObservable(Params._name).Subscribe(this); + } + + protected override void OnStart() + { + base.OnStart(); + + if (_sequenceId > long.MinValue) + { + _subscription!.Start(_sequenceId); + } + } + + protected override void OnDispose() + { + base.OnDispose(); + + _subscription?.Dispose(); + } + + protected override void SaveStateCore(IOperatorStateWriter writer) + { + base.SaveStateCore(writer); + + writer.Write(_sequenceId); + _watermark = _sequenceId; + } + + public override void OnStateSaved() + { + base.OnStateSaved(); + + _subscription!.AcknowledgeRange(_watermark); + } + + protected override void LoadStateCore(IOperatorStateReader reader) + { + base.LoadStateCore(reader); + + _sequenceId = reader.Read(); + } + + void IReliableObserver.OnNext(T item, long sequenceId) + { + OnNext(item); + + _sequenceId = sequenceId; + StateChanged = true; + } + + void IReliableObserver.OnStarted() { } + + void IReliableObserver.OnError(Exception error) + { + OnError(error); + + _sequenceId = long.MinValue; + StateChanged = true; + } + + void IReliableObserver.OnCompleted() + { + OnCompleted(); + + _sequenceId = long.MinValue; + StateChanged = true; + } + + public void Unload() => _subscription?.Dispose(); + } +} diff --git a/src/Automind.Reaqtor/Reactive/TimerObservable.cs b/src/Automind.Reaqtor/Reactive/TimerObservable.cs new file mode 100644 index 0000000..4fc0332 --- /dev/null +++ b/src/Automind.Reaqtor/Reactive/TimerObservable.cs @@ -0,0 +1,52 @@ +// Adapted from the Reaqtor Shebang sample (MIT, .NET Foundation). + +using Reaqtive; + +namespace Automind.Reaqtor.Reactive; + +/// +/// Stateless periodic source. The subscription derives from +/// so timer callbacks (arbitrary threadpool threads) are marshaled onto the engine's scheduler, +/// which is critical for checkpoint/recovery correctness. +/// +public sealed class TimerObservable : SubscribableBase +{ + private readonly TimeSpan _period; + + public TimerObservable(TimeSpan period) => _period = period; + + protected override ISubscription SubscribeCore(IObserver observer) => new Subscription(_period, observer); + + private sealed class Subscription : ContextSwitchOperator, IUnloadableOperator + { +#pragma warning disable CA2213 // "never disposed." Analyzer hasn't understood OnDispose. + private Timer? _timer; +#pragma warning restore CA2213 + + public Subscription(TimeSpan parent, IObserver observer) : base(parent, observer) + { + } + + public override string Name => "am:Timer"; + + public override Version Version => new(1, 0, 0, 0); + + protected override void OnStart() + { + base.OnStart(); + + _timer = new Timer(Tick, null, 0, (int)Params.TotalMilliseconds); + } + + protected override void OnDispose() + { + base.OnDispose(); + + _timer?.Dispose(); + } + + private void Tick(object? state) => OnNext(DateTimeOffset.Now); + + public void Unload() => _timer?.Dispose(); + } +} diff --git a/src/Automind.Reaqtor/Reactive/ToolInvokeObservable.cs b/src/Automind.Reaqtor/Reactive/ToolInvokeObservable.cs new file mode 100644 index 0000000..d491c5e --- /dev/null +++ b/src/Automind.Reaqtor/Reactive/ToolInvokeObservable.cs @@ -0,0 +1,104 @@ +using Automind.Tools; + +using Reaqtive; + +namespace Automind.Reaqtor.Reactive; + +/// +/// The tool router as an engine artifact: automind://tools/invoke(toolUri, argsJson). +/// Dynamic dispatch over the static artifact space — the tool URI travels as data, so +/// expressions only ever reference the constant router URI. Each result of the tool relation is +/// one OnNext; the async work runs off the engine scheduler and marshals back through the +/// ContextSwitchOperator queue. +/// +public sealed class ToolInvokeObservable : SubscribableBase +{ + private readonly string _toolUri; + private readonly string _argsJson; + + public ToolInvokeObservable(string toolUri, string argsJson) + { + _toolUri = toolUri; + _argsJson = argsJson; + } + + protected override ISubscription SubscribeCore(IObserver observer) => new Subscription(this, observer); + + private sealed class Subscription : ContextSwitchOperator, IUnloadableOperator + { + private IToolRegistry _tools = null!; + private CancellationTokenSource? _cts; + + public Subscription(ToolInvokeObservable parent, IObserver observer) + : base(parent, observer) + { + } + + public override string Name => "am:ToolInvoke"; + + public override Version Version => new(1, 0, 0, 0); + + public override void SetContext(IOperatorContext context) + { + base.SetContext(context); + + if (!context.TryGetElement(AutomindServiceKeys.ToolRegistry, out _tools!)) + { + throw new InvalidOperationException("Tool registry not found in the operator context."); + } + } + + protected override void OnStart() + { + base.OnStart(); + + var tool = _tools.Resolve(Params._toolUri); + + if (tool is null) + { + OnError(new InvalidOperationException($"No tool is registered at '{Params._toolUri}'.")); + return; + } + + _cts = new CancellationTokenSource(); + var token = _cts.Token; + + Task.Run(async () => + { + using var activity = Telemetry.AutomindDiagnostics.ActivitySource.StartActivity("tool.invoke"); + activity?.SetTag("automind.tool.uri", Params._toolUri); + activity?.SetTag("automind.tool.route", "artifact"); + Telemetry.AutomindDiagnostics.ToolInvocations.Add(1); + + try + { + var results = await tool.InvokeAsync(Params._argsJson, token).ConfigureAwait(false); + + foreach (var result in results) + { + OnNext(result); + } + + OnCompleted(); + } + catch (OperationCanceledException) + { + // Unloaded/disposed. + } + catch (Exception ex) + { + activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, ex.Message); + OnError(ex); + } + }); + } + + public void Unload() => _cts?.Cancel(); + + protected override void OnDispose() + { + base.OnDispose(); + _cts?.Cancel(); + } + } +} diff --git a/src/Automind.Reaqtor/Store/FileQueryEngineStateStore.cs b/src/Automind.Reaqtor/Store/FileQueryEngineStateStore.cs new file mode 100644 index 0000000..1da102f --- /dev/null +++ b/src/Automind.Reaqtor/Store/FileQueryEngineStateStore.cs @@ -0,0 +1,594 @@ +// Derived from the Reaqtor Shebang sample's InMemoryKeyValueStore (MIT, .NET Foundation), +// extended with kill-safe file persistence: both commit paths (the engine's transaction log and +// the checkpoint writer) write through to disk atomically before returning, so a process kill at +// any instant leaves either the previous snapshot or the new one — never a torn file. + +using System.Globalization; +using System.Text; +using System.Xml.Linq; + +using Reaqtor.QueryEngine; +using Reaqtor.Shebang.Service; + +namespace Automind.Reaqtor.Store; + +public sealed class FileQueryEngineStateStore : IQueryEngineStateStore +{ + private readonly Dictionary> _data; + private readonly string? _directory; + private readonly Lock _fileGate = new(); + + // Mutation version (bumped under the _data lock) vs the version last written to disk (only + // touched under the file gate). Telemetry-driven: the 5 s checkpoint timer commits an empty + // writer while the engine idles on a long LLM call, and each commit was a FULL snapshot + // rewrite — one observed derivation wrote 12.4 MB across 322 persists for a ~40 KB store. + private long _version; + private long _persistedVersion; + + private FileQueryEngineStateStore(Dictionary> data, string? directory) + { + _data = data; + _directory = directory; + } + + /// Purely in-memory store (tests, throwaway sessions). + public static FileQueryEngineStateStore InMemory() => new([], null); + + /// + /// Opens (or creates) a durable store in . Load order: + /// store.xml, then store.bak (previous good snapshot) on parse failure, + /// then a fresh store. + /// + public static FileQueryEngineStateStore Open(string directory) + { + Directory.CreateDirectory(directory); + + using var activity = Telemetry.AutomindDiagnostics.ActivitySource.StartActivity("store.open"); + var primaryUnreadable = false; + + foreach (var candidate in new[] { SnapshotPath(directory), BackupPath(directory) }) + { + if (!File.Exists(candidate)) + { + continue; + } + + try + { + var store = new FileQueryEngineStateStore(LoadData(candidate), directory); + activity?.SetTag("automind.store.source", candidate == SnapshotPath(directory) ? "snapshot" : "backup"); + activity?.SetTag("automind.store.bytes", new FileInfo(candidate).Length); + + if (primaryUnreadable) + { + // Loading the backup means the primary was torn/unreadable — everything since + // the previous good snapshot is lost. That must never be silent. + activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, + "primary snapshot unreadable — recovered from backup"); + } + + return store; + } + catch (Exception ex) when (ex is System.Xml.XmlException or FormatException or IOException) + { + // Torn or unreadable snapshot — fall through to the next candidate. + primaryUnreadable = true; + } + } + + activity?.SetTag("automind.store.source", "fresh"); + + if (primaryUnreadable) + { + activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, + "no readable snapshot — starting from an empty store"); + } + + return new FileQueryEngineStateStore([], directory); + } + + public bool IsEmpty + { + get + { + lock (_data) + { + return _data.Count == 0; + } + } + } + + private static string SnapshotPath(string directory) => Path.Combine(directory, "store.xml"); + + private static string BackupPath(string directory) => Path.Combine(directory, "store.bak"); + + private static string TempPath(string directory) => Path.Combine(directory, "store.tmp"); + + public IKeyValueStoreTransaction CreateTransaction() => new Transaction(this); + + public IKeyValueTable GetTable(string name) => new Table(name); + + public IStateReader GetReader() => new Reader(this); + + public IStateWriter GetWriter() => new Writer(this); + + // ---------------------------------------------------------------- persistence + + private void PersistSnapshot(int edits) + { + if (_directory is null) + { + return; + } + + // Writers serialize on the file gate — checkpoint commits, WAL commits, and catalog + // saves race from different threads (observed live: two threads colliding on store.tmp). + // The document is built INSIDE the gate so the last writer persists the latest data. + lock (_fileGate) + { + using var activity = Telemetry.AutomindDiagnostics.ActivitySource.StartActivity("store.persist"); + activity?.SetTag("automind.store.edits", edits); + + XDocument doc; + long version; + + lock (_data) + { + version = _version; + + if (version == _persistedVersion) + { + // Nothing changed since the last successful write — an idle-timer checkpoint + // committing an empty writer. Skip the rewrite; keep the span so the skip + // itself stays observable. + activity?.SetTag("automind.store.skipped", true); + return; + } + + doc = new XDocument( + new XElement("Tables", + _data.Select(table => + new XElement("Table", + new XAttribute("Name", table.Key), + table.Value.Select(row => + new XElement("Row", + new XAttribute("Key", row.Key), + new XCData(Convert.ToBase64String(row.Value)))))))); + } + + var temp = TempPath(_directory); + var snapshot = SnapshotPath(_directory); + var backup = BackupPath(_directory); + + long bytes; + + using (var stream = new FileStream(temp, FileMode.Create, FileAccess.Write, FileShare.None)) + { + doc.Save(stream); + stream.Flush(flushToDisk: true); + bytes = stream.Length; + } + + activity?.SetTag("automind.store.bytes", bytes); + Telemetry.AutomindDiagnostics.SnapshotBytes.Record(bytes); + + if (File.Exists(snapshot)) + { + var retries = ReplaceWithRetry(temp, snapshot, backup); + + if (retries > 0) + { + // External contention (antivirus/indexer) — visible, since repeated + // contention degrades checkpoint latency. + activity?.SetTag("automind.store.replace_retries", retries); + } + } + else + { + File.Move(temp, snapshot); + } + + _persistedVersion = version; // this write covered everything up to `version` + } + } + + /// + /// File.Replace is atomic on NTFS but fails transiently when an external process (antivirus, + /// search indexer) briefly holds the snapshot or backup (observed live: "Unable to remove the + /// file to be replaced"). A skipped checkpoint is lost durability — retry with short backoff. + /// Returns the number of retries needed (0 = clean). + /// + private static int ReplaceWithRetry(string source, string destination, string backup) + { + for (var attempt = 0; ; attempt++) + { + try + { + File.Replace(source, destination, backup); // atomic on NTFS + return attempt; + } + catch (IOException) when (attempt < 4) + { + Thread.Sleep(25 << attempt); // 25/50/100/200 ms + } + } + } + + private static Dictionary> LoadData(string fileName) + { + var doc = XDocument.Load(fileName); + + return doc.Element("Tables")!.Elements("Table").ToDictionary( + table => table.Attribute("Name")!.Value, + table => table.Elements("Row").ToDictionary( + row => row.Attribute("Key")!.Value, + row => Convert.FromBase64String(row.Value))); + } + + // ---------------------------------------------------------------- transaction (WAL path) + + private sealed class Transaction : IKeyValueStoreTransaction + { + private readonly FileQueryEngineStateStore _parent; + private readonly Dictionary> _edits = []; + + public Transaction(FileQueryEngineStateStore parent) => _parent = parent; + + public byte[] this[string tableName, string key] + { + get + { + ArgumentNullException.ThrowIfNull(tableName); + ArgumentNullException.ThrowIfNull(key); + + lock (_edits) + { + if (_edits.TryGetValue(tableName, out var table) && table.TryGetValue(key, out var value)) + { + return value ?? throw new System.Collections.Generic.KeyNotFoundException($"'{tableName}'/'{key}' was deleted in this transaction."); + } + } + + lock (_parent._data) + { + if (!_parent._data.TryGetValue(tableName, out var table) || !table.TryGetValue(key, out var value)) + { + throw new System.Collections.Generic.KeyNotFoundException($"'{tableName}'/'{key}' was not found."); + } + + return value; + } + } + } + + public void Add(string tableName, string key, byte[] value) + { + ArgumentNullException.ThrowIfNull(tableName); + ArgumentNullException.ThrowIfNull(key); + ArgumentNullException.ThrowIfNull(value); + + lock (_edits) + { + if (!_edits.TryGetValue(tableName, out var table)) + { + _edits[tableName] = table = []; + } + else if (table.TryGetValue(key, out var existing) && existing is not null) + { + throw new InvalidOperationException("Entry already exists."); + } + + lock (_parent._data) + { + if (_parent._data.TryGetValue(tableName, out var existingTable) && + existingTable.ContainsKey(key) && + (!table.TryGetValue(key, out var tombstone) || tombstone is not null)) + { + throw new InvalidOperationException("Entry already exists."); + } + } + + table[key] = value; + } + } + + public Task CommitAsync(CancellationToken token) + { + lock (_edits) + { + lock (_parent._data) + { + foreach (var table in _edits) + { + if (!_parent._data.TryGetValue(table.Key, out var existingTable)) + { + _parent._data[table.Key] = existingTable = []; + } + + foreach (var entry in table.Value) + { + if (entry.Value is null) + { + existingTable.Remove(entry.Key); + } + else + { + existingTable[entry.Key] = entry.Value; + } + } + } + + if (_edits.Count > 0) + { + _parent._version++; + } + } + } + + _parent.PersistSnapshot(_edits.Sum(t => t.Value.Count)); // durability point: the WAL survives a kill after this line + + return Task.CompletedTask; + } + + public bool Contains(string tableName, string key) + { + ArgumentNullException.ThrowIfNull(tableName); + ArgumentNullException.ThrowIfNull(key); + + lock (_edits) + { + if (_edits.TryGetValue(tableName, out var table) && table.TryGetValue(key, out var value)) + { + return value is not null; + } + } + + lock (_parent._data) + { + return _parent._data.TryGetValue(tableName, out var t) && t.ContainsKey(key); + } + } + + public void Dispose() + { + } + + public IEnumerator> GetEnumerator(string tableName) + { + ArgumentNullException.ThrowIfNull(tableName); + + return Core(); + + IEnumerator> Core() + { + var result = new Dictionary(); + + lock (_edits) + { + if (_edits.TryGetValue(tableName, out var edits)) + { + foreach (var entry in edits) + { + result[entry.Key] = entry.Value; + } + } + } + + lock (_parent._data) + { + if (_parent._data.TryGetValue(tableName, out var table)) + { + foreach (var entry in table) + { + result.TryAdd(entry.Key, entry.Value); + } + } + } + + foreach (var entry in result) + { + if (entry.Value is not null) + { + yield return new KeyValuePair(entry.Key, entry.Value); + } + } + } + } + + public void Remove(string tableName, string key) + { + ArgumentNullException.ThrowIfNull(tableName); + ArgumentNullException.ThrowIfNull(key); + + lock (_edits) + { + if (!_edits.TryGetValue(tableName, out var table)) + { + _edits[tableName] = table = []; + } + + table[key] = null; // tolerant tombstone: deleting an unknown key is a no-op at commit + } + } + + public void Rollback() + { + } + + public void Update(string tableName, string key, byte[] value) + { + ArgumentNullException.ThrowIfNull(tableName); + ArgumentNullException.ThrowIfNull(key); + ArgumentNullException.ThrowIfNull(value); + + lock (_edits) + { + if (!_edits.TryGetValue(tableName, out var table)) + { + _edits[tableName] = table = []; + } + + table[key] = value; + } + } + } + + // ---------------------------------------------------------------- table adapter + + private sealed class Table : IKeyValueTable + { + private readonly string _name; + + public Table(string name) => _name = name; + + public ITransactedKeyValueTable Enter(IKeyValueStoreTransaction transaction) => new Impl(transaction, _name); + + private sealed class Impl : ITransactedKeyValueTable + { + private readonly string _name; + private readonly IKeyValueStoreTransaction _transaction; + + public Impl(IKeyValueStoreTransaction transaction, string name) => (_transaction, _name) = (transaction, name); + + public byte[] this[string key] => _transaction[_name, key]; + + public void Add(string key, byte[] value) => _transaction.Add(_name, key, value); + + public bool Contains(string key) => _transaction.Contains(_name, key); + + public IEnumerator> GetEnumerator() => _transaction.GetEnumerator(_name); + + public void Remove(string key) => _transaction.Remove(_name, key); + + public void Update(string key, byte[] value) => _transaction.Update(_name, key, value); + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + } + + // ---------------------------------------------------------------- checkpoint reader/writer + + private sealed class Reader : IStateReader + { + private readonly FileQueryEngineStateStore _store; + + public Reader(FileQueryEngineStateStore store) => _store = store; + + public void Dispose() + { + } + + public IEnumerable GetCategories() => throw new NotImplementedException("Unused by engine."); + + public bool TryGetItemKeys(string category, out IEnumerable keys) + { + lock (_store._data) + { + if (_store._data.TryGetValue(category, out var table)) + { + keys = [.. table.Keys]; + return true; + } + } + + keys = null!; + return false; + } + + public bool TryGetItemReader(string category, string key, out Stream stream) + { + lock (_store._data) + { + if (_store._data.TryGetValue(category, out var table) && table.TryGetValue(key, out var value)) + { + stream = new MemoryStream(value); + return true; + } + } + + stream = null!; + return false; + } + } + + private sealed class Writer : IStateWriter + { + private readonly FileQueryEngineStateStore _store; + private readonly Dictionary<(string Category, string Key), MemoryStream?> _edits = []; + + public Writer(FileQueryEngineStateStore store) => _store = store; + + public CheckpointKind CheckpointKind => CheckpointKind.Differential; + + public Task CommitAsync(CancellationToken token, IProgress progress) + { + lock (_store._data) + { + foreach (var edit in _edits) + { + var (category, key) = edit.Key; + + if (!_store._data.TryGetValue(category, out var table)) + { + _store._data[category] = table = []; + } + + if (edit.Value is null) + { + table.Remove(key); // tolerant: unknown keys (bridge/tunnel cleanup) are no-ops + } + else + { + table[key] = edit.Value.ToArray(); + } + } + + if (_edits.Count > 0) + { + _store._version++; + } + } + + _store.PersistSnapshot(_edits.Count); // durability point: the checkpoint survives a kill after this line + + return Task.CompletedTask; + } + + public void DeleteItem(string category, string key) => _edits[(category, key)] = null; + + public void Dispose() + { + } + + public Stream GetItemWriter(string category, string key) + { + var stream = new MemoryStream(); + _edits[(category, key)] = stream; + return stream; + } + + public void Rollback() + { + } + } + + // ---------------------------------------------------------------- diagnostics + + public string DebugView + { + get + { + var sb = new StringBuilder(); + + lock (_data) + { + foreach (var table in _data) + { + sb.AppendLine(CultureInfo.InvariantCulture, $"Table '{table.Key}': {table.Value.Count} row(s), {table.Value.Sum(r => r.Value.Length)} bytes"); + } + } + + return sb.ToString(); + } + } +} diff --git a/src/Automind.Reaqtor/Telemetry/AutomindDiagnostics.cs b/src/Automind.Reaqtor/Telemetry/AutomindDiagnostics.cs new file mode 100644 index 0000000..97851c9 --- /dev/null +++ b/src/Automind.Reaqtor/Telemetry/AutomindDiagnostics.cs @@ -0,0 +1,114 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Text.Json; + +namespace Automind.Reaqtor.Telemetry; + +/// +/// Shared telemetry surface. The money metric for the durability story is +/// — a nonzero value after recovery is the substrate visibly +/// re-driving in-flight reasoning. For diagnosing reasoning quality the signals are +/// / / , and the +/// full internal logic flow rides on the derivation.step spans as +/// automind.trace.* events (see ). +/// +public static class AutomindDiagnostics +{ + public const string SourceName = "Automind.Substrate"; + + public static ActivitySource ActivitySource { get; } = new(SourceName); + + public static Meter Meter { get; } = new(SourceName); + + public static Counter DerivationSteps { get; } = + Meter.CreateCounter("automind.derivation.steps", description: "Derivation events processed"); + + public static Counter LlmRequests { get; } = + Meter.CreateCounter("automind.llm.requests", description: "LLM generation segments requested"); + + public static Counter LlmReissues { get; } = + Meter.CreateCounter("automind.llm.reissues", description: "Pending requests re-issued after recovery"); + + public static Counter LlmTransportRetries { get; } = + Meter.CreateCounter("automind.llm.transport_retries", description: "In-process LLM re-issues after ultimate transport failure (self-heal backoff)"); + + public static Counter ToolInvocations { get; } = + Meter.CreateCounter("automind.tool.invocations", description: "Tool invocations"); + + public static Counter Backtracks { get; } = + Meter.CreateCounter("automind.backtracks", description: "Tree-of-thought rewinds"); + + public static Counter Restarts { get; } = + Meter.CreateCounter("automind.restarts", description: "Fresh-start derivation restarts"); + + public static Counter ProtocolRepairs { get; } = + Meter.CreateCounter("automind.repairs", description: "Protocol repairs and tolerated noise"); + + public static Counter DerivationAnswers { get; } = + Meter.CreateCounter("automind.derivation.answers", description: "Answers emitted"); + + public static Counter DerivationFailures { get; } = + Meter.CreateCounter("automind.derivation.failures", description: "Derivations failed terminally"); + + public static Histogram CheckpointDuration { get; } = + Meter.CreateHistogram("automind.checkpoint.duration", unit: "ms", description: "Engine checkpoint durations"); + + public static Histogram SnapshotBytes { get; } = + Meter.CreateHistogram("automind.snapshot.bytes", unit: "By", description: "Durable store snapshot sizes (checkpoint-bloat watch)"); + + public static Histogram LlmSegmentDuration { get; } = + Meter.CreateHistogram("automind.llm.segment.duration", unit: "ms", description: "LLM segment generation durations (wall time dominates here)"); + + /// + /// Bridges one kernel trace event into the ambient derivation.step activity. The + /// kernel is pure (no clocks, no I/O) and cannot host telemetry itself — the substrate + /// observes around it: every trace payload becomes an automind.trace.{kind} + /// ActivityEvent so a span stream reconstructs the full internal logic flow (segments, + /// bindings, guards, backtracks, repairs), and the diagnostic kinds bump counters. + /// + public static void RecordTrace(Activity? activity, string traceJson) + { + var kind = "trace"; + string? reason = null; + + try + { + using var doc = JsonDocument.Parse(traceJson); + + if (doc.RootElement.TryGetProperty("$kind", out var kindProperty)) + { + kind = kindProperty.GetString() ?? kind; + } + + if (doc.RootElement.TryGetProperty("Reason", out var reasonProperty)) + { + reason = reasonProperty.GetString(); + } + } + catch (JsonException) + { + // A malformed trace payload must never break the derivation — log it raw. + } + + switch (kind) + { + case "backtrack": + Backtracks.Add(1); + + if (reason?.StartsWith("fresh start", StringComparison.Ordinal) == true) + { + Restarts.Add(1); + } + + break; + + case "repaired": + ProtocolRepairs.Add(1); + break; + } + + activity?.AddEvent(new ActivityEvent( + $"automind.trace.{kind}", + tags: new ActivityTagsCollection { ["payload"] = traceJson })); + } +} diff --git a/src/Automind.Tools/Automind.Tools.csproj b/src/Automind.Tools/Automind.Tools.csproj new file mode 100644 index 0000000..041a77a --- /dev/null +++ b/src/Automind.Tools/Automind.Tools.csproj @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/Automind.Tools/DemoTools.cs b/src/Automind.Tools/DemoTools.cs new file mode 100644 index 0000000..d93345c --- /dev/null +++ b/src/Automind.Tools/DemoTools.cs @@ -0,0 +1,281 @@ +using System.Globalization; +using System.Text.Json.Nodes; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; + +namespace Automind.Tools; + +/// +/// The primitive facts behind the papers' worked examples, implemented as canned, deterministic, +/// offline services (shaped like the real APIs — including their messy nesting and +/// numbers-as-strings, which is exactly what the pattern matcher exists for). LIST_FILES and +/// TO_PDF touch the real filesystem so the bulk-lifting demo has observable effects. +/// +public static class DemoTools +{ + // ---------------------------------------------------------------- geo / weather.gov chain + + private static readonly Dictionary s_geo = new(StringComparer.OrdinalIgnoreCase) + { + ["Palo Alto"] = (37.4419, -122.1430), + ["Seattle"] = (47.6062, -122.3321), + ["New York"] = (40.7128, -74.0060), + ["Cambridge"] = (52.2053, 0.1218), + ["London"] = (51.5074, -0.1278), + }; + + private static readonly Dictionary s_forecasts = new(StringComparer.OrdinalIgnoreCase) + { + ["Palo Alto"] = "Sunny, with a high near 80. West wind 5 to 10 mph.", + ["Seattle"] = "Overcast with light drizzle and a high near 50.", + ["New York"] = "Snow showers likely, with a high near 30.", + ["Cambridge"] = "Patchy drizzle, with a high near 58.", + ["London"] = "Fog lifting by noon, then cloudy with a high near 52.", + }; + + /// GEO_CODE(city, lat, lon) — the paper's geocoding fact (two outputs). + public static DelegateTool GeoCode() => new( + new PredicateSignature("GEO_CODE", [ + new PredicateParam("city", ParamMode.In), + new PredicateParam("lat", ParamMode.Out), + new PredicateParam("lon", ParamMode.Out), + ], "coordinates of a city"), + "latitude and longitude of a city", + isIdempotent: true, + (argsJson, _) => + { + var city = ReadString(argsJson, "city"); + + if (!s_geo.TryGetValue(city, out var coords)) + { + throw new ArgumentException($"unknown city '{city}' — try one of: {string.Join(", ", s_geo.Keys)}"); + } + + return Result(new JsonObject { ["lat"] = coords.Lat, ["lon"] = coords.Lon }); + }); + + /// WEATHER_GOV(lat, lon, response) — the NWS points API, canned in its real nested shape. + public static DelegateTool WeatherGov() => new( + new PredicateSignature("WEATHER_GOV", [ + new PredicateParam("lat", ParamMode.In), + new PredicateParam("lon", ParamMode.In), + new PredicateParam("response", ParamMode.Out), + ], "National Weather Service point metadata"), + "NWS point lookup; the forecast URL is nested inside \"properties\"", + isIdempotent: true, + (argsJson, _) => + { + var lat = ReadNumber(argsJson, "lat"); + var lon = ReadNumber(argsJson, "lon"); + var city = NearestCity(lat, lon); + + var slug = city.Replace(" ", "", StringComparison.Ordinal).ToLowerInvariant(); + + return Result(new JsonObject + { + ["id"] = $"https://api.weather.gov/points/{lat.ToString("0.####", CultureInfo.InvariantCulture)},{lon.ToString("0.####", CultureInfo.InvariantCulture)}", + ["type"] = "Feature", + ["properties"] = new JsonObject + { + ["gridId"] = slug.ToUpperInvariant()[..Math.Min(3, slug.Length)], + ["gridX"] = 92, + ["gridY"] = 88, + ["forecast"] = $"https://api.weather.gov/gridpoints/{slug}/92,88/forecast", + ["forecastHourly"] = $"https://api.weather.gov/gridpoints/{slug}/92,88/forecast/hourly", + ["timeZone"] = "America/Los_Angeles", + }, + }); + }); + + /// HTTP_GET(url, response) — canned responses for the URLs the other facts hand out. + public static DelegateTool HttpGet() => new( + new PredicateSignature("HTTP_GET", [ + new PredicateParam("url", ParamMode.In), + new PredicateParam("response", ParamMode.Out), + ], "fetch a URL and return its JSON body"), + "HTTP GET returning the response body", + isIdempotent: true, + (argsJson, _) => + { + var url = ReadString(argsJson, "url"); + + foreach (var (city, forecast) in s_forecasts) + { + var slug = city.Replace(" ", "", StringComparison.Ordinal).ToLowerInvariant(); + + if (url.Contains($"/gridpoints/{slug}/", StringComparison.OrdinalIgnoreCase)) + { + return Result(new JsonObject + { + ["properties"] = new JsonObject + { + ["updated"] = "2026-07-16T07:30:00+00:00", + ["periods"] = new JsonArray( + new JsonObject + { + ["number"] = 1, + ["name"] = "Today", + ["temperature"] = 80, + ["temperatureUnit"] = "F", + ["shortForecast"] = forecast.Split(',')[0], + ["detailedForecast"] = forecast, + }), + }, + }); + } + } + + throw new ArgumentException($"no canned response for '{url}'"); + }); + + // ---------------------------------------------------------------- stock / search + + /// STOCK(symbol, data) — the paper's quote API shape: nested, numbers as strings. + public static DelegateTool Stock() => new( + new PredicateSignature("STOCK", [ + new PredicateParam("symbol", ParamMode.In), + new PredicateParam("data", ParamMode.Out), + ], "latest stock quote data"), + "latest quote JSON for a ticker symbol (fields like \"close\" and \"volume\" are nested)", + isIdempotent: true, + (argsJson, _) => + { + var symbol = ReadString(argsJson, "symbol").ToUpperInvariant(); + + var (close, volume) = symbol switch + { + "MSFT" => ("428.90000", "18773400"), + "IBM" => ("181.58000", "3037600"), + _ => ("100.00000", "1000000"), + }; + + return Result(new JsonObject + { + ["data"] = new JsonArray( + new JsonObject + { + ["symbol"] = symbol, + ["exchange"] = "NYSE", + ["currency"] = "USD", + ["datetime"] = "2026-07-16", + ["open"] = close, + ["close"] = close, + ["volume"] = volume, + ["is_market_open"] = false, + }), + ["status"] = "ok", + }); + }); + + /// SEARCH(query, result) — the paper's web-search fact, canned for crypto prices. + public static DelegateTool Search() => new( + new PredicateSignature("SEARCH", [ + new PredicateParam("query", ParamMode.In), + new PredicateParam("result", ParamMode.Out), + ], "web search returning structured data"), + "web search; price-style queries return a JSON object with a \"price\" field", + isIdempotent: true, + (argsJson, _) => + { + var query = ReadString(argsJson, "query"); + + var price = + query.Contains("btc", StringComparison.OrdinalIgnoreCase) || + query.Contains("bitcoin", StringComparison.OrdinalIgnoreCase) ? "43250.75" + : query.Contains("msft", StringComparison.OrdinalIgnoreCase) || + query.Contains("microsoft", StringComparison.OrdinalIgnoreCase) ? "428.90" + : "1.00"; + + return Result(new JsonObject + { + ["query"] = query, + ["price"] = price, + ["currency"] = "USD", + ["source"] = "search", + }); + }); + + // ---------------------------------------------------------------- loopless file conversion + + /// LIST_FILES(directory, files) — real directory listing (full paths). + public static DelegateTool ListFiles() => new( + new PredicateSignature("LIST_FILES", [ + new PredicateParam("directory", ParamMode.In), + new PredicateParam("files", ParamMode.Out, AcceptsCollection: true), + ], "list the files in a directory"), + "all files in a directory, as a LIST of full paths (pass the list straight to a per-file tool)", + isIdempotent: true, + (argsJson, _) => + { + var directory = ReadString(argsJson, "directory"); + var files = Directory.GetFiles(directory).OrderBy(f => f, StringComparer.Ordinal); + + return Result(new JsonArray([.. files.Select(f => JsonValue.Create(f))])); + }); + + /// + /// TO_PDF(src, dst) — converts ONE file (the paper's point: the model calls it on a LIST and + /// the engine's zip lifting fans out the invocations). Writes a stub .pdf next to the source. + /// + public static DelegateTool ToPdf() => new( + new PredicateSignature("TO_PDF", [ + new PredicateParam("src", ParamMode.In), + new PredicateParam("dst", ParamMode.Out), + ], "convert a single file to PDF"), + "converts a file to PDF and returns the new path; call it ONCE with a LIST of files to convert them all (no loop needed)", + isIdempotent: true, + async (argsJson, ct) => + { + var src = ReadString(argsJson, "src"); + + if (src.Contains('*', StringComparison.Ordinal) || src.Contains('?', StringComparison.Ordinal)) + { + throw new ArgumentException( + "wildcards are not supported — LIST_FILES the directory first, then pass the list variable and the engine converts every file"); + } + + if (!File.Exists(src)) + { + throw new FileNotFoundException($"no such file: {src}"); + } + + var dst = Path.ChangeExtension(src, ".pdf"); + await File.WriteAllTextAsync(dst, $"%PDF-1.4 (stub) converted from {Path.GetFileName(src)}\n", ct); + + return [EvalEnv.ToJsonText(JsonValue.Create(dst))]; + }); + + // ---------------------------------------------------------------- helpers + + private static string NearestCity(double lat, double lon) => + s_geo.MinBy(kv => Math.Pow(kv.Value.Lat - lat, 2) + Math.Pow(kv.Value.Lon - lon, 2)).Key; + + private static string ReadString(string argsJson, string name) + { + var args = JsonNode.Parse(argsJson) as JsonObject + ?? throw new ArgumentException("tool arguments must be a JSON object"); + + return args.TryGetPropertyValue(name, out var value) && value is not null + ? value is JsonValue jv && jv.GetValueKind() == System.Text.Json.JsonValueKind.String + ? jv.GetValue() + : value.ToJsonString().Trim('"') + : throw new ArgumentException($"missing tool argument '{name}'"); + } + + private static double ReadNumber(string argsJson, string name) + { + var args = JsonNode.Parse(argsJson) as JsonObject + ?? throw new ArgumentException("tool arguments must be a JSON object"); + + if (!args.TryGetPropertyValue(name, out var value) || !NumericOps.TryCoerce(value, out var num)) + { + throw new ArgumentException($"tool argument '{name}' must be numeric"); + } + + return num.AsDouble; + } + + private static Task> Result(JsonNode value) => + Task.FromResult>([EvalEnv.ToJsonText(value)]); +} diff --git a/src/Automind.Tools/ITool.cs b/src/Automind.Tools/ITool.cs new file mode 100644 index 0000000..82b517b --- /dev/null +++ b/src/Automind.Tools/ITool.cs @@ -0,0 +1,77 @@ +using Universalis.Core.Evaluation; + +namespace Automind.Tools; + +/// +/// A primitive tool — a fact, in the papers' terms: implemented in an imperative language, +/// invoked by the reasoning engine. Tools are relations: an invocation returns 0..n +/// JSON results (fixed-mode execution takes the first alternative). +/// +public interface ITool +{ + PredicateSignature Signature { get; } + + string Description { get; } + + /// Safe to re-invoke after a crash (recovery re-issues in-flight idempotent calls). + bool IsIdempotent { get; } + + /// + /// Invokes the tool. is a JSON object keyed by in-parameter + /// names; each returned string is one JSON result value (single-output tools return the bare + /// value, multi-output tools an object keyed by out-parameter names). + /// + Task> InvokeAsync(string argsJson, CancellationToken cancellationToken); +} + +public interface IToolRegistry +{ + ITool? Resolve(string toolUri); + + IReadOnlyList<(string Uri, ITool Tool)> All { get; } +} + +public sealed class ToolRegistry : IToolRegistry +{ + private readonly Dictionary _tools = new(StringComparer.OrdinalIgnoreCase); + + public ToolRegistry Add(ITool tool) + { + _tools[UriFor(tool.Signature.Name)] = tool; + return this; + } + + public static string UriFor(string toolName) => $"automind://tools/{toolName.ToLowerInvariant()}"; + + public ITool? Resolve(string toolUri) => _tools.GetValueOrDefault(toolUri); + + public IReadOnlyList<(string Uri, ITool Tool)> All => + [.. _tools.OrderBy(kv => kv.Key, StringComparer.Ordinal).Select(kv => (kv.Key, kv.Value))]; +} + +/// Lambda-backed tool — handy for tests and simple facts. +public sealed class DelegateTool : ITool +{ + private readonly Func>> _invoke; + + public DelegateTool( + PredicateSignature signature, + string description, + bool isIdempotent, + Func>> invoke) + { + Signature = signature; + Description = description; + IsIdempotent = isIdempotent; + _invoke = invoke; + } + + public PredicateSignature Signature { get; } + + public string Description { get; } + + public bool IsIdempotent { get; } + + public Task> InvokeAsync(string argsJson, CancellationToken cancellationToken) => + _invoke(argsJson, cancellationToken); +} diff --git a/src/Automind.Tools/PrimitiveTools.cs b/src/Automind.Tools/PrimitiveTools.cs new file mode 100644 index 0000000..123499f --- /dev/null +++ b/src/Automind.Tools/PrimitiveTools.cs @@ -0,0 +1,116 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Nodes; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; +using Universalis.Core.Parsing; + +namespace Automind.Tools; + +/// +/// The v1 primitive toolset — the papers' "facts", implemented imperatively. Deterministic and +/// offline-friendly (WEATHER is a canned service) so demos never depend on external endpoints; +/// TODAY is the sanctioned door through which time enters the pure kernel. +/// +public static class PrimitiveTools +{ + public static ToolRegistry CreateDefault() => new ToolRegistry() + .Add(Weather()) + .Add(Today()) + .Add(Math()); + + /// Canned, deterministic weather relation (Palo Alto honors the paper's value). + public static DelegateTool Weather() + { + var canned = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Palo Alto"] = "Sunny and 80°F", + ["New York"] = "Snowy and 30°F", + ["Miami"] = "Sunny and 100°F", + ["Chicago"] = "Windy and 45°F", + ["Seattle"] = "Overcast and 50°F", + ["Cambridge"] = "Drizzly and 58°F", + ["London"] = "Foggy and 52°F", + }; + + string[] fallbackConditions = ["Sunny", "Cloudy", "Rainy", "Windy", "Overcast"]; + + return new DelegateTool( + new PredicateSignature("WEATHER", [ + new PredicateParam("city", ParamMode.In), + new PredicateParam("weather", ParamMode.Out), + ], "current weather for a city"), + "current weather conditions for a city, as a short description", + isIdempotent: true, + (argsJson, _) => + { + var city = ReadArg(argsJson, "city"); + + if (!canned.TryGetValue(city, out var weather)) + { + // Deterministic pseudo-weather so unknown cities still answer. + var hash = city.Aggregate(17, (h, c) => unchecked(h * 31 + char.ToLowerInvariant(c))); + weather = $"{fallbackConditions[System.Math.Abs(hash) % fallbackConditions.Length]} and {50 + System.Math.Abs(hash) % 45}°F"; + } + + return Result(JsonValue.Create(weather)); + }); + } + + /// The paper's [TODAY(@today)] — time enters the neural computer via a tool, only. + public static DelegateTool Today() => new( + new PredicateSignature("TODAY", [ + new PredicateParam("today", ParamMode.Out), + ], "today's date"), + "today's date in ISO 8601 format (yyyy-MM-dd)", + isIdempotent: true, + (_, _) => Result(JsonValue.Create(DateTime.UtcNow.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)))); + + /// Arithmetic escape hatch: evaluates a numeric expression string. + public static DelegateTool Math() => new( + new PredicateSignature("MATH", [ + new PredicateParam("expression", ParamMode.In), + new PredicateParam("result", ParamMode.Out), + ], "evaluate a numeric expression"), + "evaluates a plain numeric expression like \"(17 - 10) / 10 * 100\"", + isIdempotent: true, + (argsJson, _) => + { + var expression = ReadArg(argsJson, "expression"); + + // Reuse the Universalis arithmetic grammar/evaluator: parse as an `is` binding over + // an empty environment (numeric literals only) and evaluate. + var parsed = HedgeParser.Parse("@r is (" + expression + ")"); + + if (!parsed.Success) + { + throw new ArgumentException($"cannot parse expression '{expression}': {parsed.Error}"); + } + + var outcome = Evaluator.Evaluate( + parsed.Statement!, + EvalEnv.FromSigma(new Dictionary()), + SignatureCatalog.Empty); + + return outcome switch + { + Bound bound => Task.FromResult>([bound.Bindings[0].Json]), + EvalFailure failure => throw new ArgumentException($"cannot evaluate '{expression}': {failure.Message}"), + _ => throw new ArgumentException($"'{expression}' is not a numeric expression"), + }; + }); + + private static string ReadArg(string argsJson, string name) + { + var args = JsonNode.Parse(argsJson) as JsonObject + ?? throw new ArgumentException("tool arguments must be a JSON object"); + + return args.TryGetPropertyValue(name, out var value) && value is JsonValue jv && jv.GetValueKind() == JsonValueKind.String + ? jv.GetValue() + : value?.ToString() ?? throw new ArgumentException($"missing tool argument '{name}'"); + } + + private static Task> Result(JsonNode? value) => + Task.FromResult>([EvalEnv.ToJsonText(value)]); +} diff --git a/src/Universalis.Core/Compilation/BonsaiCompiler.cs b/src/Universalis.Core/Compilation/BonsaiCompiler.cs new file mode 100644 index 0000000..ec42de1 --- /dev/null +++ b/src/Universalis.Core/Compilation/BonsaiCompiler.cs @@ -0,0 +1,343 @@ +using System.Linq.Expressions; +using System.Text.Json; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; +using Universalis.Core.Rendering; + +namespace Universalis.Core.Compilation; + +/// +/// Compiles the intentional representation to a tree over +/// unbound parameters named by URI — the exact normalization idiom Reaqtor uses for its own +/// operators (rx://… as free variables, bound by a registry at evaluation time). The tree +/// serializes to Bonsai JSON via Nuqleon: THE durable, language-agnostic rule artifact. The +/// canonical IR JSON rides in the rule metadata constant, so one document carries both the +/// machine tree and the executable form (v1 interprets the IR; the tree is storage/interop). +/// +/// Encoding: values are object-typed; statements chain as let-bindings via beta-redexes +/// (Invoke(Lambda(x ⇒ rest), value)); the body ends in +/// universalis://result(object[] outs). +/// +public static class BonsaiCompiler +{ + public static class Uris + { + public const string Rule = "universalis://rule/v1"; + public const string Result = "universalis://result"; + public const string Display = "universalis://display"; + public const string MatchExtract = "universalis://match/extract"; + public const string QueryRun = "universalis://query/run"; + + public static string Arith(ArithOp op) => "universalis://is/" + op switch + { + ArithOp.Add => "add", + ArithOp.Sub => "sub", + ArithOp.Mul => "mul", + ArithOp.Div => "div", + _ => "mod", + }; + + public const string Negate = "universalis://is/neg"; + + public static string Compare(CompareOp op) => "universalis://cmp/" + op switch + { + CompareOp.Eq => "eq", + CompareOp.Neq => "neq", + CompareOp.Lt => "lt", + CompareOp.Le => "le", + CompareOp.Gt => "gt", + _ => "ge", + }; + + public static string Tool(string name) => "tool://" + name; + } + + public static Expression Compile(RuleDefinition rule) + { + var scope = new Dictionary(StringComparer.Ordinal); + + var inParams = rule.Signature.Params + .Where(p => p.Mode == ParamMode.In) + .Select(p => + { + var parameter = Expression.Parameter(typeof(object), p.Name); + scope[p.Name] = parameter; + return parameter; + }) + .ToList(); + + var outNames = rule.Signature.Params + .Where(p => p.Mode == ParamMode.Out) + .Select(p => p.Name) + .ToList(); + + var executable = Flatten(rule.Body.Items).ToList(); + var chain = CompileChain(executable, 0, scope, outNames); + var body = Expression.Lambda(chain, inParams); + + var ruleOp = Expression.Parameter( + typeof(Func<,,>).MakeGenericType(typeof(string), body.Type, typeof(object)), + Uris.Rule); + + return Expression.Invoke(ruleOp, Expression.Constant(IrJson.Serialize(rule)), body); + } + + /// Executable statements in order; comments are IR-only, blocks stay structural. + private static IEnumerable Flatten(IEnumerable items) => + items.Where(i => i is not Comment); + + private static Expression CompileChain( + List items, + int index, + Dictionary scope, + List outNames) + { + if (index >= items.Count) + { + var outs = outNames + .Select(name => scope.TryGetValue(name, out var p) ? (Expression)p : Expression.Constant(null, typeof(object))) + .ToList(); + + return Expression.Invoke( + Operation(Uris.Result, 1), + Expression.NewArrayInit(typeof(object), outs)); + } + + switch (items[index]) + { + case HedgeItem { Statement: IsBinding isb }: + return Let(isb.Var, CompileArith(isb.Expr, scope), items, index, scope, outNames); + + case HedgeItem { Statement: BindStmt { Left: VarTerm v } bind }: + return Let(v.Name, CompileTerm(bind.Right, scope), items, index, scope, outNames); + + case HedgeItem { Statement: DisplayStmt display }: + return Let( + "__display" + index, + Expression.Invoke( + Operation(Uris.Display, 2), + Expression.Constant(ConcreteRenderer.RenderTerm(display.Value, RenderMode.Formulas, null)), + CompileTerm(display.Value, scope)), + items, index, scope, outNames); + + case HedgeItem { Statement: Comparison cmp }: + return Let( + "__assert" + index, + Expression.Invoke( + Operation(Uris.Compare(cmp.Op), 2), + CompileTerm(cmp.Left, scope), + CompileTerm(cmp.Right, scope)), + items, index, scope, outNames); + + case HedgeItem { Statement: PredicateCall call }: + return CompileCall(call, items, index, scope, outNames); + + case ConditionalBlock conditional: + return CompileConditional(conditional, items, index, scope, outNames); + + case ComprehensionBlock comprehension: + { + var source = scope.TryGetValue(comprehension.SourceVar, out var p) + ? (Expression)p + : Expression.Constant(null, typeof(object)); + + var pipeline = IrJson.Options is var _ + ? JsonSerializer.Serialize(comprehension, IrJson.Options) + : ""; + + return Let( + comprehension.IntoVar ?? "__query" + index, + Expression.Invoke(Operation(Uris.QueryRun, 2), Expression.Constant(pipeline), source), + items, index, scope, outNames); + } + + default: + return CompileChain(items, index + 1, scope, outNames); + } + } + + private static Expression CompileCall( + PredicateCall call, + List items, + int index, + Dictionary scope, + List outNames) + { + // Fixed-mode convention: leading args are inputs; trailing fresh-variable/pattern args + // are outputs. Without a catalog at compile time, classify by shape: a VarTerm not in + // scope, or a pattern, is an output. + var inArgs = new List(); + var outTerms = new List(); + + foreach (var arg in call.Args) + { + if ((arg is VarTerm v && !scope.ContainsKey(v.Name)) || arg is ObjectPatternTerm or ArrayPatternTerm) + { + outTerms.Add(arg); + } + else + { + inArgs.Add(CompileTerm(arg, scope)); + } + } + + var invocation = Expression.Invoke(Operation(Uris.Tool(call.Name), inArgs.Count), inArgs); + + var resultParameter = Expression.Parameter(typeof(object), "__call" + index); + var innerScope = new Dictionary(scope, StringComparer.Ordinal); + + // Each output variable extracts from the call result. + Expression Extract(List<(string Name, Term Pattern)> pending, int k) + { + if (k >= pending.Count) + { + return CompileChain(items, index + 1, innerScope, outNames); + } + + var (name, pattern) = pending[k]; + var parameter = Expression.Parameter(typeof(object), name); + innerScope[name] = parameter; + + var patternJson = pattern is VarTerm + ? "" + : JsonSerializer.Serialize(pattern, IrJson.Options); + + return Expression.Invoke( + Expression.Lambda(Extract(pending, k + 1), parameter), + Expression.Invoke( + Operation(Uris.MatchExtract, 3), + resultParameter, + Expression.Constant(patternJson), + Expression.Constant(name))); + } + + var extractions = outTerms + .SelectMany(t => t switch + { + VarTerm v => [(v.Name, (Term)v)], + _ => CollectPatternVars(t).Select(n => (n, t)), + }) + .ToList(); + + return Expression.Invoke(Expression.Lambda(Extract(extractions, 0), resultParameter), invocation); + } + + private static IEnumerable CollectPatternVars(Term term) => term switch + { + VarTerm v => [v.Name], + ObjectPatternTerm o => o.Fields.SelectMany(f => CollectPatternVars(f.Value)), + ArrayPatternTerm a => a.Items.SelectMany(CollectPatternVars), + _ => [], + }; + + private static Expression CompileConditional( + ConditionalBlock block, + List items, + int index, + Dictionary scope, + List outNames) + { + // Nested if/elif: each branch's body chains into the SHARED continuation (branch-local + // lets stay inside their arm). + return BuildBranch(0); + + Expression BuildBranch(int branch) + { + if (branch >= block.Branches.Length) + { + return CompileChain(items, index + 1, scope, outNames); + } + + var b = block.Branches[branch]; + var branchScope = new Dictionary(scope, StringComparer.Ordinal); + var bodyItems = Flatten(b.Body).Concat(items.Skip(index + 1)).ToList(); + var bodyChain = CompileChain(bodyItems, 0, branchScope, outNames); + + if (b.Guard is null) + { + return bodyChain; + } + + var guard = b.Guard switch + { + Comparison cmp => Expression.Invoke( + Operation(Uris.Compare(cmp.Op), 2), + CompileTerm(cmp.Left, scope), + CompileTerm(cmp.Right, scope)), + BindStmt bind => Expression.Invoke( + Operation(Uris.Compare(CompareOp.Eq), 2), + CompileTerm(bind.Left, scope), + CompileTerm(bind.Right, scope)), + _ => (Expression)Expression.Constant(false, typeof(object)), + }; + + return Expression.Condition( + Expression.Convert(guard, typeof(bool)), + bodyChain, + BuildBranch(branch + 1), + typeof(object)); + } + } + + private static Expression Let( + string name, + Expression value, + List items, + int index, + Dictionary scope, + List outNames) + { + var parameter = Expression.Parameter(typeof(object), name); + var innerScope = new Dictionary(scope, StringComparer.Ordinal) + { + [name] = parameter, + }; + + var rest = CompileChain(items, index + 1, innerScope, outNames); + + return Expression.Invoke(Expression.Lambda(rest, parameter), value); + } + + private static Expression CompileArith(ArithExpr expr, Dictionary scope) => expr switch + { + ArithNum n => Expression.Constant(NumericOps.CanonicalText(n.Value), typeof(string)), + ArithVar v => Reference(v.Name, scope), + ArithNeg neg => Expression.Invoke(Operation(Uris.Negate, 1), CompileArith(neg.Operand, scope)), + ArithBinary bin => Expression.Invoke( + Operation(Uris.Arith(bin.Op), 2), + CompileArith(bin.Left, scope), + CompileArith(bin.Right, scope)), + _ => throw new NotSupportedException(expr.GetType().Name), + }; + + private static Expression CompileTerm(Term term, Dictionary scope) => term switch + { + VarTerm v => Reference(v.Name, scope), + StrTerm s => Expression.Constant(s.Value, typeof(string)), + NumTerm n => Expression.Constant(NumericOps.CanonicalText(n.Value), typeof(string)), + BoolTerm b => Expression.Constant(b.Value ? "true" : "false", typeof(string)), + NullTerm => Expression.Constant("null", typeof(string)), + ExprTerm e => CompileArith(e.Expr, scope), + _ => Expression.Constant(JsonSerializer.Serialize(term, IrJson.Options), typeof(string)), + }; + + private static Expression Reference(string name, Dictionary scope) => + scope.TryGetValue(name, out var parameter) + ? parameter + : Expression.Parameter(typeof(object), name); // free variable — deliberately unbound + + /// An unbound operation parameter: Func<object…, object> named by URI. + private static ParameterExpression Operation(string uri, int arity) + { + var type = arity switch + { + 1 => typeof(Func), + 2 => typeof(Func), + 3 => typeof(Func), + _ => typeof(Func), + }; + + return Expression.Parameter(type, uri); + } +} diff --git a/src/Universalis.Core/Compilation/BonsaiSerialization.cs b/src/Universalis.Core/Compilation/BonsaiSerialization.cs new file mode 100644 index 0000000..2ab5068 --- /dev/null +++ b/src/Universalis.Core/Compilation/BonsaiSerialization.cs @@ -0,0 +1,38 @@ +using System.Linq.Expressions; +using System.Linq.Expressions.Bonsai.Serialization; + +namespace Universalis.Core.Compilation; + +/// +/// Expression tree ⇄ Bonsai JSON via Nuqleon — the durable wire format for compiled rules. +/// Constants in our encoding are strings only (canonical JSON text / IR JSON), so the lift and +/// reduce object serializers are identities. +/// +public static class BonsaiSerialization +{ + public static string ToBonsaiJson(Expression expression) + { + var slim = expression.ToExpressionSlim(); + return CreateSerializer().Serialize(slim).ToString(); + } + + public static Expression FromBonsaiJson(string bonsaiJson) + { + var parsed = Nuqleon.Json.Expressions.Expression.Parse(bonsaiJson); + var slim = CreateSerializer().Deserialize(parsed); + return slim.ToExpression(); + } + + // Lift turns a constant VALUE into JSON text (the serializer parses it into the document); + // reduce receives the document's Json.Expression node back. All our constants are CLR + // strings by construction. + private static ExpressionSlimBonsaiSerializer CreateSerializer() => new( + liftFactory: _ => static value => System.Text.Json.JsonSerializer.Serialize((string)value!), + reduceFactory: _ => static value => value switch + { + Nuqleon.Json.Expressions.Expression node => System.Text.Json.JsonSerializer.Deserialize(node.ToString())!, + string text => System.Text.Json.JsonSerializer.Deserialize(text)!, + _ => value!, + }, + BonsaiVersion.Default); +} diff --git a/src/Universalis.Core/Evaluation/EvalEnv.cs b/src/Universalis.Core/Evaluation/EvalEnv.cs new file mode 100644 index 0000000..107c907 --- /dev/null +++ b/src/Universalis.Core/Evaluation/EvalEnv.cs @@ -0,0 +1,76 @@ +using System.Collections.Immutable; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Universalis.Core.Evaluation; + +/// +/// Read-only view over the environment σ (plus optional inner scopes for rule frames and +/// comprehension rows). Values are canonical JSON text — the checkpointed representation — +/// with a lazily built, non-serialized memo for evaluation speed. +/// Resolution order: innermost scope first. +/// +public sealed class EvalEnv +{ + private readonly ImmutableArray> _scopes; // innermost first + private readonly Dictionary _memo = new(StringComparer.Ordinal); + private readonly HashSet _memoized = new(StringComparer.Ordinal); + + private EvalEnv(ImmutableArray> scopes) => _scopes = scopes; + + public static EvalEnv FromSigma(IReadOnlyDictionary sigma) => new([sigma]); + + /// Pushes an inner scope (rule frame locals, comprehension row) in front of σ. + public EvalEnv Push(IReadOnlyDictionary scope) => new(_scopes.Insert(0, scope)); + + public bool IsBound(string name) => TryGetText(name, out _); + + public bool TryGetText(string name, out string json) + { + foreach (var scope in _scopes) + { + if (scope.TryGetValue(name, out json!)) + { + return true; + } + } + + json = null!; + return false; + } + + /// + /// Parsed node for a bound variable. A bound JSON null yields a null node — use + /// to distinguish unbound from bound-to-null. + /// + public JsonNode? GetNode(string name) + { + if (_memoized.Contains(name)) + { + return _memo[name]; + } + + if (!TryGetText(name, out var json)) + { + throw new KeyNotFoundException($"variable '@{name}' is not bound"); + } + + var node = JsonNode.Parse(json); + _memo[name] = node; + _memoized.Add(name); + return node; + } + + /// + /// Serializer options for canonical σ text: relaxed escaping keeps non-ASCII (°F, émojis) + /// readable — σ values surface in displays, traces, and answers. + /// + public static JsonSerializerOptions JsonTextOptions { get; } = new() + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + /// Canonical JSON text for a node (a null node is JSON null). + public static string ToJsonText(JsonNode? node) => node?.ToJsonString(JsonTextOptions) ?? "null"; +} diff --git a/src/Universalis.Core/Evaluation/EvalTypes.cs b/src/Universalis.Core/Evaluation/EvalTypes.cs new file mode 100644 index 0000000..41099f1 --- /dev/null +++ b/src/Universalis.Core/Evaluation/EvalTypes.cs @@ -0,0 +1,125 @@ +using System.Collections.Immutable; + +using Universalis.Core.Ir; + +namespace Universalis.Core.Evaluation; + +/// A single variable binding produced by evaluation; the value is canonical JSON text. +public sealed record Binding(string Var, string Json); + +/// +/// The result of evaluating one statement against an environment. Pure data — external work +/// (tool/rule invocation) surfaces as / requests +/// for the kernel to realize as effects. +/// +public abstract record EvalOutcome; + +/// New σ bindings (from is, =, pattern matches, or tool-result binding). +public sealed record Bound(ImmutableArray Bindings) : EvalOutcome; + +/// +/// A tool must run. carries one argument payload per invocation (one for a +/// scalar call; n for a zip-lifted call). Results bind via . +/// +public sealed record NeedTool( + PredicateSignature Signature, + LiftPlan Plan, + ImmutableArray OutArgs) : EvalOutcome; + +/// A stored rule must run (milestone P7). +public sealed record NeedRule( + PredicateSignature Signature, + ImmutableArray InBindings, + ImmutableArray OutArgs) : EvalOutcome; + +/// A ground truth-value (comparisons, ground = tests, conditional guards). +public sealed record GuardResult(bool Value) : EvalOutcome; + +/// A display expression's value: shown to the user (⇝), never to the model. +public sealed record DisplayValue(string Json, string Formatted) : EvalOutcome; + +/// +/// A missing derivation — the tree-of-thought backtracking signal. carries +/// actionable, value-sanitized feedback for the model (available keys, expected arity, ...). +/// +public sealed record EvalFailure(string Code, string Message, string? Hint = null) : EvalOutcome; + +/// Well-known values. +public static class EvalFailureCodes +{ + public const string Unbound = "unbound"; + public const string Rebind = "rebind"; + public const string Mode = "mode"; + public const string Type = "type"; + public const string DivideByZero = "divide-by-zero"; + public const string MatchFailed = "match-failed"; + public const string LiftLength = "lift-length"; + public const string LiftNested = "lift-nested"; + public const string UnknownPredicate = "unknown-predicate"; + public const string Arity = "arity"; + public const string LiteralInOutPosition = "literal-in-out-position"; + public const string PatternInvalid = "pattern-invalid"; + public const string ToolResult = "tool-result"; +} + +/// +/// Execution plan for a (possibly zip-lifted) tool call: one JSON-object argument payload per +/// invocation, keyed by in-parameter name. Invocations.Length == 1 for a plain scalar call. +/// +public sealed record LiftPlan(ImmutableArray InvocationArgsJson) +{ + public int Count => InvocationArgsJson.Length; + + public bool IsLifted => Count > 1; +} + +/// Parameter of a tool or rule; fixed-mode: values must be ground at the call site. +public sealed record PredicateParam(string Name, ParamMode Mode, bool AcceptsCollection = false); + +/// Signature of an invocable predicate (primitive tool or stored rule). +public sealed record PredicateSignature( + string Name, + ImmutableArray Params, + string Description, + bool IsRule = false) +{ + public ImmutableArray InParams => [.. Params.Where(p => p.Mode == ParamMode.In)]; + + public ImmutableArray OutParams => [.. Params.Where(p => p.Mode == ParamMode.Out)]; +} + +/// Resolves predicate names to signatures; rules shadow tools of the same name. +public interface ISignatureCatalog +{ + bool TryGet(string name, out PredicateSignature signature); + + IReadOnlyCollection KnownNames { get; } +} + +/// +/// Immutable dictionary-backed catalog. Later entries win on name collisions — the papers' +/// self-learning story requires learned RULES to shadow primitive tools of the same name +/// (callers list tools first, rules second). +/// +public sealed class SignatureCatalog : ISignatureCatalog +{ + private readonly ImmutableDictionary _signatures; + + public SignatureCatalog(IEnumerable signatures) + { + var builder = ImmutableDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); + + foreach (var signature in signatures) + { + builder[signature.Name] = signature; // last wins: rules shadow tools + } + + _signatures = builder.ToImmutable(); + } + + public static SignatureCatalog Empty { get; } = new([]); + + public bool TryGet(string name, out PredicateSignature signature) => _signatures.TryGetValue(name, out signature!); + + public IReadOnlyCollection KnownNames => [.. _signatures.Keys.Order(StringComparer.OrdinalIgnoreCase)]; +} diff --git a/src/Universalis.Core/Evaluation/Evaluator.cs b/src/Universalis.Core/Evaluation/Evaluator.cs new file mode 100644 index 0000000..669dc20 --- /dev/null +++ b/src/Universalis.Core/Evaluation/Evaluator.cs @@ -0,0 +1,933 @@ +using System.Collections.Immutable; +using System.Text.Json; +using System.Text.Json.Nodes; + +using Universalis.Core.Ir; + +namespace Universalis.Core.Evaluation; + +/// +/// The fixed-mode statement evaluator: inputs must be ground, dataflow is left-to-right, matching +/// is one-way. Pure — external work surfaces as /. +/// Implicit lifting is zip: a scalar-declared position receiving an array broadcasts +/// elementwise (the paper's toPdf(@files, @dst) ≡ map(toPDF, @files, @dst)); +/// scalars broadcast; length mismatch fails; out-binding is structure-of-arrays. +/// +public static class Evaluator +{ + public static EvalOutcome Evaluate(Statement statement, EvalEnv env, ISignatureCatalog catalog) => statement switch + { + IsBinding isb => EvaluateIs(isb, env), + Comparison cmp => EvaluateComparison(cmp, env), + BindStmt bind => EvaluateBind(bind, env), + DisplayStmt disp => EvaluateDisplay(disp, env), + PredicateCall call => EvaluateCall(call, env, catalog), + _ => new EvalFailure(EvalFailureCodes.Type, $"unsupported statement '{statement.GetType().Name}'"), + }; + + // ---------------------------------------------------------------- is / arithmetic + + private static EvalOutcome EvaluateIs(IsBinding statement, EvalEnv env) + { + if (env.IsBound(statement.Var)) + { + // Re-deriving a known fact with the SAME value is narration, not rebinding — models + // re-assert bindings given in the question (observed live). Only a conflict is an error. + var redo = EvalArith(statement.Expr, env); + + if (redo.Failure is null && NumericOps.StructuralEquals(env.GetNode(statement.Var), redo.Node)) + { + return new Bound([]); + } + + // Rebinding an OBJECT-holding variable is the "make the tool result be the number" + // instinct (observed live: @pricePerShare held the SEARCH blob and the model kept + // guessing prices INTO it). The engine knows the real keys — teach the extraction. + if (env.GetNode(statement.Var) is System.Text.Json.Nodes.JsonObject obj) + { + var keys = string.Join(", ", obj.Select(kv => kv.Key).Take(6)); + + return new EvalFailure( + EvalFailureCodes.Rebind, + $"variable '@{statement.Var}' is already bound", + $"'@{statement.Var}' holds an object with fields {keys} — extract one into a FRESH variable by matching ⟨@{statement.Var} = {{ … \"fieldName\": @freshName … }}⟩ and use the fresh variable"); + } + + return new EvalFailure( + EvalFailureCodes.Rebind, + $"variable '@{statement.Var}' is already bound", + "invent a fresh variable name instead of reusing one"); + } + + var result = EvalArith(statement.Expr, env); + + return result.Failure is not null + ? result.Failure + : new Bound([new Binding(statement.Var, EvalEnv.ToJsonText(result.Node))]); + } + + private readonly record struct ArithResult(JsonNode? Node, EvalFailure? Failure) + { + public static ArithResult Ok(JsonNode? node) => new(node, null); + + public static ArithResult Fail(EvalFailure failure) => new(null, failure); + } + + /// Scalar or vector of numbers — the lifted arithmetic value domain. + private readonly record struct NumValue(Num Scalar, ImmutableArray Vector, bool IsVector); + + private static ArithResult EvalArith(ArithExpr expr, EvalEnv env) + { + var (value, failure) = EvalNum(expr, env); + + if (failure is not null) + { + return ArithResult.Fail(failure); + } + + if (!value.IsVector) + { + return ArithResult.Ok(value.Scalar.ToJson()); + } + + var array = new JsonArray(); + foreach (var n in value.Vector) + { + array.Add(n.ToJson()); + } + + return ArithResult.Ok(array); + } + + private static (NumValue Value, EvalFailure? Failure) EvalNum(ArithExpr expr, EvalEnv env) + { + switch (expr) + { + case ArithNum n: + return (new NumValue(Num.FromDecimal(n.Value), [], false), null); + + case ArithVar v: + { + if (!env.IsBound(v.Name)) + { + return (default, new EvalFailure( + EvalFailureCodes.Unbound, + $"variable '@{v.Name}' is not bound", + "inputs must be fully computed before use (fixed mode)")); + } + + var node = env.GetNode(v.Name); + + if (node is JsonArray arr) + { + var items = ImmutableArray.CreateBuilder(arr.Count); + + for (var i = 0; i < arr.Count; i++) + { + if (arr[i] is JsonArray) + { + return (default, new EvalFailure( + EvalFailureCodes.LiftNested, + $"'@{v.Name}' is an array of arrays; nested lifting is not supported")); + } + + if (!NumericOps.TryCoerce(arr[i], out var element)) + { + return (default, new EvalFailure( + EvalFailureCodes.Type, + $"element {i} of '@{v.Name}' is not numeric")); + } + + items.Add(element); + } + + return (new NumValue(default, items.ToImmutable(), true), null); + } + + if (!NumericOps.TryCoerce(node, out var num)) + { + // A JSON-object operand is the "bound the whole tool result" instinct + // (observed live): teach the pattern-extraction step with the REAL keys. + var kind = NumericOps.GetKind(node); + var hint = (string?)null; + + if (node is System.Text.Json.Nodes.JsonObject obj) + { + var keys = string.Join(", ", obj.Select(kv => kv.Key).Take(6)); + hint = $"'@{v.Name}' holds an object with fields {keys} — extract one into a FRESH variable by matching [@{v.Name} = {{ … \"fieldName\": @number … }}] and use the fresh variable"; + } + + return (default, new EvalFailure( + EvalFailureCodes.Type, + $"'@{v.Name}' is not numeric ({kind.ToString().ToLowerInvariant()})", + hint)); + } + + return (new NumValue(num, [], false), null); + } + + case ArithNeg neg: + { + var (operand, failure) = EvalNum(neg.Operand, env); + if (failure is not null) + { + return (default, failure); + } + + return operand.IsVector + ? (new NumValue(default, [.. operand.Vector.Select(n => n.Negate())], true), null) + : (new NumValue(operand.Scalar.Negate(), [], false), null); + } + + case ArithBinary bin: + { + var (left, lf) = EvalNum(bin.Left, env); + if (lf is not null) + { + return (default, lf); + } + + var (right, rf) = EvalNum(bin.Right, env); + if (rf is not null) + { + return (default, rf); + } + + try + { + return ApplyBinary(bin.Op, left, right); + } + catch (DivideByZeroException) + { + return (default, new EvalFailure(EvalFailureCodes.DivideByZero, "division by zero")); + } + } + + default: + return (default, new EvalFailure(EvalFailureCodes.Type, $"unsupported arithmetic node '{expr.GetType().Name}'")); + } + } + + private static (NumValue Value, EvalFailure? Failure) ApplyBinary(ArithOp op, NumValue left, NumValue right) + { + Num Apply(Num a, Num b) => op switch + { + ArithOp.Add => a.Add(b), + ArithOp.Sub => a.Sub(b), + ArithOp.Mul => a.Mul(b), + ArithOp.Div => a.Div(b), + _ => a.Mod(b), + }; + + if (!left.IsVector && !right.IsVector) + { + return (new NumValue(Apply(left.Scalar, right.Scalar), [], false), null); + } + + if (left.IsVector && right.IsVector) + { + if (left.Vector.Length != right.Vector.Length) + { + return (default, new EvalFailure( + EvalFailureCodes.LiftLength, + $"arrays of length {left.Vector.Length} and {right.Vector.Length} cannot be zipped")); + } + + return (new NumValue(default, [.. left.Vector.Zip(right.Vector, Apply)], true), null); + } + + // Scalar broadcast. + return left.IsVector + ? (new NumValue(default, [.. left.Vector.Select(n => Apply(n, right.Scalar))], true), null) + : (new NumValue(default, [.. right.Vector.Select(n => Apply(left.Scalar, n))], true), null); + } + + // ---------------------------------------------------------------- comparisons + + private static EvalOutcome EvaluateComparison(Comparison statement, EvalEnv env) + { + var left = TryEvalGroundTerm(statement.Left, env); + if (left.Failure is not null) + { + return left.Failure; + } + + var right = TryEvalGroundTerm(statement.Right, env); + if (right.Failure is not null) + { + return right.Failure; + } + + if (statement.Op is CompareOp.Eq or CompareOp.Neq) + { + var equal = NumericOps.StructuralEquals(left.Node, right.Node); + return new GuardResult(statement.Op == CompareOp.Eq ? equal : !equal); + } + + // Orderings require numeric coercion; arrays lift to elementwise AND (zip + broadcast). + return CompareOrdered(statement.Op, left.Node, right.Node); + } + + private static EvalOutcome CompareOrdered(CompareOp op, JsonNode? left, JsonNode? right) + { + bool Satisfies(Num a, Num b) + { + var c = a.CompareTo(b); + return op switch + { + CompareOp.Lt => c < 0, + CompareOp.Le => c <= 0, + CompareOp.Gt => c > 0, + _ => c >= 0, + }; + } + + var leftIsArray = left is JsonArray; + var rightIsArray = right is JsonArray; + + if (!leftIsArray && !rightIsArray) + { + if (!NumericOps.TryCoerce(left, out var ln) || !NumericOps.TryCoerce(right, out var rn)) + { + return new EvalFailure(EvalFailureCodes.Type, "ordering comparisons require numeric operands"); + } + + return new GuardResult(Satisfies(ln, rn)); + } + + var leftItems = leftIsArray ? ((JsonArray)left!).ToArray() : null; + var rightItems = rightIsArray ? ((JsonArray)right!).ToArray() : null; + + if (leftItems is not null && rightItems is not null && leftItems.Length != rightItems.Length) + { + return new EvalFailure( + EvalFailureCodes.LiftLength, + $"arrays of length {leftItems.Length} and {rightItems.Length} cannot be zipped"); + } + + var count = leftItems?.Length ?? rightItems!.Length; + + for (var i = 0; i < count; i++) + { + var l = leftItems is not null ? leftItems[i] : left; + var r = rightItems is not null ? rightItems[i] : right; + + if (!NumericOps.TryCoerce(l, out var ln) || !NumericOps.TryCoerce(r, out var rn)) + { + return new EvalFailure(EvalFailureCodes.Type, $"element {i} is not numeric"); + } + + if (!Satisfies(ln, rn)) + { + return new GuardResult(false); + } + } + + return new GuardResult(true); + } + + // ---------------------------------------------------------------- = (bind / match / test) + + private static EvalOutcome EvaluateBind(BindStmt statement, EvalEnv env) + { + var rightGround = IsGround(statement.Right, env); + var leftGround = IsGround(statement.Left, env); + + if (!rightGround) + { + if (leftGround) + { + // Symmetric tolerance: the model sometimes writes [value = @freshVar]. + return EvaluateBindCore(statement.Right, statement.Left, env); + } + + // A pattern on the right with a FREE left is the "destructure without a source" + // instinct (observed live: [@p = { … "position": @pos … }] outside any query). + if (statement.Right is ObjectPatternTerm) + { + return new EvalFailure( + EvalFailureCodes.Mode, + "the right side of '=' must be fully instantiated (fixed mode)", + "a pattern destructures an EXISTING value — put the BOUND variable on the left " + + "⟨@boundVar = { … }⟩, or destructure list rows by opening a query: " + + "'Consider each row ⟨@p = { … }⟩ from ⟨@theList⟩:' followed by '- …' bullets"); + } + + return new EvalFailure( + EvalFailureCodes.Mode, + "the right side of '=' must be fully instantiated (fixed mode)", + "compute inputs before using them; dependencies flow left to right"); + } + + return EvaluateBindCore(statement.Left, statement.Right, env); + } + + private static EvalOutcome EvaluateBindCore(Term left, Term right, EvalEnv env) + { + var value = TryEvalGroundTerm(right, env); + if (value.Failure is not null) + { + return value.Failure; + } + + if (IsGround(left, env)) + { + var testee = TryEvalGroundTerm(left, env); + return testee.Failure is not null + ? testee.Failure + : new GuardResult(NumericOps.StructuralEquals(testee.Node, value.Node)); + } + + var match = PatternMatcher.Match(left, value.Node, env); + + return match.Success + ? new Bound(match.Bindings) + : new EvalFailure(EvalFailureCodes.MatchFailed, match.Reason ?? "pattern did not match", match.Hint); + } + + // ---------------------------------------------------------------- display + + private static EvalOutcome EvaluateDisplay(DisplayStmt statement, EvalEnv env) + { + var value = TryEvalGroundTerm(statement.Value, env); + + return value.Failure is not null + ? value.Failure + : new DisplayValue(EvalEnv.ToJsonText(value.Node), FormatForDisplay(value.Node)); + } + + /// User-facing rendering: strings unquoted, numbers canonical, structures compact JSON. + public static string FormatForDisplay(JsonNode? node) => NumericOps.GetKind(node) switch + { + JsonValueKind.Null => "null", + JsonValueKind.String => node!.AsValue().GetValue(), + _ => node!.ToJsonString(EvalEnv.JsonTextOptions), + }; + + // ---------------------------------------------------------------- predicate calls + + private static EvalOutcome EvaluateCall(PredicateCall call, EvalEnv env, ISignatureCatalog catalog) + { + if (!catalog.TryGet(call.Name, out var signature)) + { + return new EvalFailure( + EvalFailureCodes.UnknownPredicate, + $"unknown predicate '{call.Name}'", + $"known predicates: {string.Join(", ", catalog.KnownNames)}"); + } + + // Named arguments map onto the signature by parameter name; remaining positional + // arguments fill the leftover slots in order. + if (call.Args.Any(a => a is NamedTerm)) + { + var reordered = ReorderNamedArgs(call, signature); + + if (reordered.Failure is not null) + { + return reordered.Failure; + } + + call = call with { Args = reordered.Args }; + } + + if (call.Args.Length != signature.Params.Length) + { + return new EvalFailure( + EvalFailureCodes.Arity, + $"'{signature.Name}' expects {signature.Params.Length} argument(s), got {call.Args.Length}", + $"signature: {Describe(signature)}"); + } + + var inValues = new List<(PredicateParam Param, JsonNode? Value)>(); + var outArgs = ImmutableArray.CreateBuilder(); + + for (var i = 0; i < signature.Params.Length; i++) + { + var param = signature.Params[i]; + var arg = call.Args[i]; + + if (param.Mode == ParamMode.In) + { + if (!IsGround(arg, env)) + { + var free = FreeVars(arg, env).FirstOrDefault() ?? "?"; + return new EvalFailure( + EvalFailureCodes.Unbound, + $"input '{param.Name}' of '{signature.Name}' uses unbound variable '@{free}'", + "inputs must be fully computed before the call (fixed mode)"); + } + + var value = TryEvalGroundTerm(arg, env); + if (value.Failure is not null) + { + return value.Failure; + } + + inValues.Add((param, value.Node)); + } + else + { + switch (arg) + { + case VarTerm v when env.IsBound(v.Name): + return new EvalFailure( + EvalFailureCodes.Rebind, + $"output '{param.Name}' of '{signature.Name}' reuses bound variable '@{v.Name}'", + "write a fresh variable name for every tool output"); + + case VarTerm: + case ObjectPatternTerm: + case ArrayPatternTerm: + outArgs.Add(arg); + break; + + default: + return new EvalFailure( + EvalFailureCodes.LiteralInOutPosition, + $"output '{param.Name}' of '{signature.Name}' must be a fresh variable, not a value", + "never write a value where an output belongs — outputs are named by fresh @variables"); + } + } + } + + var plan = PlanInvocations(signature, inValues); + if (plan.Failure is not null) + { + return plan.Failure; + } + + return signature.IsRule + ? new NeedRule( + signature, + [.. inValues.Select(iv => new Binding(iv.Param.Name, EvalEnv.ToJsonText(iv.Value)))], + outArgs.ToImmutable()) + : new NeedTool(signature, plan.Plan!, outArgs.ToImmutable()); + } + + /// Maps named arguments onto signature positions; false (call unchanged) when they don't fit. + public static bool TryNormalizeNamedArgs(PredicateCall call, PredicateSignature signature, out PredicateCall normalized) + { + if (!call.Args.Any(a => a is NamedTerm)) + { + normalized = call; + return true; + } + + var reordered = ReorderNamedArgs(call, signature); + normalized = reordered.Failure is null ? call with { Args = reordered.Args } : call; + return reordered.Failure is null; + } + + private static (ImmutableArray Args, EvalFailure? Failure) ReorderNamedArgs( + PredicateCall call, + PredicateSignature signature) + { + var slots = new Term?[signature.Params.Length]; + var positionals = new Queue(call.Args.Where(a => a is not NamedTerm)); + + foreach (var named in call.Args.OfType()) + { + var index = signature.Params.ToList().FindIndex(p => + string.Equals(p.Name, named.Name, StringComparison.OrdinalIgnoreCase)); + + if (index < 0) + { + return ([], new EvalFailure( + EvalFailureCodes.Arity, + $"'{named.Name}' is not a parameter of '{signature.Name}'", + $"signature: {Describe(signature)}")); + } + + if (slots[index] is not null) + { + return ([], new EvalFailure( + EvalFailureCodes.Arity, + $"parameter '{named.Name}' of '{signature.Name}' was given twice")); + } + + slots[index] = named.Value; + } + + for (var i = 0; i < slots.Length; i++) + { + if (slots[i] is null && positionals.Count > 0) + { + slots[i] = positionals.Dequeue(); + } + } + + if (slots.Any(s => s is null) || positionals.Count > 0) + { + return ([], new EvalFailure( + EvalFailureCodes.Arity, + $"'{signature.Name}' expects {signature.Params.Length} argument(s)", + $"signature: {Describe(signature)}")); + } + + return ([.. slots.Select(s => s!)], null); + } + + private static (LiftPlan? Plan, EvalFailure? Failure) PlanInvocations( + PredicateSignature signature, + List<(PredicateParam Param, JsonNode? Value)> inValues) + { + // Zip lifting: scalar-declared inputs receiving arrays are lifted together. + var liftedLength = -1; + + foreach (var (param, value) in inValues) + { + if (!param.AcceptsCollection && value is JsonArray arr) + { + if (arr.Any(item => item is JsonArray)) + { + return (null, new EvalFailure( + EvalFailureCodes.LiftNested, + $"input '{param.Name}' is an array of arrays; nested lifting is not supported")); + } + + if (liftedLength < 0) + { + liftedLength = arr.Count; + } + else if (liftedLength != arr.Count) + { + return (null, new EvalFailure( + EvalFailureCodes.LiftLength, + $"arrays of length {liftedLength} and {arr.Count} cannot be zipped")); + } + } + } + + var count = liftedLength < 0 ? 1 : liftedLength; + var payloads = ImmutableArray.CreateBuilder(count); + + for (var k = 0; k < count; k++) + { + var payload = new JsonObject(); + + foreach (var (param, value) in inValues) + { + var effective = liftedLength >= 0 && !param.AcceptsCollection && value is JsonArray arr + ? arr[k] + : value; + + payload[param.Name] = effective?.DeepClone(); + } + + payloads.Add(EvalEnv.ToJsonText(payload)); + } + + return (new LiftPlan(payloads.ToImmutable()), null); + } + + /// + /// Binds tool results to out-arguments. One result payload per invocation: a single-output + /// tool returns the bare value; a multi-output tool returns an object keyed by out-parameter + /// names. Lifted calls bind structure-of-arrays: each out variable (including variables inside + /// out patterns) collects one element per invocation, in source order. + /// + public static EvalOutcome BindToolResults( + PredicateSignature signature, + ImmutableArray outArgs, + ImmutableArray perInvocationResultJson, + EvalEnv env) + { + var outParams = signature.OutParams; + + if (outArgs.Length != outParams.Length) + { + return new EvalFailure( + EvalFailureCodes.ToolResult, + $"'{signature.Name}': {outParams.Length} output parameter(s) but {outArgs.Length} out-argument(s)"); + } + + if (outParams.Length == 0) + { + return new Bound([]); + } + + // valuesPerOut[i][k] = value of out-param i in invocation k. + var valuesPerOut = new JsonNode?[outParams.Length][]; + for (var i = 0; i < outParams.Length; i++) + { + valuesPerOut[i] = new JsonNode?[perInvocationResultJson.Length]; + } + + for (var k = 0; k < perInvocationResultJson.Length; k++) + { + JsonNode? result; + try + { + result = JsonNode.Parse(perInvocationResultJson[k]); + } + catch (JsonException ex) + { + return new EvalFailure(EvalFailureCodes.ToolResult, $"'{signature.Name}' returned invalid JSON: {ex.Message}"); + } + + if (outParams.Length == 1) + { + valuesPerOut[0][k] = result; + continue; + } + + if (result is not JsonObject obj) + { + return new EvalFailure( + EvalFailureCodes.ToolResult, + $"'{signature.Name}' has {outParams.Length} outputs and must return an object keyed by output names"); + } + + for (var i = 0; i < outParams.Length; i++) + { + if (!obj.TryGetPropertyValue(outParams[i].Name, out var value)) + { + return new EvalFailure( + EvalFailureCodes.ToolResult, + $"'{signature.Name}' result is missing output '{outParams[i].Name}'"); + } + + valuesPerOut[i][k] = value; + } + } + + var lifted = perInvocationResultJson.Length > 1; + var bindings = new Dictionary(StringComparer.Ordinal); + + for (var i = 0; i < outParams.Length; i++) + { + if (outArgs[i] is VarTerm v) + { + var value = lifted + ? new JsonArray([.. valuesPerOut[i].Select(n => n?.DeepClone())]) + : valuesPerOut[i][0]; + + if (!TryAddBinding(bindings, v.Name, EvalEnv.ToJsonText(value), out var conflict)) + { + return conflict; + } + + continue; + } + + // Pattern out-arg: match per invocation; each pattern variable collects one element + // per invocation (structure-of-arrays). + var collected = new Dictionary>(StringComparer.Ordinal); + + for (var k = 0; k < valuesPerOut[i].Length; k++) + { + var match = PatternMatcher.Match(outArgs[i], valuesPerOut[i][k], env); + + if (!match.Success) + { + return new EvalFailure( + EvalFailureCodes.MatchFailed, + $"output pattern for '{outParams[i].Name}' of '{signature.Name}' did not match: {match.Reason}", + match.Hint); + } + + foreach (var binding in match.Bindings) + { + if (!collected.TryGetValue(binding.Var, out var list)) + { + collected[binding.Var] = list = []; + } + + list.Add(JsonNode.Parse(binding.Json)); + } + } + + foreach (var (name, values) in collected) + { + var text = lifted + ? EvalEnv.ToJsonText(new JsonArray([.. values.Select(n => n?.DeepClone())])) + : EvalEnv.ToJsonText(values[0]); + + if (!TryAddBinding(bindings, name, text, out var conflict)) + { + return conflict; + } + } + } + + return new Bound([.. bindings.Select(kv => new Binding(kv.Key, kv.Value))]); + } + + private static bool TryAddBinding(Dictionary bindings, string name, string json, out EvalFailure conflict) + { + if (bindings.TryGetValue(name, out var existing) && existing != json) + { + conflict = new EvalFailure( + EvalFailureCodes.Rebind, + $"variable '@{name}' would be bound to two different values by the same call"); + return false; + } + + bindings[name] = json; + conflict = default!; + return true; + } + + // ---------------------------------------------------------------- ground terms + + public static bool IsGround(Term term, EvalEnv env) => term switch + { + VarTerm v => env.IsBound(v.Name), + StrTerm or NumTerm or BoolTerm or NullTerm => true, + ObjectPatternTerm o => o.Fields.All(f => IsGround(f.Value, env)), + ArrayPatternTerm a => a.Items.All(item => IsGround(item, env)), + ExprTerm e => FreeArithVars(e.Expr, env).Count == 0, + NamedTerm n => IsGround(n.Value, env), + _ => false, + }; + + private static IEnumerable FreeVars(Term term, EvalEnv env) + { + switch (term) + { + case VarTerm v when !env.IsBound(v.Name): + yield return v.Name; + break; + case ObjectPatternTerm o: + foreach (var f in o.Fields) + { + foreach (var name in FreeVars(f.Value, env)) + { + yield return name; + } + } + + break; + case ArrayPatternTerm a: + foreach (var item in a.Items) + { + foreach (var name in FreeVars(item, env)) + { + yield return name; + } + } + + break; + case ExprTerm e: + foreach (var name in FreeArithVars(e.Expr, env)) + { + yield return name; + } + + break; + } + } + + private static List FreeArithVars(ArithExpr expr, EvalEnv env) + { + var free = new List(); + + void Walk(ArithExpr e) + { + switch (e) + { + case ArithVar v when !env.IsBound(v.Name): + free.Add(v.Name); + break; + case ArithBinary b: + Walk(b.Left); + Walk(b.Right); + break; + case ArithNeg n: + Walk(n.Operand); + break; + } + } + + Walk(expr); + return free; + } + + private readonly record struct TermResult(JsonNode? Node, EvalFailure? Failure) + { + public static TermResult Ok(JsonNode? node) => new(node, null); + + public static TermResult Fail(EvalFailure failure) => new(null, failure); + } + + /// + /// Evaluates a ground term to a JSON value. Ground patterns act as constructors — + /// [@x = {"a": @b}] builds an object from bound parts. + /// + private static TermResult TryEvalGroundTerm(Term term, EvalEnv env) + { + switch (term) + { + case VarTerm v: + if (!env.IsBound(v.Name)) + { + return TermResult.Fail(new EvalFailure( + EvalFailureCodes.Unbound, + $"variable '@{v.Name}' is not bound", + "inputs must be fully computed before use (fixed mode)")); + } + + return TermResult.Ok(env.GetNode(v.Name)?.DeepClone()); + + case StrTerm s: + return TermResult.Ok(JsonValue.Create(s.Value)); + + case NumTerm n: + return TermResult.Ok(JsonValue.Create(NumericOps.Normalize(n.Value))); + + case BoolTerm b: + return TermResult.Ok(JsonValue.Create(b.Value)); + + case NullTerm: + return TermResult.Ok(null); + + case ObjectPatternTerm o: + { + var obj = new JsonObject(); + + foreach (var field in o.Fields) + { + var value = TryEvalGroundTerm(field.Value, env); + if (value.Failure is not null) + { + return value; + } + + obj[field.Key] = value.Node; + } + + return TermResult.Ok(obj); + } + + case ArrayPatternTerm a: + { + var arr = new JsonArray(); + + foreach (var item in a.Items) + { + var value = TryEvalGroundTerm(item, env); + if (value.Failure is not null) + { + return value; + } + + arr.Add(value.Node); + } + + return TermResult.Ok(arr); + } + + case ExprTerm e: + { + var result = EvalArith(e.Expr, env); + return result.Failure is not null ? TermResult.Fail(result.Failure) : TermResult.Ok(result.Node); + } + + default: + return TermResult.Fail(new EvalFailure(EvalFailureCodes.Type, $"unsupported term '{term.GetType().Name}'")); + } + } + + private static string Describe(PredicateSignature signature) => + $"{signature.Name}({string.Join(", ", signature.Params.Select(p => $"{p.Name}: {(p.Mode == ParamMode.In ? "in" : "out")}"))})"; +} diff --git a/src/Universalis.Core/Evaluation/NumericOps.cs b/src/Universalis.Core/Evaluation/NumericOps.cs new file mode 100644 index 0000000..411966a --- /dev/null +++ b/src/Universalis.Core/Evaluation/NumericOps.cs @@ -0,0 +1,249 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Universalis.Core.Evaluation; + +/// +/// The numeric tower: decimal-primary (exact for the papers' money/percentage domain), +/// double-fallback (overflow/precision). Coercion is deliberately lenient toward numeric JSON +/// strings — the paper's own BTC example multiplies a string-typed "close" field without +/// a conversion tool — while non-numeric strings remain type errors (@X+"hello"). +/// +public readonly struct Num +{ + private readonly decimal _dec; + private readonly double _dbl; + + public bool IsDouble { get; } + + private Num(decimal dec) + { + _dec = dec; + _dbl = 0; + IsDouble = false; + } + + private Num(double dbl) + { + _dec = 0; + _dbl = dbl; + IsDouble = true; + } + + public static Num FromDecimal(decimal value) => new(value); + + public static Num FromDouble(double value) => new(value); + + public double AsDouble => IsDouble ? _dbl : (double)_dec; + + public decimal AsDecimal => IsDouble ? (decimal)_dbl : _dec; + + public Num Negate() => IsDouble ? new Num(-_dbl) : new Num(-_dec); + + public Num Add(Num other) => Apply(other, static (a, b) => a + b, static (a, b) => a + b); + + public Num Sub(Num other) => Apply(other, static (a, b) => a - b, static (a, b) => a - b); + + public Num Mul(Num other) => Apply(other, static (a, b) => a * b, static (a, b) => a * b); + + public Num Div(Num other) + { + if (!IsDouble && !other.IsDouble && other._dec == 0m) + { + throw new DivideByZeroException(); + } + + if ((IsDouble || other.IsDouble) && other.AsDouble == 0d) + { + throw new DivideByZeroException(); + } + + return Apply(other, static (a, b) => a / b, static (a, b) => a / b); + } + + public Num Mod(Num other) + { + if (!IsDouble && !other.IsDouble && other._dec == 0m) + { + throw new DivideByZeroException(); + } + + return Apply(other, static (a, b) => a % b, static (a, b) => a % b); + } + + public int CompareTo(Num other) + { + if (!IsDouble && !other.IsDouble) + { + return _dec.CompareTo(other._dec); + } + + return AsDouble.CompareTo(other.AsDouble); + } + + private Num Apply(Num other, Func dec, Func dbl) + { + if (!IsDouble && !other.IsDouble) + { + try + { + return new Num(dec(_dec, other._dec)); + } + catch (OverflowException) + { + // Fall through to the double path. + } + } + + return new Num(dbl(AsDouble, other.AsDouble)); + } + + public JsonNode ToJson() => IsDouble + ? JsonValue.Create(_dbl) + : JsonValue.Create(NumericOps.Normalize(_dec)); + + public override string ToString() => IsDouble + ? _dbl.ToString("R", CultureInfo.InvariantCulture) + : NumericOps.CanonicalText(_dec); +} + +public static class NumericOps +{ + /// Coerces a JSON node to a number per the tower rules; false for non-numeric shapes. + public static bool TryCoerce(JsonNode? node, out Num value) + { + value = default; + + if (node is not JsonValue jv) + { + return false; + } + + switch (jv.GetValueKind()) + { + case JsonValueKind.Number: + if (jv.TryGetValue(out var dec)) + { + value = Num.FromDecimal(dec); + return true; + } + + if (jv.TryGetValue(out var dbl)) + { + value = Num.FromDouble(dbl); + return true; + } + + return false; + + case JsonValueKind.String: + var text = jv.GetValue().Trim(); + + if (decimal.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var sdec)) + { + value = Num.FromDecimal(sdec); + return true; + } + + if (double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var sdbl)) + { + value = Num.FromDouble(sdbl); + return true; + } + + return false; + + default: + return false; + } + } + + /// Drops insignificant trailing zeros (70.00 → 70) so σ text is canonical. + public static decimal Normalize(decimal value) => + decimal.Parse(value.ToString("G29", CultureInfo.InvariantCulture), NumberStyles.Float, CultureInfo.InvariantCulture); + + public static string CanonicalText(decimal value) => + Normalize(value).ToString(CultureInfo.InvariantCulture); + + /// + /// Numeric-aware structural equality: numbers compare by value (3 == 3.0 == "3"), strings + /// ordinal, objects by key set + recursive values, arrays by length + elements. + /// + public static bool StructuralEquals(JsonNode? left, JsonNode? right) + { + if (left is null || right is null) + { + return GetKind(left) == JsonValueKind.Null && GetKind(right) == JsonValueKind.Null; + } + + if (TryCoerce(left, out var ln) && TryCoerce(right, out var rn)) + { + return ln.CompareTo(rn) == 0; + } + + var lk = GetKind(left); + var rk = GetKind(right); + + if (lk != rk) + { + return false; + } + + switch (lk) + { + case JsonValueKind.String: + return string.Equals(left.AsValue().GetValue(), right.AsValue().GetValue(), StringComparison.Ordinal); + + case JsonValueKind.True: + case JsonValueKind.False: + return left.AsValue().GetValue() == right.AsValue().GetValue(); + + case JsonValueKind.Null: + return true; + + case JsonValueKind.Object: + var lo = left.AsObject(); + var ro = right.AsObject(); + + if (lo.Count != ro.Count) + { + return false; + } + + foreach (var (key, lv) in lo) + { + if (!ro.TryGetPropertyValue(key, out var rv) || !StructuralEquals(lv, rv)) + { + return false; + } + } + + return true; + + case JsonValueKind.Array: + var la = left.AsArray(); + var ra = right.AsArray(); + + if (la.Count != ra.Count) + { + return false; + } + + for (var i = 0; i < la.Count; i++) + { + if (!StructuralEquals(la[i], ra[i])) + { + return false; + } + } + + return true; + + default: + return false; + } + } + + public static JsonValueKind GetKind(JsonNode? node) => node?.GetValueKind() ?? JsonValueKind.Null; +} diff --git a/src/Universalis.Core/Evaluation/PatternMatcher.cs b/src/Universalis.Core/Evaluation/PatternMatcher.cs new file mode 100644 index 0000000..1608540 --- /dev/null +++ b/src/Universalis.Core/Evaluation/PatternMatcher.cs @@ -0,0 +1,262 @@ +using System.Collections.Immutable; +using System.Text.Json; +using System.Text.Json.Nodes; + +using Universalis.Core.Ir; + +namespace Universalis.Core.Evaluation; + +public sealed record MatchOutcome(bool Success, ImmutableArray Bindings, string? Reason, string? Hint) +{ + public static MatchOutcome Ok(ImmutableArray bindings) => new(true, bindings, null, null); + + public static MatchOutcome Fail(string reason, string? hint = null) => new(false, [], reason, hint); +} + +/// +/// One-way pattern matching (fixed mode — no unification). Open object patterns +/// { ... "key": p ... } resolve each key independently by pre-order depth-first descent +/// from the matched node (self level first, then descendants in document order) — both paper +/// examples require deep search ("volume" lives under data[0] in the STOCK blob, +/// "forecast" under properties in weather.gov). Failure hints enumerate available keys; +/// that hint is what the backtracking loop feeds the model. +/// +public static class PatternMatcher +{ + public static MatchOutcome Match(Term pattern, JsonNode? value, EvalEnv env) + { + var bindings = new Dictionary(StringComparer.Ordinal); + var result = MatchCore(pattern, value, env, bindings); + + return result is null + ? MatchOutcome.Ok([.. bindings.Select(kv => new Binding(kv.Key, kv.Value))]) + : result; + } + + /// Returns null on success (bindings accumulated), or a failure outcome. + private static MatchOutcome? MatchCore(Term pattern, JsonNode? value, EvalEnv env, Dictionary bindings) + { + switch (pattern) + { + case VarTerm v: + // Non-linear patterns: an already-bound variable acts as an equality test. + if (bindings.TryGetValue(v.Name, out var priorText)) + { + return NumericOps.StructuralEquals(JsonNode.Parse(priorText), value) + ? null + : MatchOutcome.Fail($"variable '@{v.Name}' already matched a different value"); + } + + if (env.IsBound(v.Name)) + { + return NumericOps.StructuralEquals(env.GetNode(v.Name), value) + ? null + : MatchOutcome.Fail($"variable '@{v.Name}' is already bound to a different value"); + } + + bindings[v.Name] = EvalEnv.ToJsonText(value); + return null; + + case StrTerm s: + return NumericOps.GetKind(value) == JsonValueKind.String && + string.Equals(value!.AsValue().GetValue(), s.Value, StringComparison.Ordinal) + ? null + : MatchOutcome.Fail($"expected \"{s.Value}\""); + + case NumTerm n: + return NumericOps.TryCoerce(value, out var num) && num.CompareTo(Num.FromDecimal(n.Value)) == 0 + ? null + : MatchOutcome.Fail($"expected {NumericOps.CanonicalText(n.Value)}"); + + case BoolTerm b: + return (NumericOps.GetKind(value) is JsonValueKind.True or JsonValueKind.False) && + value!.AsValue().GetValue() == b.Value + ? null + : MatchOutcome.Fail($"expected {(b.Value ? "true" : "false")}"); + + case NullTerm: + return NumericOps.GetKind(value) == JsonValueKind.Null + ? null + : MatchOutcome.Fail("expected null"); + + case ObjectPatternTerm o: + return MatchObject(o, value, env, bindings); + + case ArrayPatternTerm a: + return MatchArray(a, value, env, bindings); + + case ExprTerm: + return MatchOutcome.Fail("an arithmetic expression cannot appear inside a pattern"); + + default: + return MatchOutcome.Fail($"unsupported pattern term '{pattern.GetType().Name}'"); + } + } + + private static MatchOutcome? MatchObject(ObjectPatternTerm pattern, JsonNode? value, EvalEnv env, Dictionary bindings) + { + if (value is not JsonObject obj) + { + return MatchOutcome.Fail($"expected a JSON object, found {Describe(value)}"); + } + + if (pattern.IsOpen) + { + foreach (var field in pattern.Fields) + { + if (!TryFindKeyDfs(obj, field.Key, out var found)) + { + return MatchOutcome.Fail( + $"pattern key \"{field.Key}\" not found", + $"available keys include: {DescribeKeys(obj)}"); + } + + var inner = MatchCore(field.Value, found, env, bindings); + if (inner is not null) + { + return inner; + } + } + + return null; + } + + // Closed pattern: exact key set. + if (obj.Count != pattern.Fields.Length) + { + return MatchOutcome.Fail( + $"closed pattern expects exactly {pattern.Fields.Length} key(s), object has {obj.Count}", + $"available keys: {DescribeKeys(obj)}"); + } + + foreach (var field in pattern.Fields) + { + if (!obj.TryGetPropertyValue(field.Key, out var propValue)) + { + return MatchOutcome.Fail( + $"pattern key \"{field.Key}\" not found", + $"available keys: {DescribeKeys(obj)}"); + } + + var inner = MatchCore(field.Value, propValue, env, bindings); + if (inner is not null) + { + return inner; + } + } + + return null; + } + + private static MatchOutcome? MatchArray(ArrayPatternTerm pattern, JsonNode? value, EvalEnv env, Dictionary bindings) + { + if (value is not JsonArray arr) + { + return MatchOutcome.Fail($"expected a JSON array, found {Describe(value)}"); + } + + var k = pattern.Items.Length; + + switch (pattern.Ellipsis) + { + case EllipsisPosition.None when arr.Count != k: + return MatchOutcome.Fail($"expected an array of exactly {k} item(s), found {arr.Count}"); + + case EllipsisPosition.Leading or EllipsisPosition.Trailing when arr.Count < k: + return MatchOutcome.Fail($"expected an array of at least {k} item(s), found {arr.Count}"); + } + + var offset = pattern.Ellipsis == EllipsisPosition.Leading ? arr.Count - k : 0; + + for (var i = 0; i < k; i++) + { + var inner = MatchCore(pattern.Items[i], arr[offset + i], env, bindings); + if (inner is not null) + { + return inner; + } + } + + return null; + } + + /// + /// Pre-order depth-first key search: the node's own properties win over descendants; children + /// are explored in document order (array elements by index). First match is the match. + /// + internal static bool TryFindKeyDfs(JsonNode? node, string key, out JsonNode? found) + { + switch (node) + { + case JsonObject obj: + if (obj.TryGetPropertyValue(key, out found)) + { + return true; + } + + foreach (var (_, child) in obj) + { + if (TryFindKeyDfs(child, key, out found)) + { + return true; + } + } + + break; + + case JsonArray arr: + foreach (var item in arr) + { + if (TryFindKeyDfs(item, key, out found)) + { + return true; + } + } + + break; + } + + found = null; + return false; + } + + private static string Describe(JsonNode? node) => NumericOps.GetKind(node) switch + { + JsonValueKind.Null => "null", + JsonValueKind.Object => "an object", + JsonValueKind.Array => "an array", + JsonValueKind.String => "a string", + JsonValueKind.Number => "a number", + JsonValueKind.True or JsonValueKind.False => "a boolean", + _ => "an unknown value", + }; + + /// Keys at the top two levels (capped), for actionable match-failure hints. + private static string DescribeKeys(JsonObject obj) + { + const int Cap = 24; + var keys = new List(); + + foreach (var (key, child) in obj) + { + keys.Add(key); + + if (child is JsonObject nested) + { + keys.AddRange(nested.Select(kv => $"{key}.{kv.Key}")); + } + else if (child is JsonArray { Count: > 0 } arr && arr[0] is JsonObject first) + { + keys.AddRange(first.Select(kv => $"{key}[0].{kv.Key}")); + } + + if (keys.Count > Cap) + { + break; + } + } + + var capped = keys.Count > Cap; + return string.Join(", ", keys.Take(Cap)) + (capped ? ", …" : ""); + } +} diff --git a/src/Universalis.Core/Evaluation/QueryPipeline.cs b/src/Universalis.Core/Evaluation/QueryPipeline.cs new file mode 100644 index 0000000..975171b --- /dev/null +++ b/src/Universalis.Core/Evaluation/QueryPipeline.cs @@ -0,0 +1,319 @@ +using System.Collections.Immutable; +using System.Text.Json.Nodes; + +using Universalis.Core.Ir; + +namespace Universalis.Core.Evaluation; + +public sealed record QueryResult(ImmutableArray Bindings, int RowsIn, int RowsOut); + +/// +/// Executes a query comprehension as a LINQ pipeline over JSON rows — the paper compiles to +/// Kotlin DataFrames; LINQ is the native .NET analog (and Meijer's own invention). Shape: +/// row as-pattern match → pre-group filters → GroupBy(key) → aggregates + Collect (fully NESTED +/// results, the contrast to SQL) → post-group filters (HAVING) → array result. +/// Rows that fail the item pattern are skipped (lenient row semantics); evaluation errors inside +/// predicates fail the whole query (fixed mode). +/// +public static class QueryPipeline +{ + public static EvalOutcome Execute(ComprehensionBlock block, EvalEnv env, out int rowsIn, out int rowsOut) + { + rowsIn = 0; + rowsOut = 0; + + if (!env.IsBound(block.SourceVar)) + { + return new EvalFailure(EvalFailureCodes.Unbound, $"query source '@{block.SourceVar}' is not bound"); + } + + if (env.GetNode(block.SourceVar) is not JsonArray source) + { + return new EvalFailure(EvalFailureCodes.Type, $"query source '@{block.SourceVar}' must be a JSON array"); + } + + rowsIn = source.Count; + + // ---- row environments: as-pattern destructuring per element ---- + + var rows = new List>(); + + foreach (var element in source) + { + var row = new Dictionary(StringComparer.Ordinal) + { + [block.ItemVar] = EvalEnv.ToJsonText(element), + }; + + if (block.ItemPattern is not null) + { + var match = PatternMatcher.Match(block.ItemPattern, element, env); + + if (!match.Success) + { + continue; // lenient: rows that don't fit the pattern are skipped + } + + foreach (var binding in match.Bindings) + { + row[binding.Var] = binding.Json; + } + } + + rows.Add(row); + } + + // ---- split ops around the GroupBy pivot ---- + + var groupIndex = block.Ops.ToList().FindIndex(op => op is GroupByOp); + var preOps = groupIndex < 0 ? block.Ops : [.. block.Ops.Take(groupIndex)]; + var postOps = groupIndex < 0 ? [] : block.Ops.Skip(groupIndex + 1).ToImmutableArray(); + var groupBy = groupIndex < 0 ? null : (GroupByOp)block.Ops[groupIndex]; + + // ---- pre-group filters ---- + + foreach (var op in preOps.OfType()) + { + var retained = new List>(); + + foreach (var row in rows) + { + var outcome = Evaluator.Evaluate(op.Predicate, env.Push(row), SignatureCatalog.Empty); + + switch (outcome) + { + case GuardResult guard: + if (guard.Value) + { + retained.Add(row); + } + + break; + + case Bound bound: + // A free variable bound to a GROUND LITERAL can never reject a row + // (observed live: no row pattern, then "Retain only … [@city = "Palo + // Alto"]" — every row "passed" and the count came out 4/4). That is a + // missing destructuring, not a filter — fail with the pattern teach. + if (op.Predicate is BindStmt { Left: VarTerm freeVar, Right: StrTerm or NumTerm or BoolTerm }) + { + return new EvalFailure(EvalFailureCodes.Unbound, + $"the filter [@{freeVar.Name} = …] tests nothing because '@{freeVar.Name}' is never bound", + $"the row pattern must destructure the field first — open the query with 'Consider each [@row = {{ … \"{freeVar.Name}\": @{freeVar.Name} … }}] from …' and keep the filter bullet as the test"); + } + + // A free-variable bind DERIVED FROM THE ROW is a row-scoped ALIAS + // (observed live: "Retain only [@pa = @customer] if [@city = …]") — + // bind it into the row and keep the row; the comparisons do the filtering. + foreach (var binding in bound.Bindings) + { + row[binding.Var] = binding.Json; + } + + retained.Add(row); + break; + + case EvalFailure failure: + return failure; + + default: + return new EvalFailure(EvalFailureCodes.Type, "a filter must be a comparison over row fields"); + } + } + + rows = retained; + } + + // ---- ungrouped terminals ---- + + if (groupBy is null) + { + var bindings = ImmutableArray.CreateBuilder(); + + foreach (var op in preOps) + { + switch (op) + { + case FilterOp: + break; + + case CountIntoOp count: + if (env.IsBound(count.TargetVar)) + { + return new EvalFailure(EvalFailureCodes.Rebind, $"'@{count.TargetVar}' is already bound"); + } + + bindings.Add(new Binding(count.TargetVar, rows.Count.ToString(System.Globalization.CultureInfo.InvariantCulture))); + break; + + case AggregateOp aggregate: + { + var result = Aggregate(aggregate.Fn, rows, aggregate.OverVar); + if (result.Failure is not null) + { + return result.Failure; + } + + bindings.Add(new Binding(aggregate.ResultVar, EvalEnv.ToJsonText(result.Node))); + break; + } + + case CollectOp collect: + bindings.Add(new Binding(collect.ResultVar, EvalEnv.ToJsonText(Collect(rows, collect.OverVar)))); + break; + + default: + return new EvalFailure(EvalFailureCodes.Type, $"unsupported query operation '{op.GetType().Name}' without grouping"); + } + } + + if (bindings.Count == 0 || block.IntoVar is not null) + { + var items = new JsonArray([.. rows.Select(r => JsonNode.Parse(r[block.ItemVar]))]); + bindings.Add(new Binding(block.IntoVar ?? "queryResult", EvalEnv.ToJsonText(items))); + } + + rowsOut = rows.Count; + return new Bound(bindings.ToImmutable()); + } + + // ---- grouped pipeline ---- + + var groups = rows + .Where(r => r.ContainsKey(groupBy.KeyVar)) + .GroupBy(r => r[groupBy.KeyVar], StringComparer.Ordinal) + .ToList(); + + var records = new List<(JsonObject Record, Dictionary Scope)>(); + + foreach (var group in groups) + { + var record = new JsonObject + { + [groupBy.KeyVar] = JsonNode.Parse(group.Key), + }; + + var scope = new Dictionary(StringComparer.Ordinal) + { + [groupBy.KeyVar] = group.Key, + }; + + foreach (var op in postOps) + { + switch (op) + { + case AggregateOp aggregate: + { + var result = Aggregate(aggregate.Fn, [.. group], aggregate.OverVar); + if (result.Failure is not null) + { + return result.Failure; + } + + record[aggregate.ResultField] = result.Node?.DeepClone(); + scope[aggregate.ResultVar] = EvalEnv.ToJsonText(result.Node); + break; + } + + case CollectOp collect: + { + var items = Collect([.. group], collect.OverVar); + record[collect.ResultField] = items.DeepClone(); + scope[collect.ResultVar] = EvalEnv.ToJsonText(items); + break; + } + + case FilterOp: + break; // HAVING — evaluated below, after all group fields exist + + default: + return new EvalFailure(EvalFailureCodes.Type, $"unsupported grouped operation '{op.GetType().Name}'"); + } + } + + records.Add((record, scope)); + } + + // ---- HAVING filters ---- + + foreach (var having in postOps.OfType()) + { + var retained = new List<(JsonObject Record, Dictionary Scope)>(); + + foreach (var (record, scope) in records) + { + var outcome = Evaluator.Evaluate(having.Predicate, env.Push(scope), SignatureCatalog.Empty); + + switch (outcome) + { + case GuardResult guard: + if (guard.Value) + { + retained.Add((record, scope)); + } + + break; + + case EvalFailure failure: + return failure; + + default: + return new EvalFailure(EvalFailureCodes.Type, "a group filter must be a comparison over group fields"); + } + } + + records = retained; + } + + rowsOut = records.Count; + + var array = new JsonArray([.. records.Select(r => r.Record.DeepClone())]); + return new Bound([new Binding(block.IntoVar ?? "queryResult", EvalEnv.ToJsonText(array))]); + } + + private static (JsonNode? Node, EvalFailure? Failure) Aggregate( + AggregateFn fn, + List> rows, + string overVar) + { + if (fn == AggregateFn.Count) + { + return (JsonValue.Create(rows.Count), null); + } + + var values = new List(); + + foreach (var row in rows) + { + if (!row.TryGetValue(overVar, out var json)) + { + return (null, new EvalFailure(EvalFailureCodes.Unbound, $"row variable '@{overVar}' is not bound by the item pattern")); + } + + if (!NumericOps.TryCoerce(JsonNode.Parse(json), out var num)) + { + return (null, new EvalFailure(EvalFailureCodes.Type, $"'@{overVar}' contains a non-numeric value")); + } + + values.Add(num); + } + + if (values.Count == 0) + { + return (null, new EvalFailure("empty-aggregate", $"cannot aggregate '@{overVar}' over an empty group")); + } + + var result = fn switch + { + AggregateFn.Sum => values.Aggregate((a, b) => a.Add(b)), + AggregateFn.Min => values.Aggregate((a, b) => a.CompareTo(b) <= 0 ? a : b), + AggregateFn.Max => values.Aggregate((a, b) => a.CompareTo(b) >= 0 ? a : b), + _ => values.Aggregate((a, b) => a.Add(b)).Div(Num.FromDecimal(values.Count)), // Mean + }; + + return (result.ToJson(), null); + } + + private static JsonArray Collect(List> rows, string overVar) => + new([.. rows.Where(r => r.ContainsKey(overVar)).Select(r => JsonNode.Parse(r[overVar]))]); +} diff --git a/src/Universalis.Core/Ir/IrJson.cs b/src/Universalis.Core/Ir/IrJson.cs new file mode 100644 index 0000000..7d603c5 --- /dev/null +++ b/src/Universalis.Core/Ir/IrJson.cs @@ -0,0 +1,30 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Universalis.Core.Ir; + +/// +/// Canonical JSON (de)serialization of the intentional representation. The structured IR is the +/// persistence format; the paper's [{comment}|{expression}] shape is an interchange view +/// (see PaperShape) — render is total, parse is fallible, so we store the executable form. +/// +public static class IrJson +{ + public static JsonSerializerOptions Options { get; } = new() + { + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + public static string Serialize(UniversalisProgram program) => JsonSerializer.Serialize(program, Options); + + public static UniversalisProgram DeserializeProgram(string json) => + JsonSerializer.Deserialize(json, Options) + ?? throw new JsonException("null Universalis program"); + + public static string Serialize(RuleDefinition rule) => JsonSerializer.Serialize(rule, Options); + + public static RuleDefinition DeserializeRule(string json) => + JsonSerializer.Deserialize(json, Options) + ?? throw new JsonException("null Universalis rule"); +} diff --git a/src/Universalis.Core/Ir/PaperShape.cs b/src/Universalis.Core/Ir/PaperShape.cs new file mode 100644 index 0000000..e37f110 --- /dev/null +++ b/src/Universalis.Core/Ir/PaperShape.cs @@ -0,0 +1,168 @@ +using System.Collections.Immutable; +using System.Text.Json; +using System.Text.Json.Nodes; + +using Universalis.Core.Parsing; +using Universalis.Core.Rendering; + +namespace Universalis.Core.Ir; + +/// +/// The papers' interchange view of the intentional representation: a JSON array of +/// {"comment": ...} / {"expression": ...} objects. Export is total; import is +/// fallible (expressions re-parse through the same hedge grammar) — which is exactly why the +/// structured IR, not this shape, is the persistence format. +/// +public static class PaperShape +{ + public static string Export(UniversalisProgram program) + { + var array = new JsonArray(); + + void Append(ProgramItem item) + { + switch (item) + { + case Comment c: + if (!string.IsNullOrWhiteSpace(c.Text)) + { + array.Add(new JsonObject { ["comment"] = c.Text.Trim() }); + } + + break; + + case HedgeItem h: + array.Add(new JsonObject { ["expression"] = ConcreteRenderer.RenderHedge(h, RenderMode.Formulas) }); + break; + + case ConditionalBlock cond: + foreach (var branch in cond.Branches) + { + array.Add(new JsonObject { ["comment"] = (branch.Guard is null ? "Otherwise:" : "If:") + branch.GuardProse }); + + if (branch.Guard is not null) + { + array.Add(new JsonObject { ["expression"] = ConcreteRenderer.RenderStatement(branch.Guard, RenderMode.Formulas) }); + } + + foreach (var inner in branch.Body) + { + Append(inner); + } + } + + break; + + case ComprehensionBlock comp: + foreach (var op in comp.Ops) + { + array.Add(new JsonObject { ["comment"] = op.Prose }); + } + + break; + } + } + + foreach (var item in program.Items) + { + Append(item); + } + + return array.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); + } + + public static ProgramParseResult Import(string json) + { + JsonNode? root; + try + { + root = JsonNode.Parse(json); + } + catch (JsonException ex) + { + return new ProgramParseResult(null, [], $"invalid JSON: {ex.Message}"); + } + + if (root is not JsonArray array) + { + return new ProgramParseResult(null, [], "expected a JSON array of {comment}/{expression} objects"); + } + + var items = ImmutableArray.CreateBuilder(); + var warnings = ImmutableArray.CreateBuilder(); + + foreach (var element in array) + { + if (element is not JsonObject obj) + { + return new ProgramParseResult(null, warnings.ToImmutable(), "array items must be objects"); + } + + // An item may carry BOTH keys — the Mode B decoding schema permits it, and small + // models pair prose with its code per item; the intent (comment, then expression) + // is unambiguous, so both import instead of the expression being silently dropped + // (review finding). Non-string values teach instead of throwing (review finding — + // GetValue on {"comment": 42} escaped as an internal error). + var hasAny = false; + + if (obj.TryGetPropertyValue("comment", out var comment)) + { + if (comment is not JsonValue commentValue || !commentValue.TryGetValue(out var commentText)) + { + return new ProgramParseResult(null, warnings.ToImmutable(), + $"'comment' must be a string, got: {comment?.ToJsonString() ?? "null"}"); + } + + items.Add(new Comment(commentText + " ")); + hasAny = true; + } + + if (obj.TryGetPropertyValue("expression", out var expression)) + { + if (expression is not JsonValue expressionValue || !expressionValue.TryGetValue(out var text)) + { + return new ProgramParseResult(null, warnings.ToImmutable(), + $"'expression' must be a string, got: {expression?.ToJsonString() ?? "null"}"); + } + + var parsed = HedgeParser.Parse(text); + + // Tolerance: models emitting the interchange form often keep the hedge + // brackets inside the expression string — "[@cash is 250]". Strip ONE + // balanced surrounding pair and retry before rejecting. + if (!parsed.Success && text.Length > 1 && text.TrimStart().StartsWith('[') && text.TrimEnd().EndsWith(']')) + { + var stripped = text.Trim()[1..^1]; + var retry = HedgeParser.Parse(stripped); + + if (retry.Success) + { + warnings.Add($"stripped surrounding brackets from '{text}'"); + text = stripped; + parsed = retry; + } + } + + if (!parsed.Success) + { + return new ProgramParseResult(null, warnings.ToImmutable(), $"cannot parse expression '{text}': {parsed.Error}"); + } + + if (parsed.Healed) + { + warnings.Add($"healed a surplus ')' in '{text}'"); + } + + items.Add(new HedgeItem(parsed.Statement!, text)); + hasAny = true; + } + + if (!hasAny) + { + return new ProgramParseResult(null, warnings.ToImmutable(), "each item must have a 'comment' or 'expression' key"); + } + } + + return new ProgramParseResult(new UniversalisProgram(items.ToImmutable(), [], []), warnings.ToImmutable(), null); + } +} diff --git a/src/Universalis.Core/Ir/Program.cs b/src/Universalis.Core/Ir/Program.cs new file mode 100644 index 0000000..c67f402 --- /dev/null +++ b/src/Universalis.Core/Ir/Program.cs @@ -0,0 +1,130 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace Universalis.Core.Ir; + +/// +/// One item of a literate Universalis program: prose, an executable hedge, or a structured block. +/// A program is the paper's intentional representation — concrete syntax is a rendering. +/// +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$kind")] +[JsonDerivedType(typeof(Comment), "comment")] +[JsonDerivedType(typeof(HedgeItem), "hedge")] +[JsonDerivedType(typeof(ConditionalBlock), "conditional")] +[JsonDerivedType(typeof(ComprehensionBlock), "comprehension")] +public abstract record ProgramItem; + +/// +/// Natural-language prose between hedges. Semantically inert; rhetorically load-bearing. +/// marks engine-injected steering (backtrack feedback) — part of the +/// model-facing trace but excluded from the user-facing answer. +/// +public sealed record Comment(string Text, bool Aside = false) : ProgramItem; + +/// +/// An executable hedge. preserves the exact source text (as emitted by +/// the model or written by the user) so rendering is lossless even where the renderer normalizes. +/// +public sealed record HedgeItem(Statement Statement, string ConcreteText) : ProgramItem; + +/// +/// A conditional checklist: - If [guard], then ... bullets. Execution is if/elif — the +/// first true branch runs; all guards are still evaluated for the trace, and untaken branches are +/// recorded (the UI renders them crossed out, per the paper). +/// +public sealed record ConditionalBlock(ImmutableArray Branches) : ProgramItem; + +/// A branch of a checklist; a null is an else-branch ("Otherwise, ..."). +public sealed record ConditionalBranch( + Statement? Guard, + string GuardProse, + ImmutableArray Body); + +/// +/// A query comprehension: Consider each [@x = pattern] from [@source]: followed by +/// operation bullets. Compiles to a LINQ pipeline over JSON rows with fully nested results. +/// +public sealed record ComprehensionBlock( + string ItemVar, + ObjectPatternTerm? ItemPattern, + string SourceVar, + ImmutableArray Ops, + string? IntoVar) : ProgramItem; + +/// One operation bullet of a comprehension; keeps the literate phrasing. +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$kind")] +[JsonDerivedType(typeof(FilterOp), "filter")] +[JsonDerivedType(typeof(GroupByOp), "groupBy")] +[JsonDerivedType(typeof(AggregateOp), "aggregate")] +[JsonDerivedType(typeof(CollectOp), "collect")] +[JsonDerivedType(typeof(CountIntoOp), "countInto")] +public abstract record QueryOp(string Prose); + +/// Retain/Keep only rows (pre-GroupBy) or groups (post-GroupBy, i.e. HAVING) satisfying the predicate. +public sealed record FilterOp(Statement Predicate, string Prose) : QueryOp(Prose); + +public sealed record GroupByOp(string KeyVar, string Prose) : QueryOp(Prose); + +/// Aggregate per group into { ResultField: @ResultVar }. +public sealed record AggregateOp( + AggregateFn Fn, + string OverVar, + string ResultField, + string ResultVar, + string Prose) : QueryOp(Prose); + +/// Collect the group's elements into { ResultField: @ResultVar } — the nested-result superpower. +public sealed record CollectOp( + string OverVar, + string ResultField, + string ResultVar, + string Prose) : QueryOp(Prose); + +/// Count retained rows into a σ variable (increment [@total] by one ...). +public sealed record CountIntoOp(string TargetVar, string Prose) : QueryOp(Prose); + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum AggregateFn +{ + Mean, + Min, + Max, + Sum, + Count, +} + +/// +/// A contract clause: an optional guard (implication form, e.g. If [@s >= @b], then ... [@p >= 0]), +/// the checkable condition, and the natural-language rationale shown to the user on violation. +/// Contracts are vanilla Universalis, per the paper. +/// +public sealed record ContractClause( + Statement? Guard, + Statement Condition, + string Rationale); + +/// A complete Universalis program with optional pre/post-condition contracts. +public sealed record UniversalisProgram( + ImmutableArray Items, + ImmutableArray Pre, + ImmutableArray Post) +{ + public static UniversalisProgram Empty { get; } = new([], [], []); +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ParamMode +{ + In, + Out, +} + +public sealed record RuleParam(string Name, ParamMode Mode); + +public sealed record RuleSignature(string Name, ImmutableArray Params, string Description); + +/// +/// A stored rule head :- body: the durable, reusable unit of the self-learning system. +/// Rules shadow tools of the same name at dispatch time. +/// +public sealed record RuleDefinition(RuleSignature Signature, string HeadProse, UniversalisProgram Body); diff --git a/src/Universalis.Core/Ir/Statements.cs b/src/Universalis.Core/Ir/Statements.cs new file mode 100644 index 0000000..5fc6b76 --- /dev/null +++ b/src/Universalis.Core/Ir/Statements.cs @@ -0,0 +1,56 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace Universalis.Core.Ir; + +/// +/// The executable content of one hedge [...]. Fixed-mode discipline applies throughout: +/// inputs must be fully instantiated, dataflow runs left to right, matching is one-way. +/// +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$kind")] +[JsonDerivedType(typeof(PredicateCall), "call")] +[JsonDerivedType(typeof(IsBinding), "is")] +[JsonDerivedType(typeof(Comparison), "compare")] +[JsonDerivedType(typeof(BindStmt), "bind")] +[JsonDerivedType(typeof(DisplayStmt), "display")] +public abstract record Statement; + +/// +/// A predicate call [NAME(arg, ...)] — a tool invocation, a stored-rule invocation, or +/// (in a rule definition head position) the rule head. Out-arguments are fresh variables or +/// patterns over fresh variables; the tool/rule signature declares parameter modes. +/// +public sealed record PredicateCall(string Name, ImmutableArray Args) : Statement; + +/// Arithmetic binding [@d is (@s - @b)]: evaluates the ground right side, binds the variable. +public sealed record IsBinding(string Var, ArithExpr Expr) : Statement; + +/// +/// A comparison [@s >= @b]. Equality/inequality are numeric-aware structural comparisons; +/// orderings require numeric coercion of both sides. +/// +public sealed record Comparison(CompareOp Op, Term Left, Term Right) : Statement; + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum CompareOp +{ + Eq, + Neq, + Lt, + Le, + Gt, + Ge, +} + +/// +/// One-way match [@x = t] (also written == when testing): a free-variable left side +/// binds; a pattern left side destructures; ground-ground compares for equality. The right side +/// must always be fully instantiated (fixed mode — no unification). +/// +public sealed record BindStmt(Term Left, Term Right) : Statement; + +/// +/// Display expression [@x]: the engine evaluates the term and shows the value to the +/// user only (the paper's ⇝) — never to the model, preserving the register discipline. +/// +public sealed record DisplayStmt(Term Value) : Statement; diff --git a/src/Universalis.Core/Ir/Terms.cs b/src/Universalis.Core/Ir/Terms.cs new file mode 100644 index 0000000..4be987c --- /dev/null +++ b/src/Universalis.Core/Ir/Terms.cs @@ -0,0 +1,97 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace Universalis.Core.Ir; + +/// +/// A term appearing inside a hedge: a variable, a literal, or a (possibly nested) pattern. +/// Terms form the argument space of predicates and the operand space of bindings and matches. +/// +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$kind")] +[JsonDerivedType(typeof(VarTerm), "var")] +[JsonDerivedType(typeof(StrTerm), "str")] +[JsonDerivedType(typeof(NumTerm), "num")] +[JsonDerivedType(typeof(BoolTerm), "bool")] +[JsonDerivedType(typeof(NullTerm), "null")] +[JsonDerivedType(typeof(ObjectPatternTerm), "object")] +[JsonDerivedType(typeof(ArrayPatternTerm), "array")] +[JsonDerivedType(typeof(ExprTerm), "expr")] +[JsonDerivedType(typeof(NamedTerm), "named")] +public abstract record Term; + +/// +/// A named argument name: value inside a predicate call. The papers use positional +/// arguments, but models write named ones instinctively (observed live, repeatedly) — the +/// evaluator maps them onto the signature's parameters by name. Only legal in call argument +/// position. +/// +public sealed record NamedTerm(string Name, Term Value) : Term; + +/// A variable reference @name; excludes the @ sigil. +public sealed record VarTerm(string Name) : Term; + +public sealed record StrTerm(string Value) : Term; + +/// +/// A numeric literal. Literals always fit (the parser rejects others); +/// values only arise at runtime from tool results, never from source text. +/// +public sealed record NumTerm(decimal Value) : Term; + +public sealed record BoolTerm(bool Value) : Term; + +public sealed record NullTerm : Term; + +/// +/// A JSON object pattern { ... "key": term ... }. is true when an +/// ellipsis (...) appears anywhere in the pattern body: open patterns resolve each key by +/// pre-order depth-first descent from the matched node; closed patterns require the exact key set. +/// +public sealed record ObjectPatternTerm(ImmutableArray Fields, bool IsOpen) : Term; + +public sealed record PatternField(string Key, Term Value); + +/// +/// A JSON array pattern [a, b] / [..., x] / [x, ...]. A closed pattern +/// requires exact length; a leading (trailing) ellipsis matches a suffix (prefix). +/// +public sealed record ArrayPatternTerm(ImmutableArray Items, EllipsisPosition Ellipsis) : Term; + +public enum EllipsisPosition +{ + None, + Leading, + Trailing, +} + +/// +/// A compound arithmetic expression used as a comparison operand, e.g. the right side of +/// [@total >= @price * @count]. Simple operands parse as plain terms instead. +/// +public sealed record ExprTerm(ArithExpr Expr) : Term; + +/// Arithmetic expression: the right side of is and compound comparison operands. +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$kind")] +[JsonDerivedType(typeof(ArithVar), "var")] +[JsonDerivedType(typeof(ArithNum), "num")] +[JsonDerivedType(typeof(ArithBinary), "binary")] +[JsonDerivedType(typeof(ArithNeg), "neg")] +public abstract record ArithExpr; + +public sealed record ArithVar(string Name) : ArithExpr; + +public sealed record ArithNum(decimal Value) : ArithExpr; + +public sealed record ArithBinary(ArithOp Op, ArithExpr Left, ArithExpr Right) : ArithExpr; + +public sealed record ArithNeg(ArithExpr Operand) : ArithExpr; + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ArithOp +{ + Add, + Sub, + Mul, + Div, + Mod, +} diff --git a/src/Universalis.Core/Parsing/ComprehensionCompiler.cs b/src/Universalis.Core/Parsing/ComprehensionCompiler.cs new file mode 100644 index 0000000..f6729b1 --- /dev/null +++ b/src/Universalis.Core/Parsing/ComprehensionCompiler.cs @@ -0,0 +1,281 @@ +using System.Collections.Immutable; + +using Universalis.Core.Ir; + +namespace Universalis.Core.Parsing; + +/// +/// Compiles a structurally-recognized comprehension draft into executable s +/// by the paper's phrase conventions: "Retain/Keep only …" filters, "Group … by …" pivots, +/// "Determine/Find/Compute the <agg> … as [{field: @var}]" aggregates, "Collect … as […]" +/// nesting, "increment [@total] by one" counting. Unrecognized bullets WITH hedges are errors +/// (fed back to the model as teaching hints); pure-narration bullets are skipped. +/// +public static class ComprehensionCompiler +{ + public static (ComprehensionBlock? Block, string? Error) Compile(ComprehensionDraft draft) + { + if (draft.SourceVar is null) + { + return (null, "the query has no source collection ('Consider each [@x] from [@source]:')"); + } + + var ops = ImmutableArray.CreateBuilder(); + + foreach (var bullet in draft.Bullets) + { + var prose = bullet.Prose.ToLowerInvariant(); + var lead = prose.TrimStart('-', ' ', '\n', '\r', '\t'); + var statements = bullet.Hedges.Select(h => h.Statement).ToList(); + + // Retain/Keep only … [predicate] (…and [predicate]) — checked FIRST so + // "Keep only groups where …" never reads as a grouping. + if (prose.Contains("retain only", StringComparison.Ordinal) || prose.Contains("keep only", StringComparison.Ordinal)) + { + var predicates = statements.Where(s => s is Comparison or BindStmt).ToList(); + + if (predicates.Count == 0) + { + return (null, $"the filter needs a condition hedge like [@city = \"Palo Alto\"]: {bullet.Prose}"); + } + + foreach (var predicate in predicates) + { + ops.Add(new FilterOp(predicate, bullet.Prose)); + } + + // Fused filter+count (observed live): "- Retain only … [@city = "Palo Alto"] and + // increment [@total] by one for each." — the count rides in the same bullet. + if (prose.Contains("increment", StringComparison.Ordinal) && + CountTarget(statements, draft.ItemVar) is { } fusedTarget) + { + ops.Add(new CountIntoOp(fusedTarget, bullet.Prose)); + } + + continue; + } + + // "- If [condition], then increment/skip …" (observed live, customers count): a query + // has no branches. Models that already stated the query declaratively often re-narrate + // it as a loop walkthrough — if this bullet's conditions merely restate registered + // filters (verbatim, or negated with skip/exclude prose), it is narration: skip it, + // harvesting a count the walkthrough carries that the declaration lacked. Anything + // else is an error — an If bullet whose prose says "increment" would otherwise compile + // as an UNCONDITIONAL count, a silently wrong answer. + if (System.Text.RegularExpressions.Regex.IsMatch(lead, @"^(?:if|otherwise)\b") && + statements.Any(s => s is not DisplayStmt)) + { + var conditions = statements.Where(s => s is Comparison or BindStmt).ToList(); + var skipProse = prose.Contains("skip", StringComparison.Ordinal) || + prose.Contains("exclude", StringComparison.Ordinal) || + prose.Contains("ignore", StringComparison.Ordinal) || + prose.Contains("discard", StringComparison.Ordinal) || + prose.Contains("continue", StringComparison.Ordinal) || + prose.Contains("next", StringComparison.Ordinal); + + if (conditions.Count > 0 && + conditions.All(c => CoveredByFilters(ops, c, skipProse))) + { + if (prose.Contains("increment", StringComparison.Ordinal) && + CountTarget(statements, draft.ItemVar) is { } narratedTarget && + !ops.OfType().Any(op => op.TargetVar == narratedTarget)) + { + ops.Add(new CountIntoOp(narratedTarget, bullet.Prose)); + } + + continue; + } + + var condition = bullet.Hedges + .FirstOrDefault(h => h.Statement is Comparison or BindStmt)?.ConcreteText.Trim(); + + return (null, "a query checklist has no 'If' bullets — filter on its own bullet " + + $"'- Retain only the matching items [{condition ?? "@field = \"value\""}].' " + + "and then count with '- Subsequently, increment [@total] by one.'"); + } + + // Group … by … [@Key] — sentence-initial only: "played BY any player in this GROUP" + // must not read as a grouping. + if (lead.StartsWith("group", StringComparison.Ordinal) && prose.Contains(" by ", StringComparison.Ordinal)) + { + var key = statements.OfType().Select(d => d.Value).OfType().LastOrDefault(); + + if (key is null) + { + return (null, $"the grouping needs a key variable, e.g. 'Group each item by [@key]': {bullet.Prose}"); + } + + ops.Add(new GroupByOp(key.Name, bullet.Prose)); + continue; + } + + // increment [@total] by one … + if (prose.Contains("increment", StringComparison.Ordinal)) + { + // Fused count+filter (observed live): "… where [@city = "Palo Alto"] and + // increment [@total] by one …" — dropping the condition would count every row. + foreach (var predicate in statements.Where(s => s is Comparison or BindStmt)) + { + ops.Add(new FilterOp(predicate, bullet.Prose)); + } + + var target = CountTarget(statements, draft.ItemVar); + + if (target is null) + { + return (null, $"the count needs a target variable, e.g. 'increment [@total] by one': {bullet.Prose}"); + } + + ops.Add(new CountIntoOp(target, bullet.Prose)); + continue; + } + + // Aggregates and Collect: "… [@over] … as [{ \"field\": @var }] …" + var resultField = statements + .OfType() + .Select(d => d.Value) + .OfType() + .Where(p => p.Fields.Length == 1 && p.Fields[0].Value is VarTerm) + .Select(p => (p.Fields[0].Key, ((VarTerm)p.Fields[0].Value).Name)) + .FirstOrDefault(); + + var overVar = statements.OfType().Select(d => d.Value).OfType().FirstOrDefault(); + + if (prose.Contains("collect", StringComparison.Ordinal)) + { + if (resultField == default) + { + return (null, $"collect needs 'as [{{ \"field\": @var }}]': {bullet.Prose}"); + } + + ops.Add(new CollectOp(overVar?.Name ?? draft.ItemVar, resultField.Key, resultField.Name, bullet.Prose)); + continue; + } + + var fn = DetectAggregate(prose); + + if (fn is not null) + { + if (resultField == default) + { + return (null, $"the aggregate needs 'as [{{ \"field\": @var }}]': {bullet.Prose}"); + } + + if (fn != AggregateFn.Count && overVar is null) + { + return (null, $"the aggregate needs the row variable to aggregate, e.g. [@stats]: {bullet.Prose}"); + } + + ops.Add(new AggregateOp(fn.Value, overVar?.Name ?? draft.ItemVar, resultField.Key, resultField.Name, bullet.Prose)); + continue; + } + + // Counter-initialization narration (observed live): "- Initially, [@total = 0]." — a + // loopless query zero-initializes its counts itself; tolerate, don't error. + if ((lead.StartsWith("initial", StringComparison.Ordinal) || lead.StartsWith("start", StringComparison.Ordinal)) && + statements.All(s => s is DisplayStmt + or BindStmt { Left: VarTerm, Right: NumTerm { Value: 0 } } + or IsBinding { Expr: ArithNum { Value: 0 } })) + { + continue; + } + + if (statements.Count > 0 && statements.Any(s => s is not DisplayStmt)) + { + return (null, $"could not understand this query bullet — use Retain only/Group by/Determine…as/Collect…as/increment: {bullet.Prose}"); + } + + // Pure narration ("This ensures that …") — skipped. + } + + return (new ComprehensionBlock(draft.ItemVar, draft.ItemPattern, draft.SourceVar, ops.ToImmutable(), null), null); + } + + /// + /// The variable a count binds into: the first displayed variable that is NOT the row variable + /// ("increment [@total] by one for each customer [@c]" must pick @total), falling back to the + /// first displayed variable. + /// + private static string? CountTarget(List statements, string itemVar) + { + var displays = statements.OfType().Select(d => d.Value).OfType() + .Select(v => v.Name).ToList(); + + return displays.FirstOrDefault(n => n != itemVar) ?? displays.FirstOrDefault(); + } + + /// + /// True when a re-narrated If condition merely restates a registered filter: verbatim, the + /// same equality in either spelling (⟨@x = "v"⟩ binds, ⟨@x == "v"⟩ compares), or the + /// NEGATION when the prose says to skip/exclude the row ("If [@city != "Palo Alto"], then + /// skip" is the filter, stated from the other side). + /// + private static bool CoveredByFilters(IEnumerable ops, Statement condition, bool skipProse) + { + var canon = CanonEquality(condition); + + foreach (var filter in ops.OfType()) + { + if (filter.Predicate.Equals(condition)) + { + return true; + } + + if (canon is null || CanonEquality(filter.Predicate) is not { } fc) + { + continue; + } + + if (fc.Var == canon.Value.Var && fc.Value.Equals(canon.Value.Value) && + (fc.Equal == canon.Value.Equal || skipProse)) + { + return true; + } + } + + return false; + } + + /// Scalar equality in canonical form, across its two statement spellings. + private static (string Var, Term Value, bool Equal)? CanonEquality(Statement s) => s switch + { + BindStmt { Left: VarTerm v, Right: var t } when t is StrTerm or NumTerm or BoolTerm => (v.Name, t, true), + Comparison { Op: CompareOp.Eq, Left: VarTerm v, Right: var t } => (v.Name, t, true), + Comparison { Op: CompareOp.Neq, Left: VarTerm v, Right: var t } => (v.Name, t, false), + _ => null, + }; + + /// Order matters: "minimum number of games" must read as Min, not Count. + private static AggregateFn? DetectAggregate(string prose) + { + if (prose.Contains("minimum", StringComparison.Ordinal) || prose.Contains("min ", StringComparison.Ordinal) || + prose.Contains("least", StringComparison.Ordinal) || prose.Contains("fewest", StringComparison.Ordinal)) + { + return AggregateFn.Min; + } + + if (prose.Contains("maximum", StringComparison.Ordinal) || prose.Contains("max ", StringComparison.Ordinal) || + prose.Contains("largest", StringComparison.Ordinal) || prose.Contains("highest", StringComparison.Ordinal)) + { + return AggregateFn.Max; + } + + if (prose.Contains("average", StringComparison.Ordinal) || prose.Contains("mean", StringComparison.Ordinal)) + { + return AggregateFn.Mean; + } + + if (prose.Contains("sum", StringComparison.Ordinal) || prose.Contains("total", StringComparison.Ordinal)) + { + return AggregateFn.Sum; + } + + if (prose.Contains("count", StringComparison.Ordinal) || prose.Contains("how many", StringComparison.Ordinal) || + prose.Contains("number of", StringComparison.Ordinal)) + { + return AggregateFn.Count; + } + + return null; + } +} diff --git a/src/Universalis.Core/Parsing/HedgeParser.cs b/src/Universalis.Core/Parsing/HedgeParser.cs new file mode 100644 index 0000000..18973d4 --- /dev/null +++ b/src/Universalis.Core/Parsing/HedgeParser.cs @@ -0,0 +1,840 @@ +using System.Collections.Immutable; +using System.Globalization; +using System.Text; + +using Universalis.Core.Ir; + +namespace Universalis.Core.Parsing; + +public sealed record HedgeParseResult(Statement? Statement, string? Error, bool Healed) +{ + public bool Success => Statement is not null; + + public static HedgeParseResult Ok(Statement statement, bool healed = false) => new(statement, null, healed); + + public static HedgeParseResult Fail(string error) => new(null, error, false); +} + +/// +/// Recursive-descent parser for the content of a single hedge (brackets excluded). Documented +/// tolerances for 8B-model robustness: a single surplus trailing ) is healed (the paper +/// itself prints one), commas are optional around ... and before }, keywords are +/// case-insensitive, and … is accepted for .... +/// +public static class HedgeParser +{ + public static HedgeParseResult Parse(string content) + { + // Imperative-accumulator instincts (observed live: ⟨@xs += @x⟩, ⟨@xs is []⟩, + // ⟨@xs is @xs + [@x]⟩): there is no mutation — teach the query checklist instead. + if (content.Contains("+=", StringComparison.Ordinal) || + System.Text.RegularExpressions.Regex.IsMatch(content, @"\bis\s*\[|\+\s*\[")) + { + return HedgeParseResult.Fail( + "there are no accumulators — never build a list element by element; process a whole " + + "collection with the query checklist 'Consider each … from …:' and bullets such as " + + "'- Collect … as …' or '- Subsequently, increment … by one'"); + } + + // A whole checklist bullet inside one hedge (observed live: ⟨- If ⟨@city = "Palo Alto"⟩, + // then retain @c⟩): 'If' is prose, never hedge content — teach the bullet shape instead of + // the dead-end "found 'If'" token error. + if (System.Text.RegularExpressions.Regex.IsMatch( + content, @"^\s*-?\s*(?:if|otherwise)\b(?!\s*\()", + System.Text.RegularExpressions.RegexOptions.IgnoreCase)) + { + return HedgeParseResult.Fail( + "'If' is prose, never code — write the bullet as plain text '- If [condition], " + + "then …' with ONLY the condition inside brackets; in a query, filter with " + + "'- Retain only … [condition]'"); + } + + // Imperative-verb instinct (observed live: ⟨increment @total by 1⟩): mutation words are + // prose. A bare identifier can only ever start a call 'NAME(...)', so the '(' lookahead + // keeps real tool calls parsing. + if (System.Text.RegularExpressions.Regex.IsMatch( + content, @"^\s*(?:increment|decrement|add|set|update|initialize|initialise|reset|append)\b(?!\s*\()", + System.Text.RegularExpressions.RegexOptions.IgnoreCase)) + { + return HedgeParseResult.Fail( + "imperative words are prose, never code — count inside a query with the bullet " + + "'- Subsequently, increment [@total] by one' (only the variable in brackets); " + + "otherwise compute a NEW value with [@newVar is …]"); + } + + List tokens; + try + { + tokens = Tokenize(content); + } + catch (FormatException ex) + { + return HedgeParseResult.Fail(ex.Message); + } + + var parser = new Parser(tokens); + try + { + var statement = parser.ParseStatement(); + + // Tolerance: exactly one surplus trailing ')' is healed (cf. the paper's "(@D/@B)*100)"). + var healed = false; + if (parser.Current.Kind == TokenKind.RParen && parser.Peek(1).Kind == TokenKind.End) + { + parser.Advance(); + healed = true; + } + + if (parser.Current.Kind == TokenKind.Comma) + { + // Models with Prolog instincts write conjunctions: [F(@a), G(@b)]. Teach, don't guess. + return HedgeParseResult.Fail( + "a hedge contains exactly ONE call or calculation — never join two with a comma; " + + "write each call in its own [ ... ] hedge"); + } + + if (parser.Current.Kind == TokenKind.Ident && + parser.Current.Text.Equals("for", StringComparison.OrdinalIgnoreCase)) + { + // Python-comprehension instinct: [F(@x, @y) for @x in @xs]. Teach lifting. + return HedgeParseResult.Fail( + "there are no for-loops — call the tool ONCE with the LIST variable itself " + + "and the engine applies it to every element"); + } + + if (parser.Current.Kind != TokenKind.End) + { + if (statement is IsBinding && + parser.Current.Kind is TokenKind.Lt or TokenKind.Le or TokenKind.Gt or TokenKind.Ge or TokenKind.EqEq or TokenKind.Neq) + { + // Boolean-variable instinct (observed live): [@canBuy is @btc >= @cost]. + return HedgeParseResult.Fail( + "'is' computes a VALUE and there are no boolean variables — write the decision " + + "as a checklist: '- If ⟨condition⟩, then …' and '- Otherwise, …' bullets"); + } + + return HedgeParseResult.Fail($"unexpected '{parser.Current.Text}' after complete statement"); + } + + return HedgeParseResult.Ok(statement, healed); + } + catch (FormatException ex) + { + // Call-wrapped binding instinct (observed live: [MATH(@costPerShare is (@shares * + // 100 / @btc))]): a call argument can never contain 'is' or '=', so the wrapper is + // decoration around the ONE statement inside — heal to the statement the model + // meant. Only bindings unwrap; anything else keeps its original diagnostic. + var wrapped = System.Text.RegularExpressions.Regex.Match( + content, @"^\s*[A-Za-z_][A-Za-z0-9_]*\s*\((?.*)\)\s*$", + System.Text.RegularExpressions.RegexOptions.Singleline); + + if (wrapped.Success) + { + var inner = Parse(wrapped.Groups["inner"].Value); + + if (inner.Success && inner.Statement is IsBinding or BindStmt) + { + return HedgeParseResult.Ok(inner.Statement, healed: true); + } + } + + return HedgeParseResult.Fail(ex.Message); + } + } + + // ---------------------------------------------------------------- tokenizer + + private enum TokenKind + { + Ident, Var, Number, String, True, False, Null, Is, + LParen, RParen, LBrace, RBrace, LBracket, RBracket, + Comma, Colon, Ellipsis, + Plus, Minus, Star, Slash, Percent, + Assign, EqEq, Neq, Lt, Le, Gt, Ge, + End, + } + + private readonly record struct Token(TokenKind Kind, string Text, decimal Number); + + private static List Tokenize(string s) + { + var tokens = new List(); + var i = 0; + + while (i < s.Length) + { + var c = s[i]; + + if (char.IsWhiteSpace(c)) + { + i++; + continue; + } + + switch (c) + { + case '(': tokens.Add(new(TokenKind.LParen, "(", 0)); i++; continue; + case ')': tokens.Add(new(TokenKind.RParen, ")", 0)); i++; continue; + case '{': tokens.Add(new(TokenKind.LBrace, "{", 0)); i++; continue; + case '}': tokens.Add(new(TokenKind.RBrace, "}", 0)); i++; continue; + case '[': tokens.Add(new(TokenKind.LBracket, "[", 0)); i++; continue; + case ']': tokens.Add(new(TokenKind.RBracket, "]", 0)); i++; continue; + case ',': tokens.Add(new(TokenKind.Comma, ",", 0)); i++; continue; + case ':': tokens.Add(new(TokenKind.Colon, ":", 0)); i++; continue; + case '+': tokens.Add(new(TokenKind.Plus, "+", 0)); i++; continue; + case '-': tokens.Add(new(TokenKind.Minus, "-", 0)); i++; continue; + case '*': tokens.Add(new(TokenKind.Star, "*", 0)); i++; continue; + case '/': tokens.Add(new(TokenKind.Slash, "/", 0)); i++; continue; + case '%': tokens.Add(new(TokenKind.Percent, "%", 0)); i++; continue; + case '…': tokens.Add(new(TokenKind.Ellipsis, "...", 0)); i++; continue; + + case '.': + if (i + 2 < s.Length && s[i + 1] == '.' && s[i + 2] == '.') + { + tokens.Add(new(TokenKind.Ellipsis, "...", 0)); + i += 3; + continue; + } + + // Field-access instinct in all its variants (observed live: @x."close"). + throw new FormatException( + "there is no dot-path field access — destructure JSON with a pattern: " + + "{ ... \"field\": @field ... } in the tool call's output position, or match [@obj = { ... }]"); + + case '=': + if (i + 1 < s.Length && s[i + 1] == '=') + { + tokens.Add(new(TokenKind.EqEq, "==", 0)); + i += 2; + } + else + { + tokens.Add(new(TokenKind.Assign, "=", 0)); + i++; + } + + continue; + + case '!': + if (i + 1 < s.Length && s[i + 1] == '=') + { + tokens.Add(new(TokenKind.Neq, "!=", 0)); + i += 2; + continue; + } + + throw new FormatException("unexpected '!'"); + + case '<': + if (i + 1 < s.Length && s[i + 1] == '=') + { + tokens.Add(new(TokenKind.Le, "<=", 0)); + i += 2; + } + else + { + tokens.Add(new(TokenKind.Lt, "<", 0)); + i++; + } + + continue; + + case '>': + if (i + 1 < s.Length && s[i + 1] == '=') + { + tokens.Add(new(TokenKind.Ge, ">=", 0)); + i += 2; + } + else + { + tokens.Add(new(TokenKind.Gt, ">", 0)); + i++; + } + + continue; + + case '"': + tokens.Add(new(TokenKind.String, ReadString(s, ref i), 0)); + continue; + + case '@': + { + i++; + var name = ReadIdent(s, ref i); + if (name.Length == 0) + { + throw new FormatException("'@' must be followed by a variable name"); + } + + ThrowIfDotPath(name); + tokens.Add(new(TokenKind.Var, name, 0)); + continue; + } + + case '$': + { + // Sigil tolerance (observed live): money-priming prose makes models write + // ⟨$cash is 120⟩ for @cash, ⟨@price is $200⟩ for a dollar amount, and even + // stacked ⟨$@price is 300⟩. '$name' reads as a variable; '$123' as the + // bare number; a '$' directly before '@' is dropped. + i++; + + if (i < s.Length && (char.IsAsciiDigit(s[i]) || s[i] == '@')) + { + continue; + } + + var name = ReadIdent(s, ref i); + if (name.Length == 0) + { + throw new FormatException("'$' must be followed by a variable name or an amount"); + } + + ThrowIfDotPath(name); + tokens.Add(new(TokenKind.Var, name, 0)); + continue; + } + } + + if (char.IsAsciiDigit(c)) + { + var start = i; + while (i < s.Length && (char.IsAsciiDigit(s[i]) || s[i] == '.')) + { + // Don't swallow an ellipsis that follows a number ("1..."). + if (s[i] == '.' && i + 1 < s.Length && s[i + 1] == '.') + { + break; + } + + i++; + } + + // Tolerate exponent notation from model output. + if (i < s.Length && (s[i] is 'e' or 'E') && i + 1 < s.Length && (char.IsAsciiDigit(s[i + 1]) || s[i + 1] is '+' or '-')) + { + i += 2; + while (i < s.Length && char.IsAsciiDigit(s[i])) + { + i++; + } + } + + var text = s[start..i]; + if (!decimal.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var value)) + { + throw new FormatException($"invalid number '{text}'"); + } + + tokens.Add(new(TokenKind.Number, text, value)); + continue; + } + + if (char.IsLetter(c) || c == '_') + { + var ident = ReadIdent(s, ref i); + var kind = ident.ToLowerInvariant() switch + { + "is" => TokenKind.Is, + "true" => TokenKind.True, + "false" => TokenKind.False, + "null" => TokenKind.Null, + _ => TokenKind.Ident, + }; + tokens.Add(new(kind, ident, 0)); + continue; + } + + if (c == '?') + { + // Ternary instinct (observed live): [@left is @a >= @b ? @x : @y]. + throw new FormatException( + "there is no '?:' ternary — write the decision as a checklist: " + + "'- If ⟨condition⟩, then …' and '- Otherwise, …' bullets"); + } + + throw new FormatException($"unexpected character '{c}'"); + } + + tokens.Add(new(TokenKind.End, "", 0)); + return tokens; + } + + /// + /// Dot-path instinct (observed live: ⟨@weatherData.properties.observation.condition⟩, + /// ⟨@player.position⟩): there is no field access — teach pattern destructuring instead. + /// ReadIdent keeps consuming the dots so the WHOLE path lands in the teaching message. + /// + private static void ThrowIfDotPath(string name) + { + if (name.Contains('.')) + { + throw new FormatException( + $"there is no dot-path field access like '@{name}' — destructure JSON with a pattern: " + + "put { ... \"field\": @field ... } in the tool call's output position instead"); + } + } + + private static string ReadIdent(string s, ref int i) + { + var start = i; + while (i < s.Length && (char.IsLetterOrDigit(s[i]) || s[i] == '_' || + (s[i] == '.' && i > start && i + 1 < s.Length && char.IsLetter(s[i + 1])))) + { + i++; + } + + return s[start..i]; + } + + private static string ReadString(string s, ref int i) + { + i++; // opening quote + var sb = new StringBuilder(); + + while (i < s.Length) + { + var c = s[i]; + + if (c == '"') + { + i++; + return sb.ToString(); + } + + if (c == '\\' && i + 1 < s.Length) + { + i++; + var e = s[i]; + switch (e) + { + case '"': sb.Append('"'); break; + case '\\': sb.Append('\\'); break; + case '/': sb.Append('/'); break; + case 'n': sb.Append('\n'); break; + case 'r': sb.Append('\r'); break; + case 't': sb.Append('\t'); break; + case 'b': sb.Append('\b'); break; + case 'f': sb.Append('\f'); break; + case 'u' when i + 4 < s.Length && + ushort.TryParse(s.AsSpan(i + 1, 4), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var code): + sb.Append((char)code); + i += 4; + break; + default: + // Unknown escape: preserve BOTH characters. Models write Windows paths + // with single backslashes ("C:\Users\…") — swallowing the backslash + // silently corrupts them (observed live). + sb.Append('\\').Append(e); + break; + } + + i++; + continue; + } + + sb.Append(c); + i++; + } + + throw new FormatException("unterminated string literal"); + } + + // ---------------------------------------------------------------- parser + + private sealed class Parser + { + private readonly List _tokens; + private int _pos; + + public Parser(List tokens) => _tokens = tokens; + + public Token Current => _tokens[_pos]; + + public Token Peek(int n) => _tokens[Math.Min(_pos + n, _tokens.Count - 1)]; + + public void Advance() => _pos++; + + private Token Expect(TokenKind kind, string what) + { + if (Current.Kind != kind) + { + throw new FormatException($"expected {what}, found '{Current.Text}'"); + } + + var t = Current; + Advance(); + return t; + } + + public Statement ParseStatement() + { + // Predicate call: IDENT '(' ... ')' + if (Current.Kind == TokenKind.Ident && Peek(1).Kind == TokenKind.LParen) + { + return ParseCall(); + } + + // [@x is ] + if (Current.Kind == TokenKind.Var && Peek(1).Kind == TokenKind.Is) + { + var name = Current.Text; + Advance(); // var + Advance(); // is + + // Natural-syntax tolerance (observed live): [@x is TOOL(args)] ≡ [TOOL(args, @x)] + // for a single-output call — models write the assignment form instinctively. + if (Current.Kind == TokenKind.Ident && Peek(1).Kind == TokenKind.LParen) + { + var call = ParseCall(); + return call with { Args = call.Args.Add(new VarTerm(name)) }; + } + + var expr = ParseArith(); + return new IsBinding(name, expr); + } + + var left = ParseOperand(); + + switch (Current.Kind) + { + case TokenKind.Assign: + Advance(); + return new BindStmt(left, ParseOperand()); + case TokenKind.EqEq: + Advance(); + return new Comparison(CompareOp.Eq, left, ParseOperand()); + case TokenKind.Neq: + Advance(); + return new Comparison(CompareOp.Neq, left, ParseOperand()); + case TokenKind.Lt: + Advance(); + return new Comparison(CompareOp.Lt, left, ParseOperand()); + case TokenKind.Le: + Advance(); + return new Comparison(CompareOp.Le, left, ParseOperand()); + case TokenKind.Gt: + Advance(); + return new Comparison(CompareOp.Gt, left, ParseOperand()); + case TokenKind.Ge: + Advance(); + return new Comparison(CompareOp.Ge, left, ParseOperand()); + default: + return new DisplayStmt(left); + } + } + + private PredicateCall ParseCall() + { + var name = Expect(TokenKind.Ident, "predicate name").Text; + Expect(TokenKind.LParen, "'('"); + + var args = ImmutableArray.CreateBuilder(); + + if (Current.Kind != TokenKind.RParen) + { + while (true) + { + args.Add(ParseOperand()); + + if (Current.Kind == TokenKind.Comma) + { + Advance(); + continue; + } + + break; + } + } + + Expect(TokenKind.RParen, "')'"); + return new PredicateCall(name, args.ToImmutable()); + } + + /// + /// Parses a term with arithmetic capability: a plain term stays a plain term; consuming + /// any arithmetic operator promotes the result to . + /// + private Term ParseOperand() + { + // Named-argument style (src: "a.txt") is a strong model instinct — accept it; the + // evaluator maps names onto the signature's parameters. + if (Current.Kind == TokenKind.Ident && Peek(1).Kind == TokenKind.Colon) + { + var name = Current.Text; + Advance(); // ident + Advance(); // ':' + return new NamedTerm(name, ParseOperand()); + } + + // Patterns and strings can never start arithmetic. + switch (Current.Kind) + { + case TokenKind.LBrace: + return ParseObjectPattern(); + case TokenKind.LBracket: + return ParseArrayPattern(); + case TokenKind.String: + { + var t = new StrTerm(Current.Text); + Advance(); + return t; + } + case TokenKind.True: + Advance(); + return new BoolTerm(true); + case TokenKind.False: + Advance(); + return new BoolTerm(false); + case TokenKind.Null: + Advance(); + return new NullTerm(); + } + + var expr = ParseArith(); + + // Collapse trivial expressions back to plain terms. + return expr switch + { + ArithVar v => new VarTerm(v.Name), + ArithNum n => new NumTerm(n.Value), + _ => new ExprTerm(expr), + }; + } + + private ArithExpr ParseArith() + { + var left = ParseArithTerm(); + + while (Current.Kind is TokenKind.Plus or TokenKind.Minus) + { + var op = Current.Kind == TokenKind.Plus ? ArithOp.Add : ArithOp.Sub; + Advance(); + left = new ArithBinary(op, left, ParseArithTerm()); + } + + return left; + } + + private ArithExpr ParseArithTerm() + { + var left = ParseArithFactor(); + + while (Current.Kind is TokenKind.Star or TokenKind.Slash or TokenKind.Percent) + { + var op = Current.Kind switch + { + TokenKind.Star => ArithOp.Mul, + TokenKind.Slash => ArithOp.Div, + _ => ArithOp.Mod, + }; + Advance(); + left = new ArithBinary(op, left, ParseArithFactor()); + } + + return left; + } + + private ArithExpr ParseArithFactor() + { + switch (Current.Kind) + { + case TokenKind.Minus: + Advance(); + return new ArithNeg(ParseArithFactor()); + + case TokenKind.Number: + { + var value = Current.Number; + Advance(); + return new ArithNum(value); + } + + case TokenKind.Var: + { + var name = Current.Text; + Advance(); + return new ArithVar(name); + } + + case TokenKind.LParen: + { + Advance(); + var inner = ParseArith(); + Expect(TokenKind.RParen, "')'"); + return inner; + } + + default: + throw new FormatException($"expected a value, variable, or '(', found '{Current.Text}'"); + } + } + + private ObjectPatternTerm ParseObjectPattern() + { + Expect(TokenKind.LBrace, "'{'"); + + var fields = ImmutableArray.CreateBuilder(); + var isOpen = false; + + while (Current.Kind != TokenKind.RBrace) + { + if (Current.Kind == TokenKind.Ellipsis) + { + isOpen = true; + Advance(); + + // Tolerance: commas optional around '...'. + if (Current.Kind == TokenKind.Comma) + { + Advance(); + } + + continue; + } + + string key; + if (Current.Kind is TokenKind.String or TokenKind.Ident) + { + key = Current.Text; + Advance(); + } + else if (Current.Kind == TokenKind.Var) + { + // Sigil-in-key tolerance (observed live): {@price: @msftPrice} means the + // field "price"; a bare {@price} is the field pun { "price": @price }. + key = Current.Text; + Advance(); + + if (Current.Kind != TokenKind.Colon) + { + fields.Add(new PatternField(key, new VarTerm(key))); + + if (Current.Kind == TokenKind.Comma) + { + Advance(); + } + + continue; + } + } + else + { + throw new FormatException($"expected a pattern key, found '{Current.Text}'"); + } + + Expect(TokenKind.Colon, "':'"); + var value = ParsePatternValue(); + fields.Add(new PatternField(key, value)); + + if (Current.Kind == TokenKind.Comma) + { + Advance(); + } + } + + Expect(TokenKind.RBrace, "'}'"); + return new ObjectPatternTerm(fields.ToImmutable(), isOpen); + } + + private ArrayPatternTerm ParseArrayPattern() + { + Expect(TokenKind.LBracket, "'['"); + + var items = ImmutableArray.CreateBuilder(); + var ellipsis = EllipsisPosition.None; + + if (Current.Kind == TokenKind.Ellipsis) + { + ellipsis = EllipsisPosition.Leading; + Advance(); + if (Current.Kind == TokenKind.Comma) + { + Advance(); + } + } + + while (Current.Kind != TokenKind.RBracket) + { + if (Current.Kind == TokenKind.Ellipsis) + { + if (ellipsis != EllipsisPosition.None) + { + throw new FormatException("an array pattern may contain at most one '...'"); + } + + ellipsis = EllipsisPosition.Trailing; + Advance(); + if (Current.Kind == TokenKind.Comma) + { + Advance(); + } + + continue; + } + + if (ellipsis == EllipsisPosition.Trailing) + { + throw new FormatException("no items may follow a trailing '...' in an array pattern"); + } + + items.Add(ParsePatternValue()); + + if (Current.Kind == TokenKind.Comma) + { + Advance(); + } + } + + Expect(TokenKind.RBracket, "']'"); + return new ArrayPatternTerm(items.ToImmutable(), ellipsis); + } + + private Term ParsePatternValue() + { + switch (Current.Kind) + { + case TokenKind.Var: + { + var t = new VarTerm(Current.Text); + Advance(); + return t; + } + case TokenKind.String: + { + var t = new StrTerm(Current.Text); + Advance(); + return t; + } + case TokenKind.Number: + { + var t = new NumTerm(Current.Number); + Advance(); + return t; + } + case TokenKind.Minus when Peek(1).Kind == TokenKind.Number: + Advance(); + { + var t = new NumTerm(-Current.Number); + Advance(); + return t; + } + case TokenKind.True: + Advance(); + return new BoolTerm(true); + case TokenKind.False: + Advance(); + return new BoolTerm(false); + case TokenKind.Null: + Advance(); + return new NullTerm(); + case TokenKind.LBrace: + return ParseObjectPattern(); + case TokenKind.LBracket: + return ParseArrayPattern(); + default: + throw new FormatException($"expected a pattern value, found '{Current.Text}'"); + } + } + } +} diff --git a/src/Universalis.Core/Parsing/HedgeScanner.cs b/src/Universalis.Core/Parsing/HedgeScanner.cs new file mode 100644 index 0000000..8958c36 --- /dev/null +++ b/src/Universalis.Core/Parsing/HedgeScanner.cs @@ -0,0 +1,162 @@ +using System.Collections.Immutable; + +namespace Universalis.Core.Parsing; + +public enum ScanMode +{ + Prose, + InHedge, + InHedgeString, +} + +/// Serializable scanner state — part of the checkpointed derivation state. +public readonly record struct ScannerState(ScanMode Mode, int SquareDepth, bool Escaped) +{ + public static ScannerState Initial => new(ScanMode.Prose, 0, false); +} + +public enum ScanEvent +{ + /// Character belongs to prose. + Prose, + + /// The opening [ of a hedge (not part of the hedge content). + HedgeOpened, + + /// Character belongs to the current hedge's content. + HedgeContent, + + /// + /// The closing ] that balances the hedge (not part of the content). Per the papers, + /// the engine owns this bracket: generation is cut before it, the engine executes + /// the hedge, and resumes the model by appending ] itself. + /// + HedgeClosed, +} + +/// +/// Pure incremental automaton that splits literate Universalis text into prose and hedges. +/// Only square brackets govern nesting (hedges legally contain nested [...] in JSON array +/// patterns); double-quoted strings are shielded so a ] inside a string never closes a +/// hedge. Parentheses/braces are not tracked here — the hedge parser validates those. +/// +public sealed class HedgeScanner +{ + public ScannerState State { get; private set; } = ScannerState.Initial; + + public HedgeScanner() + { + } + + public HedgeScanner(ScannerState state) => State = state; + + public ScanEvent Push(char c) + { + var (mode, depth, escaped) = State; + + switch (mode) + { + case ScanMode.Prose: + if (c == '[') + { + State = new ScannerState(ScanMode.InHedge, 1, false); + return ScanEvent.HedgeOpened; + } + + return ScanEvent.Prose; + + case ScanMode.InHedge: + switch (c) + { + case '"': + State = new ScannerState(ScanMode.InHedgeString, depth, false); + return ScanEvent.HedgeContent; + case '[': + State = new ScannerState(ScanMode.InHedge, depth + 1, false); + return ScanEvent.HedgeContent; + case ']' when depth == 1: + State = ScannerState.Initial; + return ScanEvent.HedgeClosed; + case ']': + State = new ScannerState(ScanMode.InHedge, depth - 1, false); + return ScanEvent.HedgeContent; + default: + return ScanEvent.HedgeContent; + } + + case ScanMode.InHedgeString: + if (escaped) + { + State = new ScannerState(ScanMode.InHedgeString, depth, false); + return ScanEvent.HedgeContent; + } + + if (c == '\\') + { + State = new ScannerState(ScanMode.InHedgeString, depth, true); + return ScanEvent.HedgeContent; + } + + if (c == '"') + { + State = new ScannerState(ScanMode.InHedge, depth, false); + } + + return ScanEvent.HedgeContent; + + default: + throw new InvalidOperationException($"Unknown scan mode {mode}."); + } + } + + /// + /// Splits complete (or truncated) literate text into segments. A trailing unterminated hedge + /// surfaces as a segment with set — the caller decides + /// whether that means "cut here and wait" (streaming) or "truncated generation" (recovery). + /// + public static ImmutableArray Split(string text) + { + var scanner = new HedgeScanner(); + var segments = ImmutableArray.CreateBuilder(); + var buffer = new System.Text.StringBuilder(); + var inHedge = false; + + void Flush(bool asHedge, bool open = false) + { + if (buffer.Length > 0 || asHedge) + { + segments.Add(new TextSegment(buffer.ToString(), asHedge, open)); + } + + buffer.Clear(); + } + + foreach (var c in text) + { + switch (scanner.Push(c)) + { + case ScanEvent.Prose: + buffer.Append(c); + break; + case ScanEvent.HedgeOpened: + Flush(asHedge: false); + inHedge = true; + break; + case ScanEvent.HedgeContent: + buffer.Append(c); + break; + case ScanEvent.HedgeClosed: + Flush(asHedge: true); + inHedge = false; + break; + } + } + + Flush(asHedge: inHedge, open: inHedge); + + return segments.ToImmutable(); + } +} + +/// A prose or hedge span produced by . Hedge text excludes the brackets. +public sealed record TextSegment(string Text, bool IsHedge, bool IsOpenHedge = false); diff --git a/src/Universalis.Core/Parsing/LiterateRecognizer.cs b/src/Universalis.Core/Parsing/LiterateRecognizer.cs new file mode 100644 index 0000000..27412af --- /dev/null +++ b/src/Universalis.Core/Parsing/LiterateRecognizer.cs @@ -0,0 +1,521 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; + +using Universalis.Core.Ir; + +namespace Universalis.Core.Parsing; + +/// What the recognizer tells the kernel to do with content that just arrived. +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$kind")] +[JsonDerivedType(typeof(ProseEmitted), "prose")] +[JsonDerivedType(typeof(ExecuteHedge), "execute")] +[JsonDerivedType(typeof(ConditionalCompleted), "conditional")] +[JsonDerivedType(typeof(ComprehensionCompleted), "comprehension")] +public abstract record RecognizerEvent; + +/// Top-level prose to append to the program. +public sealed record ProseEmitted(string Text) : RecognizerEvent; + +/// A top-level hedge to execute immediately (the standard interception path). +public sealed record ExecuteHedge(HedgeItem Hedge) : RecognizerEvent; + +/// +/// A checklist finished arriving. The kernel evaluates guards in order, executes the first true +/// branch's body, and traces the untaken branches as crossed out. +/// +public sealed record ConditionalCompleted(ConditionalBlock Block) : RecognizerEvent; + +/// A comprehension finished arriving; ops compile to the LINQ pipeline (milestone P6). +public sealed record ComprehensionCompleted(ComprehensionDraft Draft) : RecognizerEvent; + +/// One operation bullet of an in-flight comprehension: its prose and embedded hedges. +public sealed record ComprehensionBullet(string Prose, ImmutableArray Hedges); + +/// +/// Structural capture of a comprehension before op-phrase compilation: header (item var/pattern, +/// source var) plus raw bullets. The P6 compiler turns bullets into s. +/// +public sealed record ComprehensionDraft( + string ItemVar, + ObjectPatternTerm? ItemPattern, + string? SourceVar, + ImmutableArray Bullets); + +public enum RecognizerContext +{ + Top, + AwaitingConditionalGuard, + InConditionalBranch, + AwaitingComprehensionItem, + AwaitingComprehensionSource, + InComprehensionBullets, +} + +/// Serializable recognizer state — part of the checkpointed derivation state. +public sealed record RecognizerState( + RecognizerContext Context, + ImmutableArray Branches, + string PendingGuardProse, + ImmutableArray CurrentBranchBody, + ComprehensionDraft? Comprehension, + string PendingBulletProse, + ImmutableArray CurrentBulletHedges) +{ + public static RecognizerState Initial { get; } = new(RecognizerContext.Top, [], "", [], null, "", []); +} + +/// +/// The incremental literate-structure classifier: consumes (prose, hedge) chunks as generation +/// streams in and recognizes the block conventions AROUND hedges — - If [guard], then ... +/// checklists and Consider each ... from [...]: comprehensions. Purely structural: it +/// buffers block content and emits completed blocks; guard evaluation, branch gating, and +/// comprehension execution belong to the kernel. +/// +public sealed partial class LiterateRecognizer +{ + public RecognizerState State { get; private set; } + + public LiterateRecognizer() => State = RecognizerState.Initial; + + public LiterateRecognizer(RecognizerState state) => State = state; + + [GeneratedRegex(@"(^|\n)\s*-\s*If\b[^\n]*$", RegexOptions.IgnoreCase)] + private static partial Regex IfBulletLeadIn(); + + [GeneratedRegex(@"(^|\n)\s*-\s*(Otherwise|Else)\b[^\n]*$", RegexOptions.IgnoreCase)] + private static partial Regex ElseBulletLeadIn(); + + [GeneratedRegex(@"(^|\n)\s*-\s*[^\n]*$")] + private static partial Regex AnyBulletLeadIn(); + + [GeneratedRegex(@"Consider\s+each\b[^\n\[]*$", RegexOptions.IgnoreCase)] + private static partial Regex ComprehensionLeadIn(); + + [GeneratedRegex(@"\bfrom\b[^\n\[\]]*$", RegexOptions.IgnoreCase)] + private static partial Regex FromLeadIn(); + + /// + /// Feeds the next chunk: the prose that preceded a hedge, and the hedge itself (null when the + /// generation ended with trailing prose). Returns directives in order. + /// + public ImmutableArray Advance(string prose, HedgeItem? hedge) + { + var events = ImmutableArray.CreateBuilder(); + + ProcessProse(prose, events); + + if (hedge is not null) + { + ProcessHedge(hedge, events); + } + + return events.ToImmutable(); + } + + /// Generation ended: close any open block. + public ImmutableArray Finish() + { + var events = ImmutableArray.CreateBuilder(); + CloseOpenBlock(events); + return events.ToImmutable(); + } + + // ---------------------------------------------------------------- prose + + private void ProcessProse(string prose, ImmutableArray.Builder events) + { + if (prose.Length == 0) + { + return; + } + + // Inside a block, a non-bulleted line at column 0 after a newline closes the block; the + // remainder flows to the top level. Bullet lines extend the block. + if (State.Context != RecognizerContext.Top) + { + var boundary = FindBlockBoundary(prose); + + if (boundary >= 0) + { + var inside = prose[..boundary]; + var outside = prose[boundary..]; + + AbsorbBlockProse(inside, events); + CloseOpenBlock(events); + ProcessProse(outside, events); + return; + } + + AbsorbBlockProse(prose, events); + return; + } + + // Top level: does the prose tail open a block for the NEXT hedge? + if (IfBulletLeadIn().IsMatch(prose)) + { + var split = LastBulletStart(prose); + EmitTopProse(prose[..split], events); + State = State with { Context = RecognizerContext.AwaitingConditionalGuard, PendingGuardProse = prose[split..] }; + return; + } + + var comprehension = ComprehensionLeadIn().Match(prose); + if (comprehension.Success) + { + // The lead-in text lives in the block (the renderer re-synthesizes the header). + EmitTopProse(prose[..comprehension.Index], events); + State = State with { Context = RecognizerContext.AwaitingComprehensionItem }; + return; + } + + EmitTopProse(prose, events); + } + + /// Index of the first newline followed by a block-closing line, or -1. + private int FindBlockBoundary(string prose) + { + var index = 0; + + while (true) + { + var nl = prose.IndexOf('\n', index); + if (nl < 0) + { + return -1; + } + + var lineStart = nl + 1; + var rest = prose[lineStart..]; + var line = rest.Split('\n', 2)[0]; + + if (line.TrimStart().StartsWith('-')) + { + index = lineStart; // another bullet: still inside the block + continue; + } + + if (line.Trim().Length == 0) + { + index = lineStart; // blank line: tolerated inside the block + continue; + } + + if (char.IsWhiteSpace(line[0])) + { + index = lineStart; // indented continuation of the current bullet + continue; + } + + // Inside a comprehension, the paper writes some sections at column 0 without + // bullets ("Group each player …", "For each group …:", "Keep only groups where …"). + if (State.Context is RecognizerContext.InComprehensionBullets or + RecognizerContext.AwaitingComprehensionItem or + RecognizerContext.AwaitingComprehensionSource && + IsComprehensionSection(line)) + { + index = lineStart; + continue; + } + + return lineStart; // column-0 non-bullet content: the block ended before this line + } + } + + private static int LastBulletStart(string prose) + { + var match = AnyBulletLeadIn().Match(prose); + return match.Success ? match.Index + match.Groups[1].Length : prose.Length; + } + + private static readonly string[] s_sectionKeywords = + [ + "group ", "for each", "keep only", "retain only", "collect ", + "determine ", "find ", "compute ", "increment ", "subsequently", + ]; + + private static bool IsComprehensionSection(string line) + { + var trimmed = line.TrimStart(); + + if (trimmed.StartsWith('-')) + { + return true; + } + + return s_sectionKeywords.Any(k => trimmed.StartsWith(k, StringComparison.OrdinalIgnoreCase)); + } + + /// + /// Start index of the LAST bullet or section within the chunk, or -1. Sections start at line + /// starts AND at sentence starts (". Subsequently, increment …") — 8B models flow bullets + /// into running prose (observed live). + /// + private static int LastSectionStart(string prose) + { + var best = -1; + var lineStart = 0; + + while (lineStart <= prose.Length) + { + var lineEnd = prose.IndexOf('\n', lineStart); + var line = lineEnd < 0 ? prose[lineStart..] : prose[lineStart..lineEnd]; + + if (line.Trim().Length > 0 && IsComprehensionSection(line)) + { + best = lineStart; + } + + // Sentence starts within the line: ". Keyword …" + for (var dot = line.IndexOf('.'); dot >= 0 && dot + 1 < line.Length; dot = line.IndexOf('.', dot + 1)) + { + var rest = line[(dot + 1)..]; + + if (rest.TrimStart().Length > 0 && IsComprehensionSection(rest)) + { + var offset = dot + 1; + while (offset < line.Length && char.IsWhiteSpace(line[offset])) + { + offset++; + } + + best = lineStart + offset; + } + } + + if (lineEnd < 0) + { + break; + } + + lineStart = lineEnd + 1; + } + + return best; + } + + private void AbsorbBlockProse(string prose, ImmutableArray.Builder events) + { + switch (State.Context) + { + case RecognizerContext.AwaitingConditionalGuard: + State = State with { PendingGuardProse = State.PendingGuardProse + prose }; + break; + + case RecognizerContext.InConditionalBranch: + if (IfBulletLeadIn().IsMatch(prose) || ElseBulletLeadIn().IsMatch(prose)) + { + // A new bullet: seal the current branch. + var split = LastBulletStart(prose); + SealCurrentBranch(prose[..split]); + + var lead = prose[split..]; + + if (ElseBulletLeadIn().IsMatch(prose)) + { + // Else-branch has no guard hedge: open it directly. + State = State with + { + Context = RecognizerContext.InConditionalBranch, + Branches = State.Branches.Add(new ConditionalBranch(null, lead, [])), + PendingGuardProse = "", + CurrentBranchBody = [], + }; + } + else + { + State = State with { Context = RecognizerContext.AwaitingConditionalGuard, PendingGuardProse = lead }; + } + } + else + { + State = State with { CurrentBranchBody = State.CurrentBranchBody.Add(new Comment(prose)) }; + } + + break; + + case RecognizerContext.AwaitingComprehensionItem: + // Prose between "Consider each" and the item hedge: descriptive, ignored structurally. + break; + + case RecognizerContext.AwaitingComprehensionSource: + // Expect "from" between item and source hedges; tolerated free-form. + break; + + case RecognizerContext.InComprehensionBullets: + { + // A new bullet OR a column-0 section line ("Group … by …", "Keep only groups + // where …") seals the current bullet — the paper mixes both styles. + var split = LastSectionStart(prose); + + if (split > 0 || (split == 0 && (State.PendingBulletProse.Length > 0 || State.CurrentBulletHedges.Length > 0))) + { + SealCurrentBullet(prose[..split]); + State = State with { PendingBulletProse = prose[split..] }; + } + else + { + State = State with { PendingBulletProse = State.PendingBulletProse + prose }; + } + + break; + } + } + } + + private void EmitTopProse(string prose, ImmutableArray.Builder events) + { + if (prose.Length > 0) + { + events.Add(new ProseEmitted(prose)); + } + } + + // ---------------------------------------------------------------- hedges + + private void ProcessHedge(HedgeItem hedge, ImmutableArray.Builder events) + { + switch (State.Context) + { + case RecognizerContext.Top: + events.Add(new ExecuteHedge(hedge)); + break; + + case RecognizerContext.AwaitingConditionalGuard: + State = State with + { + Context = RecognizerContext.InConditionalBranch, + Branches = State.Branches.Add(new ConditionalBranch(hedge.Statement, State.PendingGuardProse, [])), + PendingGuardProse = "", + CurrentBranchBody = [], + }; + break; + + case RecognizerContext.InConditionalBranch: + State = State with { CurrentBranchBody = State.CurrentBranchBody.Add(hedge) }; + break; + + case RecognizerContext.AwaitingComprehensionItem: + { + // Header hedge: [@c = { pattern }] or [@c]. + var (itemVar, pattern) = hedge.Statement switch + { + BindStmt { Left: VarTerm v, Right: ObjectPatternTerm p } => (v.Name, (ObjectPatternTerm?)p), + DisplayStmt { Value: VarTerm v } => (v.Name, null), + _ => (null, null), + }; + + if (itemVar is null) + { + // Not actually a comprehension header — degrade gracefully to top level. + State = State with { Context = RecognizerContext.Top }; + events.Add(new ExecuteHedge(hedge)); + return; + } + + State = State with + { + Context = RecognizerContext.AwaitingComprehensionSource, + Comprehension = new ComprehensionDraft(itemVar, pattern, null, []), + }; + break; + } + + case RecognizerContext.AwaitingComprehensionSource: + { + if (hedge.Statement is DisplayStmt { Value: VarTerm src }) + { + State = State with + { + Context = RecognizerContext.InComprehensionBullets, + Comprehension = State.Comprehension! with { SourceVar = src.Name }, + PendingBulletProse = "", + CurrentBulletHedges = [], + }; + } + else + { + // Malformed header: degrade to top level. + State = State with { Context = RecognizerContext.Top, Comprehension = null }; + events.Add(new ExecuteHedge(hedge)); + } + + break; + } + + case RecognizerContext.InComprehensionBullets: + State = State with { CurrentBulletHedges = State.CurrentBulletHedges.Add(hedge) }; + break; + } + } + + // ---------------------------------------------------------------- block closing + + private void SealCurrentBranch(string trailingProse) + { + if (State.Branches.Length == 0) + { + return; + } + + var body = trailingProse.Trim().Length > 0 + ? State.CurrentBranchBody.Add(new Comment(trailingProse)) + : State.CurrentBranchBody; + + var sealedBranch = State.Branches[^1] with { Body = body }; + + State = State with + { + Branches = State.Branches.SetItem(State.Branches.Length - 1, sealedBranch), + CurrentBranchBody = [], + }; + } + + private void SealCurrentBullet(string trailingProse) + { + var prose = (State.PendingBulletProse + trailingProse).Trim(); + + // Skip structural noise (e.g. the ":" between the header and the first bullet). + if (State.CurrentBulletHedges.Length == 0 && !prose.Any(char.IsLetter)) + { + return; + } + + State = State with + { + Comprehension = State.Comprehension! with + { + Bullets = State.Comprehension.Bullets.Add(new ComprehensionBullet(prose, State.CurrentBulletHedges)), + }, + PendingBulletProse = "", + CurrentBulletHedges = [], + }; + } + + private void CloseOpenBlock(ImmutableArray.Builder events) + { + switch (State.Context) + { + case RecognizerContext.AwaitingConditionalGuard or RecognizerContext.InConditionalBranch: + SealCurrentBranch(""); + + if (State.Branches.Length > 0) + { + events.Add(new ConditionalCompleted(new ConditionalBlock(State.Branches))); + } + + State = RecognizerState.Initial; + break; + + case RecognizerContext.InComprehensionBullets: + SealCurrentBullet(""); + events.Add(new ComprehensionCompleted(State.Comprehension!)); + State = RecognizerState.Initial; + break; + + case RecognizerContext.AwaitingComprehensionItem or RecognizerContext.AwaitingComprehensionSource: + State = RecognizerState.Initial; + break; + } + } +} diff --git a/src/Universalis.Core/Parsing/UniversalisParser.cs b/src/Universalis.Core/Parsing/UniversalisParser.cs new file mode 100644 index 0000000..459e876 --- /dev/null +++ b/src/Universalis.Core/Parsing/UniversalisParser.cs @@ -0,0 +1,68 @@ +using System.Collections.Immutable; + +using Universalis.Core.Ir; + +namespace Universalis.Core.Parsing; + +public sealed record ProgramParseResult( + UniversalisProgram? Program, + ImmutableArray Warnings, + string? Error) +{ + public bool Success => Program is not null; +} + +/// +/// Batch parser for complete literate Universalis text (flat prose + hedges; structured blocks — +/// conditionals, comprehensions — are recognized by the streaming LiterateRecognizer). +/// Tolerance: a hedge whose content fails to parse and contains no variables or calls is +/// reclassified as prose with a warning instead of failing the program. +/// +public static class UniversalisParser +{ + public static ProgramParseResult ParseProgram(string text) + { + var items = ImmutableArray.CreateBuilder(); + var warnings = ImmutableArray.CreateBuilder(); + + foreach (var segment in HedgeScanner.Split(text)) + { + if (!segment.IsHedge) + { + items.Add(new Comment(segment.Text)); + continue; + } + + if (segment.IsOpenHedge) + { + return new ProgramParseResult(null, warnings.ToImmutable(), $"unterminated hedge: [{segment.Text}"); + } + + var parsed = HedgeParser.Parse(segment.Text); + + if (parsed.Success) + { + if (parsed.Healed) + { + warnings.Add($"healed a surplus ')' in [{segment.Text}]"); + } + + items.Add(new HedgeItem(parsed.Statement!, segment.Text)); + continue; + } + + // Degenerate hedge (markdown damage etc.): no variables, no call shape → prose. + if (!segment.Text.Contains('@', StringComparison.Ordinal) && + !segment.Text.Contains('(', StringComparison.Ordinal)) + { + warnings.Add($"treating unparseable hedge as prose: [{segment.Text}] ({parsed.Error})"); + items.Add(new Comment("[" + segment.Text + "]")); + continue; + } + + return new ProgramParseResult(null, warnings.ToImmutable(), $"cannot parse hedge [{segment.Text}]: {parsed.Error}"); + } + + return new ProgramParseResult(new UniversalisProgram(items.ToImmutable(), [], []), warnings.ToImmutable(), null); + } +} diff --git a/src/Universalis.Core/Rendering/AnswerAssembler.cs b/src/Universalis.Core/Rendering/AnswerAssembler.cs new file mode 100644 index 0000000..d3627ea --- /dev/null +++ b/src/Universalis.Core/Rendering/AnswerAssembler.cs @@ -0,0 +1,65 @@ +using System.Text; +using System.Text.RegularExpressions; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; + +namespace Universalis.Core.Rendering; + +/// +/// Assembles the user-facing literate answer from a completed program: prose verbatim (engine +/// asides excluded), display hedges AND value-producing bindings replaced inline by their +/// formatted values (the paper's values view: "the profit is $[@D is (@S-@B)]" reads +/// "the profit is $7"), tool calls and guards elided. +/// +public static partial class AnswerAssembler +{ + public static string Assemble(IEnumerable items, EvalEnv env) + { + var sb = new StringBuilder(); + + foreach (var item in items) + { + switch (item) + { + case Comment { Aside: true }: + break; // engine steering: model-facing, never user-facing + + case Comment c: + sb.Append(c.Text); + break; + + case HedgeItem { Statement: DisplayStmt display }: + var outcome = Evaluator.Evaluate(display, env, SignatureCatalog.Empty); + sb.Append(outcome is DisplayValue dv ? dv.Formatted : "?"); + break; + + case HedgeItem { Statement: IsBinding isb } when env.IsBound(isb.Var): + sb.Append(Evaluator.FormatForDisplay(env.GetNode(isb.Var))); + break; + + case HedgeItem { Statement: BindStmt { Left: VarTerm bound } } when env.IsBound(bound.Name): + sb.Append(Evaluator.FormatForDisplay(env.GetNode(bound.Name))); + break; + + // Tool calls and guards are elided from the answer. + } + } + + return NormalizeWhitespace(sb.ToString()); + } + + private static string NormalizeWhitespace(string text) + { + var collapsed = Whitespace().Replace(text, " ").Trim(); + + // Rejoin punctuation that hedge elision left dangling ("there . The" → "there. The"). + return SpaceBeforePunctuation().Replace(collapsed, "$1"); + } + + [GeneratedRegex(@"\s+")] + private static partial Regex Whitespace(); + + [GeneratedRegex(@"\s+([.,;:!?])")] + private static partial Regex SpaceBeforePunctuation(); +} diff --git a/src/Universalis.Core/Rendering/ConcreteRenderer.cs b/src/Universalis.Core/Rendering/ConcreteRenderer.cs new file mode 100644 index 0000000..b774e75 --- /dev/null +++ b/src/Universalis.Core/Rendering/ConcreteRenderer.cs @@ -0,0 +1,262 @@ +using System.Text; +using System.Text.Json.Nodes; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; + +namespace Universalis.Core.Rendering; + +public enum RenderMode +{ + /// The paper's concrete syntax: [@P is ((@D / @B) * 100)]. + Formulas, + + /// The live-programming view with σ substituted: [70 is ((7 / 10) * 100)]. + Values, +} + +/// +/// Renders the intentional representation back to concrete literate syntax. Formulas mode prefers +/// the preserved (lossless); Values mode re-renders with σ +/// bindings substituted, the paper's formulas⇄values toggle. +/// +public static class ConcreteRenderer +{ + public static string RenderProgram(UniversalisProgram program, RenderMode mode, EvalEnv? env = null) + { + var sb = new StringBuilder(); + + foreach (var item in program.Items) + { + AppendItem(sb, item, mode, env); + } + + return sb.ToString(); + } + + private static void AppendItem(StringBuilder sb, ProgramItem item, RenderMode mode, EvalEnv? env) + { + switch (item) + { + case Comment c: + sb.Append(c.Text); + break; + + case HedgeItem h: + sb.Append('[').Append(RenderHedge(h, mode, env)).Append(']'); + break; + + case ConditionalBlock cond: + foreach (var branch in cond.Branches) + { + // GuardProse preserves the bullet lead-in verbatim ("\n- If "). + sb.Append(branch.GuardProse.Length > 0 + ? branch.GuardProse + : branch.Guard is null ? "\n- Otherwise, " : "\n- If "); + + if (branch.Guard is not null) + { + sb.Append('[').Append(RenderStatement(branch.Guard, mode, env)).Append(']'); + } + + foreach (var inner in branch.Body) + { + AppendItem(sb, inner, mode, env); + } + } + + break; + + case ComprehensionBlock comp: + sb.Append("Consider each [@").Append(comp.ItemVar); + + if (comp.ItemPattern is not null) + { + sb.Append(" = ").Append(RenderTerm(comp.ItemPattern, RenderMode.Formulas, null)); + } + + sb.Append("] from [@").Append(comp.SourceVar).Append("]:"); + + foreach (var op in comp.Ops) + { + sb.Append("\n- ").Append(op.Prose); + } + + sb.Append('\n'); + break; + } + } + + public static string RenderHedge(HedgeItem hedge, RenderMode mode, EvalEnv? env = null) + { + if (!string.IsNullOrEmpty(hedge.ConcreteText)) + { + // Formulas: the preserved source, losslessly. Values: the paper substitutes bound + // values INTO the original formula shape ([@D is (@S-@B)] → [7 is (17-10)]), so we + // substitute textually in the preserved source rather than re-render the tree. + return mode == RenderMode.Formulas + ? hedge.ConcreteText + : SubstituteVars(hedge.ConcreteText, env); + } + + return RenderStatement(hedge.Statement, mode, env); + } + + /// + /// Replaces bound @var occurrences with rendered values. NB: a bound variable's name + /// inside a string literal would also be substituted — acceptable at POC fidelity. + /// + private static string SubstituteVars(string concreteText, EvalEnv? env) + { + if (env is null) + { + return concreteText; + } + + return System.Text.RegularExpressions.Regex.Replace( + concreteText, + "@([A-Za-z_][A-Za-z0-9_]*)", + m => env.IsBound(m.Groups[1].Value) ? RenderValue(env.GetNode(m.Groups[1].Value)) : m.Value); + } + + public static string RenderStatement(Statement statement, RenderMode mode, EvalEnv? env = null) => statement switch + { + IsBinding isb => $"{RenderVar(isb.Var, mode, env)} is {RenderArith(isb.Expr, mode, env, parent: null)}", + Comparison cmp => $"{RenderTerm(cmp.Left, mode, env)} {RenderOp(cmp.Op)} {RenderTerm(cmp.Right, mode, env)}", + BindStmt bind => $"{RenderTerm(bind.Left, mode, env)} = {RenderTerm(bind.Right, mode, env)}", + DisplayStmt disp => RenderTerm(disp.Value, mode, env), + PredicateCall call => $"{call.Name}({string.Join(", ", call.Args.Select(a => RenderTerm(a, mode, env)))})", + _ => throw new NotSupportedException(statement.GetType().Name), + }; + + private static string RenderOp(CompareOp op) => op switch + { + CompareOp.Eq => "==", + CompareOp.Neq => "!=", + CompareOp.Lt => "<", + CompareOp.Le => "<=", + CompareOp.Gt => ">", + _ => ">=", + }; + + public static string RenderTerm(Term term, RenderMode mode, EvalEnv? env) => term switch + { + NamedTerm n => $"{n.Name}: {RenderTerm(n.Value, mode, env)}", + VarTerm v => RenderVar(v.Name, mode, env), + StrTerm s => Quote(s.Value), + NumTerm n => NumericOps.CanonicalText(n.Value), + BoolTerm b => b.Value ? "true" : "false", + NullTerm => "null", + ObjectPatternTerm o => RenderObjectPattern(o, mode, env), + ArrayPatternTerm a => RenderArrayPattern(a, mode, env), + ExprTerm e => RenderArith(e.Expr, mode, env, parent: null), + _ => throw new NotSupportedException(term.GetType().Name), + }; + + private static string RenderVar(string name, RenderMode mode, EvalEnv? env) + { + if (mode == RenderMode.Values && env is not null && env.IsBound(name)) + { + return RenderValue(env.GetNode(name)); + } + + return "@" + name; + } + + /// A bound value rendered as a literal: strings quoted, numbers raw, structures compact JSON. + public static string RenderValue(JsonNode? node) => node switch + { + null => "null", + JsonValue v when v.GetValueKind() == System.Text.Json.JsonValueKind.String => Quote(v.GetValue()), + _ => node.ToJsonString(EvalEnv.JsonTextOptions), + }; + + private static string Quote(string s) => + "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal) + "\""; + + private static string RenderObjectPattern(ObjectPatternTerm pattern, RenderMode mode, EvalEnv? env) + { + var sb = new StringBuilder("{ "); + + if (pattern.IsOpen) + { + sb.Append("... "); + } + + for (var i = 0; i < pattern.Fields.Length; i++) + { + if (i > 0) + { + sb.Append(pattern.IsOpen ? " ... " : ", "); + } + + sb.Append(Quote(pattern.Fields[i].Key)).Append(": ").Append(RenderTerm(pattern.Fields[i].Value, mode, env)); + } + + if (pattern.IsOpen) + { + sb.Append(" ..."); + } + + sb.Append(" }"); + return sb.ToString(); + } + + private static string RenderArrayPattern(ArrayPatternTerm pattern, RenderMode mode, EvalEnv? env) + { + var items = pattern.Items.Select(i => RenderTerm(i, mode, env)); + + return pattern.Ellipsis switch + { + EllipsisPosition.Leading => "[..., " + string.Join(", ", items) + "]", + EllipsisPosition.Trailing => "[" + string.Join(", ", items) + ", ...]", + _ => "[" + string.Join(", ", items) + "]", + }; + } + + private static string RenderArith(ArithExpr expr, RenderMode mode, EvalEnv? env, ArithExpr? parent) + { + switch (expr) + { + case ArithNum n: + return NumericOps.CanonicalText(n.Value); + + case ArithVar v: + return RenderVar(v.Name, mode, env); + + case ArithNeg neg: + return "-" + RenderArith(neg.Operand, mode, env, expr); + + case ArithBinary bin: + { + var op = bin.Op switch + { + ArithOp.Add => "+", + ArithOp.Sub => "-", + ArithOp.Mul => "*", + ArithOp.Div => "/", + _ => "%", + }; + + var left = RenderArith(bin.Left, mode, env, bin); + var right = bin.Right is ArithBinary rb && Precedence(rb.Op) <= Precedence(bin.Op) + ? "(" + RenderArith(bin.Right, mode, env, null) + ")" + : RenderArith(bin.Right, mode, env, bin); + + if (bin.Left is ArithBinary lb && Precedence(lb.Op) < Precedence(bin.Op)) + { + left = "(" + left + ")"; + } + + var rendered = $"{left} {op} {right}"; + + return parent is ArithNeg ? "(" + rendered + ")" : rendered; + } + + default: + throw new NotSupportedException(expr.GetType().Name); + } + } + + private static int Precedence(ArithOp op) => op is ArithOp.Mul or ArithOp.Div or ArithOp.Mod ? 2 : 1; +} diff --git a/src/Universalis.Core/Universalis.Core.csproj b/src/Universalis.Core/Universalis.Core.csproj new file mode 100644 index 0000000..8a990f7 --- /dev/null +++ b/src/Universalis.Core/Universalis.Core.csproj @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/tests/Automind.Integration.Tests/Automind.Integration.Tests.csproj b/tests/Automind.Integration.Tests/Automind.Integration.Tests.csproj new file mode 100644 index 0000000..2d17260 --- /dev/null +++ b/tests/Automind.Integration.Tests/Automind.Integration.Tests.csproj @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/tests/Automind.Integration.Tests/MemoryPagerTests.cs b/tests/Automind.Integration.Tests/MemoryPagerTests.cs new file mode 100644 index 0000000..358e14d --- /dev/null +++ b/tests/Automind.Integration.Tests/MemoryPagerTests.cs @@ -0,0 +1,86 @@ +using Automind.Memory; + +namespace Automind.Integration.Tests; + +/// +/// Exercises the in-process ONNX embedding pipeline (bge-micro-v2) — no Ollama involved at all. +/// Skipped as inconclusive when the model files haven't been fetched. +/// +[TestClass] +public sealed class MemoryPagerTests +{ + private static string ModelDirectory => + Environment.GetEnvironmentVariable("AUTOMIND_EMBEDDINGS") ?? @"D:\poc2\models\bge-micro-v2"; + + private static async Task RequirePagerAsync() + { + var pager = await OnnxMemoryPager.TryCreateAsync(ModelDirectory); + + if (pager is null) + { + Assert.Inconclusive($"embedding model not found in {ModelDirectory} — run scripts/fetch-embedding-model.ps1"); + } + + return pager!; + } + + [TestMethod] + [TestCategory("RequiresEmbeddingModel")] + [Timeout(120_000)] + public async Task Recall_FindsTheSemanticallyRelevantChunk() + { + using var pager = await RequirePagerAsync(); + + await pager.IndexAsync(new MemoryChunk("rule:profitpct", MemoryChunk.RuleKind, "profitpct", + "computes the profit percentage from a buying price and a selling price")); + await pager.IndexAsync(new MemoryChunk("rule:cityweather", MemoryChunk.RuleKind, "cityweather", + "looks up the current weather conditions for a city")); + await pager.IndexAsync(new MemoryChunk("doc:reaqtor#0", MemoryChunk.DocKind, "reaqtor-notes", + "Reaqtor is a reliable, stateful, distributed event processing engine built on Rx. " + + "Its checkpointing lets standing queries survive process failures.")); + + var money = await pager.RecallAsync("how much profit margin did the sale make", top: 1); + Assert.AreEqual("rule:profitpct", money[0].Id); + + var outside = await pager.RecallAsync("is it raining in Seattle today", top: 1); + Assert.AreEqual("rule:cityweather", outside[0].Id); + + var tech = await pager.RecallAsync("which engine keeps event queries alive across crashes", top: 1); + Assert.AreEqual("doc:reaqtor#0", tech[0].Id); + } + + [TestMethod] + [TestCategory("RequiresEmbeddingModel")] + [Timeout(120_000)] + public async Task RuleLibraryBeyondContextBudget_StillRetrievesTheRightRule() + { + using var pager = await RequirePagerAsync(); + + // Simulate a rule library too large to advertise wholesale: 40 distractors + 1 target. + for (var i = 0; i < 40; i++) + { + await pager.IndexAsync(new MemoryChunk($"rule:noise{i}", MemoryChunk.RuleKind, $"noise{i}", + $"utility rule number {i} for miscellaneous bookkeeping task {i}")); + } + + await pager.IndexAsync(new MemoryChunk("rule:tempconvert", MemoryChunk.RuleKind, "tempconvert", + "converts a temperature from Fahrenheit to Celsius degrees")); + + var recalled = await pager.RecallAsync("turn 80 degrees Fahrenheit into Celsius", top: 3); + + Assert.IsTrue(recalled.Any(c => c.Id == "rule:tempconvert"), + $"expected tempconvert among: {string.Join(", ", recalled.Select(c => c.Id))}"); + } + + [TestMethod] + public void ChunkDocument_SplitsOnParagraphs() + { + var text = string.Join("\n\n", Enumerable.Range(0, 10).Select(i => new string((char)('a' + i), 400))); + + var chunks = OnnxMemoryPager.ChunkDocument("doc", text, maxChars: 1000).ToList(); + + Assert.IsGreaterThan(2, chunks.Count); + Assert.IsTrue(chunks.All(c => c.Text.Length <= 1300), "chunks stay near the budget"); + Assert.AreEqual("doc#0", chunks[0].Id); + } +} diff --git a/tests/Automind.Integration.Tests/OllamaLiveTests.cs b/tests/Automind.Integration.Tests/OllamaLiveTests.cs new file mode 100644 index 0000000..58b3632 --- /dev/null +++ b/tests/Automind.Integration.Tests/OllamaLiveTests.cs @@ -0,0 +1,402 @@ +using System.Collections.Immutable; + +using Automind.Kernel.Prompting; +using Automind.Reaqtor.Llm; +using Automind.Reaqtor.Reactive; +using Automind.Reaqtor.Tests; +using Automind.Tools; + +using Microsoft.Extensions.AI; + +using OllamaSharp; + +using Reaqtive.Scheduler; + +namespace Automind.Integration.Tests; + +/// +/// Live tests against local Ollama (skipped as inconclusive when unreachable). These are the +/// protocol conformance gate for a model: prefill continuation, hedge-cut streaming, and two +/// full derivations through the engine — the crucible for the 8B-protocol-conformance risk. +/// +[TestClass] +public sealed class OllamaLiveTests +{ + private static PhysicalScheduler s_scheduler = null!; + + [ClassInitialize] + public static void ClassInitialize(TestContext _) => s_scheduler = PhysicalScheduler.Create(); + + [ClassCleanup] + public static void ClassCleanup() => s_scheduler.Dispose(); + + private static string Endpoint => + Environment.GetEnvironmentVariable("AUTOMIND_OLLAMA") ?? "http://localhost:11434"; + + private static string Model => + Environment.GetEnvironmentVariable("AUTOMIND_OLLAMA_MODEL") ?? "granite3.3:8b"; + + private static IChatClient CreateChatClient() => new OllamaApiClient(new Uri(Endpoint), Model); + + private static async Task RequireOllamaAsync() + { + try + { + using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(3) }; + (await http.GetAsync(new Uri(Endpoint + "/api/tags"))).EnsureSuccessStatusCode(); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + Assert.Inconclusive($"Ollama is not reachable at {Endpoint} — live tests skipped."); + } + } + + private static readonly TimeSpan LiveTimeout = TimeSpan.FromSeconds(240); + + private static void DumpTranscript(SubstrateHarness h, string name) + { + var path = Path.Combine(Path.GetTempPath(), "automind-tests", $"transcript-{name}.txt"); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, h.Transcript()); + Console.WriteLine($"transcript: {path}"); + } + + // ================================================================ conformance gate + + [TestMethod] + [TestCategory("RequiresOllama")] + [Timeout(300_000)] + public async Task Conformance_PrefillContinuation_And_HedgeCut() + { + await RequireOllamaAsync(); + + var streamer = new OllamaSegmentStreamer(CreateChatClient(), Model); + + // 1. Prefill continuation: the model must continue the partial assistant message. + var continuation = await streamer.StreamSegmentAsync(new PromptState( + SystemPrompt: "You count numbers as lowercase words. Continue exactly where the text stops.", + QuestionText: "Count slowly from one to ten as words separated by commas.", + AssistantPrefill: "one, two, three,", + HardStopSequences: ["\nQuestion:"], + MaxSegmentTokens: 48, + Temperature: 0, + Seed: 42), CancellationToken.None); + + Assert.Contains("four", continuation.Text, $"prefill continuation broken; got: '{continuation.Text}'"); + + // 2. Hedge cut: the stream must stop exactly before the closing bracket. + var cut = await streamer.StreamSegmentAsync(new PromptState( + SystemPrompt: "Repeat the user's sentence EXACTLY, character for character. Do not add anything.", + QuestionText: "To check the weather, use [WEATHER(\"Palo Alto\", @w)] right away.", + AssistantPrefill: "", + HardStopSequences: ["\nQuestion:"], + MaxSegmentTokens: 64, + Temperature: 0, + Seed: 42), CancellationToken.None); + + Assert.IsTrue(cut.StoppedAtHedge, $"expected a hedge cut; got natural stop: '{cut.Text}'"); + Assert.EndsWith("@w)", cut.Text.TrimEnd(), $"cut should land right before ']'; got: '{cut.Text}'"); + Assert.DoesNotContain("]", cut.Text, "the closing bracket belongs to the engine"); + } + + // ================================================================ live derivations through the engine + + [TestMethod] + [TestCategory("RequiresOllama")] + [Timeout(300_000)] + public async Task LiveDerivation_Apples_PureArithmetic() + { + await RequireOllamaAsync(); + + await using var h = new SubstrateHarness( + s_scheduler, + Path.Combine(Path.GetTempPath(), "automind-tests", Guid.NewGuid().ToString("N"))); + + h.Llm = new OllamaLlmService(CreateChatClient(), Model); + foreach (var (_, tool) in PrimitiveTools.CreateDefault().All) + { + h.Tools.Add(tool); + } + + await h.StartFreshAsync(); + await h.AskAsync("apples", "Alice bought a kilo of apples for $12. She sold them for $18. How much percent profit or loss did Alice make?"); + + try + { + await h.WaitUntilAsync( + () => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0 || h.OutputsOfKind(DerivationOutput.FailedKind).Count > 0, + LiveTimeout, + "a live answer (or failure)"); + } + finally + { + DumpTranscript(h, "apples"); + } + + var failures = h.OutputsOfKind(DerivationOutput.FailedKind); + Assert.IsEmpty(failures, + $"derivation failed: {(failures.Count > 0 ? failures[0].PayloadJson : "")}\n--- transcript ---\n{h.Transcript()}"); + + var answer = h.OutputsOfKind(DerivationOutput.AnswerKind)[0].PayloadJson; + Assert.Contains("50", answer, $"expected 50% profit in: '{answer}'\n--- transcript ---\n{h.Transcript()}"); + } + + [TestMethod] + [TestCategory("RequiresOllama")] + [Timeout(300_000)] + public async Task LiveDerivation_Conditional_ChecklistDecision() + { + await RequireOllamaAsync(); + + await using var h = new SubstrateHarness( + s_scheduler, + Path.Combine(Path.GetTempPath(), "automind-tests", Guid.NewGuid().ToString("N"))); + + h.Llm = new OllamaLlmService(CreateChatClient(), Model); + foreach (var (_, tool) in PrimitiveTools.CreateDefault().All) + { + h.Tools.Add(tool); + } + + await h.StartFreshAsync(); + await h.AskAsync("ticket", "Sam has $120 and a game costs $200. Decide whether Sam can buy it, and how many dollars remain either way."); + + try + { + await h.WaitUntilAsync( + () => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0 || h.OutputsOfKind(DerivationOutput.FailedKind).Count > 0, + LiveTimeout, + "a live answer (or failure)"); + } + finally + { + DumpTranscript(h, "ticket"); + } + + var failures = h.OutputsOfKind(DerivationOutput.FailedKind); + Assert.IsEmpty(failures, + $"derivation failed: {(failures.Count > 0 ? failures[0].PayloadJson : "")}\n--- transcript ---\n{h.Transcript()}"); + + var answer = h.OutputsOfKind(DerivationOutput.AnswerKind)[0].PayloadJson; + Assert.Contains("120", answer, $"Sam cannot afford the game, so $120 remains; got: '{answer}'\n--- transcript ---\n{h.Transcript()}"); + } + + [TestMethod] + [TestCategory("RequiresOllama")] + [Timeout(300_000)] + public async Task LiveDerivation_Comprehension_CustomerCount() + { + await RequireOllamaAsync(); + + await using var h = new SubstrateHarness( + s_scheduler, + Path.Combine(Path.GetTempPath(), "automind-tests", Guid.NewGuid().ToString("N"))); + + h.Llm = new OllamaLlmService(CreateChatClient(), Model); + foreach (var (_, tool) in PrimitiveTools.CreateDefault().All) + { + h.Tools.Add(tool); + } + + await h.StartFreshAsync(); + + var envelope = new Automind.Kernel.Contract.QuestionEnvelope( + "How many of the customers in @customers live in Palo Alto?", + InitialBindings: new Dictionary + { + ["customers"] = """[{"city":"Palo Alto"},{"city":"Seattle"},{"city":"Palo Alto"},{"city":"Austin"}]""", + }.ToImmutableDictionary(), + ExpectedOutputs: ["total"], + LearnRuleOnSuccess: false); + + var topic = "automind/out/customers"; + await h.Conversations.UpsertAsync(new Automind.Reaqtor.Catalog.ConversationRecord( + "customers", topic, envelope.ToJson(), Automind.Reaqtor.Catalog.ConversationRecord.PendingStatus)); + + // Ask with the full envelope (initial bindings ride along). + await h.AskEnvelopeAsync("customers", envelope); + + try + { + await h.WaitUntilAsync( + () => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0 || h.OutputsOfKind(DerivationOutput.FailedKind).Count > 0, + LiveTimeout, + "a live answer (or failure)"); + } + finally + { + DumpTranscript(h, "customers"); + } + + var failures = h.OutputsOfKind(DerivationOutput.FailedKind); + Assert.IsEmpty(failures, + $"derivation failed: {(failures.Count > 0 ? failures[0].PayloadJson : "")}\n--- transcript ---\n{h.Transcript()}"); + + var answer = h.OutputsOfKind(DerivationOutput.AnswerKind)[0].PayloadJson; + Assert.Contains("2", answer, $"two customers live in Palo Alto; got: '{answer}'\n--- transcript ---\n{h.Transcript()}"); + } + + [TestMethod] + [TestCategory("RequiresOllama")] + [Timeout(300_000)] + public async Task LiveDerivation_ModeB_WholeProgramSynthesis() + { + await RequireOllamaAsync(); + + await using var h = new SubstrateHarness( + s_scheduler, + Path.Combine(Path.GetTempPath(), "automind-tests", Guid.NewGuid().ToString("N"))); + + h.Llm = new OllamaLlmService(CreateChatClient(), Model); + foreach (var (_, tool) in PrimitiveTools.CreateDefault().All) + { + h.Tools.Add(tool); + } + + await h.StartFreshAsync(); + + var envelope = new Automind.Kernel.Contract.QuestionEnvelope( + "Alice bought a kilo of apples for $12. She sold them for $18. How much percent profit did Alice make? Bind the percentage as @profitPct.", + InitialBindings: System.Collections.Immutable.ImmutableDictionary.Empty, + ExpectedOutputs: ["profitPct"], + LearnRuleOnSuccess: false, + ModeB: true); + + await h.Conversations.UpsertAsync(new Automind.Reaqtor.Catalog.ConversationRecord( + "modeb", "automind/out/modeb", envelope.ToJson(), Automind.Reaqtor.Catalog.ConversationRecord.PendingStatus)); + + await h.AskEnvelopeAsync("modeb", envelope); + + try + { + await h.WaitUntilAsync( + () => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0 || h.OutputsOfKind(DerivationOutput.FailedKind).Count > 0, + LiveTimeout, + "a live answer (or failure)"); + } + finally + { + DumpTranscript(h, "modeb"); + } + + var failures = h.OutputsOfKind(DerivationOutput.FailedKind); + Assert.IsEmpty(failures, + $"derivation failed: {(failures.Count > 0 ? failures[0].PayloadJson : "")}\n--- transcript ---\n{h.Transcript()}"); + + var answer = h.OutputsOfKind(DerivationOutput.AnswerKind)[0].PayloadJson; + Assert.Contains("50", answer, $"expected 50% profit in: '{answer}'\n--- transcript ---\n{h.Transcript()}"); + } + + [TestMethod] + [TestCategory("RequiresOllama")] + [Timeout(300_000)] + public async Task LiveDerivation_McpBridgedTool_InvokedAsPredicate() + { + await RequireOllamaAsync(); + + // The P9 acceptance: an MCP server's tools invocable as Universalis predicates. The + // self-contained sample server rides its own stdio child process, exactly like a real one. + var serverDll = Path.GetFullPath(Path.Combine( + AppContext.BaseDirectory, "..", "..", "..", "..", "..", + "tools", "McpSampleServer", "bin", "Debug", "net10.0", "McpSampleServer.dll")); + + if (!File.Exists(serverDll)) + { + Assert.Inconclusive($"sample MCP server not built at {serverDll}"); + } + + await using var bridge = await Automind.Mcp.McpToolBridge.ConnectAsync( + [$"dotnet \"{serverDll}\""], allowlist: null, log: _ => { }); + + Assert.IsTrue(bridge.Tools.Count >= 2, "expected the sample server's tools to bridge"); + + await using var h = new SubstrateHarness( + s_scheduler, + Path.Combine(Path.GetTempPath(), "automind-tests", Guid.NewGuid().ToString("N"))); + + h.Llm = new OllamaLlmService(CreateChatClient(), Model); + foreach (var (_, tool) in PrimitiveTools.CreateDefault().All) + { + h.Tools.Add(tool); + } + + foreach (var tool in bridge.Tools) + { + h.Tools.Add(tool); + } + + await h.StartFreshAsync(); + + var envelope = new Automind.Kernel.Contract.QuestionEnvelope( + "Reverse the word 'reaqtor' and bind the reversed text as @reversed.", + InitialBindings: System.Collections.Immutable.ImmutableDictionary.Empty, + ExpectedOutputs: ["reversed"], + LearnRuleOnSuccess: false); + + await h.Conversations.UpsertAsync(new Automind.Reaqtor.Catalog.ConversationRecord( + "mcp", "automind/out/mcp", envelope.ToJson(), Automind.Reaqtor.Catalog.ConversationRecord.PendingStatus)); + + await h.AskEnvelopeAsync("mcp", envelope); + + try + { + await h.WaitUntilAsync( + () => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0 || h.OutputsOfKind(DerivationOutput.FailedKind).Count > 0, + LiveTimeout, + "a live answer (or failure)"); + } + finally + { + DumpTranscript(h, "mcp"); + } + + var failures = h.OutputsOfKind(DerivationOutput.FailedKind); + Assert.IsEmpty(failures, + $"derivation failed: {(failures.Count > 0 ? failures[0].PayloadJson : "")}\n--- transcript ---\n{h.Transcript()}"); + + var answer = h.OutputsOfKind(DerivationOutput.AnswerKind)[0].PayloadJson; + Assert.Contains("rotqaer", answer, + $"the MCP REVERSE tool's result should surface; got: '{answer}'\n--- transcript ---\n{h.Transcript()}"); + } + + [TestMethod] + [TestCategory("RequiresOllama")] + [Timeout(300_000)] + public async Task LiveDerivation_Weather_ToolCallLoop() + { + await RequireOllamaAsync(); + + await using var h = new SubstrateHarness( + s_scheduler, + Path.Combine(Path.GetTempPath(), "automind-tests", Guid.NewGuid().ToString("N"))); + + h.Llm = new OllamaLlmService(CreateChatClient(), Model); + foreach (var (_, tool) in PrimitiveTools.CreateDefault().All) + { + h.Tools.Add(tool); + } + + await h.StartFreshAsync(); + await h.AskAsync("weather", "What is the current weather in Palo Alto?"); + + try + { + await h.WaitUntilAsync( + () => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0 || h.OutputsOfKind(DerivationOutput.FailedKind).Count > 0, + LiveTimeout, + "a live answer (or failure)"); + } + finally + { + DumpTranscript(h, "weather"); + } + + var failures = h.OutputsOfKind(DerivationOutput.FailedKind); + Assert.IsEmpty(failures, + $"derivation failed: {(failures.Count > 0 ? failures[0].PayloadJson : "")}\n--- transcript ---\n{h.Transcript()}"); + + var answer = h.OutputsOfKind(DerivationOutput.AnswerKind)[0].PayloadJson; + Assert.Contains("Sunny and 80°F", answer, + $"the canned Palo Alto weather should surface via a display hedge; got: '{answer}'\n--- transcript ---\n{h.Transcript()}"); + } +} diff --git a/tests/Automind.Kernel.Tests/Automind.Kernel.Tests.csproj b/tests/Automind.Kernel.Tests/Automind.Kernel.Tests.csproj new file mode 100644 index 0000000..148f6ad --- /dev/null +++ b/tests/Automind.Kernel.Tests/Automind.Kernel.Tests.csproj @@ -0,0 +1,7 @@ + + + + + + + diff --git a/tests/Automind.Kernel.Tests/DerivationHarness.cs b/tests/Automind.Kernel.Tests/DerivationHarness.cs new file mode 100644 index 0000000..12898eb --- /dev/null +++ b/tests/Automind.Kernel.Tests/DerivationHarness.cs @@ -0,0 +1,95 @@ +using System.Collections.Immutable; + +using Automind.Kernel; +using Automind.Kernel.Contract; +using Automind.Kernel.Prompting; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; + +namespace Automind.Kernel.Tests; + +/// +/// Golden-transcript driver: feeds scripted events through the pure step function and records +/// everything, so scenarios read as (event → expected effects + state assertions). +/// +public sealed class DerivationHarness +{ + private readonly IStepFunction _step = new DerivationStep(); + + public DerivationState State { get; private set; } + + public StepContext Context { get; } + + public List LastEffects { get; private set; } = []; + + public List Traces { get; } = []; + + public List AllEffects { get; } = []; + + public DerivationHarness(StepContext? context = null, string conversationId = "conv-1") + { + State = DerivationState.New(conversationId); + Context = context ?? DefaultContext(); + } + + public static StepContext DefaultContext() => new( + [ + new ToolBinding( + new PredicateSignature("WEATHER", [ + new PredicateParam("city", ParamMode.In), + new PredicateParam("weather", ParamMode.Out), + ], "current weather for a city"), + "automind://tools/weather", + IsIdempotent: true, + "current weather conditions for a city"), + new ToolBinding( + new PredicateSignature("STOCK", [ + new PredicateParam("symbol", ParamMode.In), + new PredicateParam("data", ParamMode.Out), + ], "latest stock quote data"), + "automind://tools/stock", + IsIdempotent: true, + "latest quote JSON for a ticker symbol"), + ], + Rules: []); + + public List Apply(DerivationEvent evt) + { + var result = _step.Step(State, evt, Context); + State = result.State; + LastEffects = [.. result.Effects]; + AllEffects.AddRange(result.Effects); + Traces.AddRange(result.Effects.OfType().Select(t => TraceEvent.FromJson(t.TraceJson))); + return LastEffects; + } + + // ---------------------------------------------------------------- convenience + + public List Ask(string question) => + Apply(new QuestionReceived(QuestionEnvelope.ForText(question).ToJson())); + + public List Ask(QuestionEnvelope envelope) => + Apply(new QuestionReceived(envelope.ToJson())); + + /// Delivers a generation segment against the currently pending LLM request. + public List Segment(string text, bool stoppedAtHedge = true, bool truncated = false) => + Apply(new LlmCompleted(PendingLlmRequestId(), text, stoppedAtHedge, truncated)); + + public List ToolResult(string requestId, params string[] resultsJson) => + Apply(new ToolSucceeded(requestId, [.. resultsJson])); + + public string PendingLlmRequestId() + { + Assert.IsInstanceOfType(State.Phase, "expected an outstanding LLM request"); + return State.PendingRequestIds.Single(); + } + + public RequestLlm LastLlmRequest() => LastEffects.OfType().Single(); + + public PromptState LastPrompt() => PromptState.FromJson(LastLlmRequest().PromptStateJson); + + public List LastToolInvocations() => [.. LastEffects.OfType()]; + + public string AnswerText() => AllEffects.OfType().Single().Text; +} diff --git a/tests/Automind.Kernel.Tests/GoldenScenarioTests.cs b/tests/Automind.Kernel.Tests/GoldenScenarioTests.cs new file mode 100644 index 0000000..3545fca --- /dev/null +++ b/tests/Automind.Kernel.Tests/GoldenScenarioTests.cs @@ -0,0 +1,787 @@ +using System.Collections.Immutable; + +using Automind.Kernel.Contract; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; + +namespace Automind.Kernel.Tests; + +[TestClass] +public sealed class GoldenScenarioTests +{ + // ================================================================ scenario 1: weather happy path + + [TestMethod] + public void Weather_HappyPath_InterceptionProtocolEndToEnd() + { + var h = new DerivationHarness(); + + // Question → the kernel issues the first generation request. + var effects = h.Ask("What is the weather between Mountain View and Menlo Park?"); + Assert.IsInstanceOfType(h.State.Phase); + var prompt0 = h.LastPrompt(); + Assert.AreEqual("", prompt0.AssistantPrefill); + Assert.Contains("WEATHER(city: in, weather: out)", prompt0.SystemPrompt); + + // Segment 1: the model reasons and opens a tool call; the bridge cuts before ']'. + h.Segment("The city between Mountain View and Menlo Park is Palo Alto. " + + "Let's find the weather there [WEATHER(\"Palo Alto\", @weatherPaloAlto)"); + + Assert.IsInstanceOfType(h.State.Phase); + var invocation = h.LastToolInvocations().Single(); + Assert.AreEqual("automind://tools/weather", invocation.ToolUri); + Assert.AreEqual("""{"city":"Palo Alto"}""", invocation.ArgsJson); + + // Tool result → σ binds, the engine closes the bracket, generation resumes. + h.ToolResult(invocation.RequestId, "\"Sunny and 80°F\""); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.AreEqual("\"Sunny and 80°F\"", h.State.Sigma["weatherPaloAlto"]); + Assert.EndsWith("]", h.LastPrompt().AssistantPrefill); + Assert.IsTrue(h.Traces.OfType().Any(t => t.Name == "weatherPaloAlto" && t.Source == "WEATHER")); + + // Segment 2: display hedge — value shown to the USER only; model resumes blind. + h.Segment(". The current weather in Palo Alto is [@weatherPaloAlto"); + + var display = h.Traces.OfType().Single(); + Assert.AreEqual("Sunny and 80°F", display.Formatted); + Assert.IsInstanceOfType(h.State.Phase); + Assert.DoesNotContain("Sunny", h.LastPrompt().AssistantPrefill, "the model must never see values"); + + // Segment 3: natural stop, no hedge → completion. + h.Segment(".", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.AreEqual( + "The city between Mountain View and Menlo Park is Palo Alto. " + + "Let's find the weather there. The current weather in Palo Alto is Sunny and 80°F.", + h.AnswerText()); + } + + // ================================================================ scenario 2: apples + inputs + expected outputs + + [TestMethod] + public void Apples_InitialBindings_ArithmeticChain_ExpectedOutputs() + { + var h = new DerivationHarness(); + + h.Ask(new QuestionEnvelope( + "Alice bought a kilo of apples for $B. She sold them for $S. How much percent profit or loss did Alice make?", + InitialBindings: new Dictionary { ["B"] = "10", ["S"] = "17" }.ToImmutableDictionary(), + ExpectedOutputs: ["P"], + LearnRuleOnSuccess: false)); + + Assert.Contains("@B = 10", h.LastPrompt().QuestionText); + + h.Segment("The apples cost $B and sold for $S, so the profit is [@D is (@S - @B)"); + Assert.AreEqual("7", h.State.Sigma["D"]); + + h.Segment(". The profit percentage is therefore [@P is (@D / @B) * 100"); + Assert.AreEqual("70", h.State.Sigma["P"]); + + h.Segment(" percent. Alice made a profit of [@P"); + h.Segment(" percent.", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.Contains("Alice made a profit of 70 percent.", h.AnswerText()); + Assert.EndsWith("@P = 70", h.AnswerText()); + } + + // ================================================================ scenario 3: tool failure → backtrack → recover + + [TestMethod] + public void ToolFailure_Backtracks_WithSteeringAndTemperatureLadder() + { + var h = new DerivationHarness(); + + h.Ask("What's the weather in Palo Alto?"); + Assert.AreEqual(0.2, h.LastPrompt().Temperature, 1e-9); + + h.Segment("Let me check [WEATHER(\"Palo Alto\", @w)"); + var requestId = h.LastToolInvocations().Single().RequestId; + + h.Apply(new ToolFailed(requestId, "HTTP 503 service unavailable")); + + // Backtracked: engine note recorded, steering sentence in the prefill, temperature laddered. + Assert.IsInstanceOfType(h.State.Phase); + Assert.IsTrue(h.Traces.OfType().Any()); + Assert.Contains("503", h.State.EngineNotes[0]); + + var retryPrompt = h.LastPrompt(); + Assert.AreEqual(0.5, retryPrompt.Temperature, 1e-9); + Assert.Contains("Let me try a different approach", retryPrompt.AssistantPrefill); + Assert.Contains("503", retryPrompt.SystemPrompt); + + // The model tries again and succeeds this time. + h.Segment("Retrying the weather service [WEATHER(\"Palo Alto\", @weatherNow)"); + h.ToolResult(h.LastToolInvocations().Single().RequestId, "\"Cloudy and 60°F\""); + h.Segment(". It is [@weatherNow"); + h.Segment(" outside.", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.Contains("Cloudy and 60°F", h.AnswerText()); + Assert.DoesNotContain("Let me check", h.AnswerText(), "the abandoned branch must not leak into the answer"); + } + + // ================================================================ scenario 4: pattern mismatch → hint feeds the model + + [TestMethod] + public void PatternMismatch_HintWithAvailableKeys_ReachesEngineNotes() + { + var h = new DerivationHarness(); + + h.Ask("What did IBM close at?"); + h.Segment("Let's look it up [STOCK(\"IBM\", { ... \"closing\": @p ... })"); + + var requestId = h.LastToolInvocations().Single().RequestId; + h.ToolResult(requestId, """{"data":[{"close":"181.58","volume":"3037600"}],"status":"ok"}"""); + + // The out-pattern didn't match: backtrack with the available-keys hint. + Assert.IsInstanceOfType(h.State.Phase); + Assert.IsTrue(h.Traces.OfType().Any(t => t.Reason.Contains("closing"))); + Assert.Contains("available keys", h.State.EngineNotes[0]); + Assert.Contains("close", h.State.EngineNotes[0]); + } + + // ================================================================ scenario 5: auto-repair (discard-and-replace) + + [TestMethod] + public void LiteralInOutPosition_AutoRepaired_NotBacktracked() + { + var h = new DerivationHarness(); + + h.Ask("Weather in Palo Alto?"); + + // Paper 1's vanilla-ReAct failure mode: the model hallucinates the result value. + h.Segment("Checking [WEATHER(\"Palo Alto\", \"Sunny and 10000°F\")"); + + Assert.IsInstanceOfType(h.State.Phase, "repair should proceed to the tool call, not backtrack"); + Assert.IsTrue(h.Traces.OfType().Any(t => t.What == "literal-in-out-position")); + Assert.IsEmpty(h.Traces.OfType().ToList()); + + // The engine's value wins; the hallucination is discarded. + h.ToolResult(h.LastToolInvocations().Single().RequestId, "\"Rainy and 40°F\""); + Assert.IsTrue(h.State.Sigma.Values.Contains("\"Rainy and 40°F\"")); + } + + // ================================================================ scenario 5b: repeat-after-success no-op + + [TestMethod] + public void RepeatedCall_WithBoundOutputs_IsNoOpNarration() + { + var h = new DerivationHarness(); + + h.Ask("Weather in Palo Alto?"); + h.Segment("Checking [WEATHER(\"Palo Alto\", @w)"); + h.ToolResult(h.LastToolInvocations().Single().RequestId, "\"Sunny and 80°F\""); + + // The live-observed pathology: the model re-narrates the completed call verbatim. + h.Segment(" Let me check the weather [WEATHER(\"Palo Alto\", @w)"); + + Assert.IsInstanceOfType(h.State.Phase, "a duplicate call must not re-invoke the tool"); + Assert.IsEmpty(h.LastToolInvocations(), "no second tool invocation"); + Assert.IsEmpty(h.Traces.OfType().ToList(), "no backtrack for redundant narration"); + Assert.IsTrue(h.Traces.OfType().Any(t => t.What == "duplicate-call")); + + // The derivation continues normally. + h.Segment(". The weather is [@w"); + h.Segment(".", stoppedAtHedge: false); + Assert.IsInstanceOfType(h.State.Phase); + Assert.Contains("Sunny and 80°F", h.AnswerText()); + } + + [TestMethod] + public void ReboundOutput_WithDifferentIntent_GetsFreshVariable() + { + var h = new DerivationHarness(); + + h.Ask("Compare weather in two cities?"); + h.Segment("First [WEATHER(\"Palo Alto\", @w)"); + h.ToolResult(h.LastToolInvocations().Single().RequestId, "\"Sunny and 80°F\""); + + // Same OUT variable, different city: a genuinely new call with a reused name. + h.Segment(". Now Seattle [WEATHER(\"Seattle\", @w)"); + + Assert.IsInstanceOfType(h.State.Phase, "a new call with a rebound output should execute under a fresh variable"); + Assert.IsTrue(h.Traces.OfType().Any(t => t.What == "rebound-output")); + Assert.AreEqual("""{"city":"Seattle"}""", h.LastToolInvocations().Single().ArgsJson); + } + + [TestMethod] + public void EngineNotesAndSteering_NeverContainSquareBrackets() + { + var h = new DerivationHarness(); + + h.Ask("What did IBM close at?"); + h.Segment("Looking it up [STOCK(\"IBM\", { ... \"closing\": @p ... })"); + h.ToolResult(h.LastToolInvocations().Single().RequestId, """{"data":[{"close":"181.58"}]}"""); + + Assert.DoesNotContain("[", h.State.EngineNotes[0], "bracketed hedges in notes get parroted and re-executed"); + Assert.DoesNotContain("[", h.LastPrompt().AssistantPrefill[h.State.ChoicePoints[^1].AssistantPrefill.Length..], + "the steering sentence must be bracket-free"); + } + + // ================================================================ scenario 5c: essays backtrack + + [TestMethod] + public void ProseOnlyCompletion_Backtracks_ActDontDescribe() + { + var h = new DerivationHarness(); + + h.Ask("Weather in Palo Alto?"); + + // Observed live: the model narrates what it WOULD do instead of doing it. + h.Segment("To answer this, one would use the WEATHER tool and report its result.", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase, "an essay must backtrack, not complete"); + Assert.IsTrue(h.Traces.OfType().Any(t => t.Reason.Contains("no executable hedges"))); + + // The steered retry acts properly and completes. + h.Segment("Checking [WEATHER(\"Palo Alto\", @w)"); + h.ToolResult(h.LastToolInvocations().Single().RequestId, "\"Sunny and 80°F\""); + h.Segment(". It is [@w"); + h.Segment(".", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.Contains("Sunny and 80°F", h.AnswerText()); + } + + // ================================================================ scenario 5d: lifted batches track outstanding work + + [TestMethod] + public void LiftedBatch_PartialResults_ShrinkPendingRequestIds() + { + var catalog = DerivationHarness.DefaultContext() with + { + Tools = + [ + new ToolBinding(new Universalis.Core.Evaluation.PredicateSignature("CONVERT", [ + new Universalis.Core.Evaluation.PredicateParam("src", Universalis.Core.Ir.ParamMode.In), + new Universalis.Core.Evaluation.PredicateParam("dst", Universalis.Core.Ir.ParamMode.Out), + ], "convert one file"), "automind://tools/convert", true, "converts one file"), + ], + }; + + var h = new DerivationHarness(catalog); + + h.Ask(QuestionEnvelope.ForText("Convert everything in @files.") with + { + InitialBindings = new Dictionary { ["files"] = """["a","b","c"]""" }.ToImmutableDictionary(), + }); + + h.Segment("Converting them all at once [CONVERT(@files, @out)"); + + var invocations = h.LastToolInvocations(); + Assert.HasCount(3, invocations, "zip lifting fans out three invocations"); + Assert.HasCount(3, h.State.PendingRequestIds); + + // One result lands: the outstanding set must SHRINK (recovery re-issues exactly these — + // a kill between partial results must not orphan the rest of the batch). + h.ToolResult(invocations[1].RequestId, "\"b.pdf\""); + + Assert.HasCount(2, h.State.PendingRequestIds); + CollectionAssert.AreEquivalent( + new[] { invocations[0].RequestId, invocations[2].RequestId }, + h.State.PendingRequestIds.ToArray()); + + h.ToolResult(invocations[0].RequestId, "\"a.pdf\""); + h.ToolResult(invocations[2].RequestId, "\"c.pdf\""); + + Assert.AreEqual("""["a.pdf","b.pdf","c.pdf"]""", h.State.Sigma["out"], "structure-of-arrays in source order"); + Assert.IsInstanceOfType(h.State.Phase); + } + + // ================================================================ scenario 6: idempotent redelivery + + [TestMethod] + public void DuplicateAndStaleEvents_AreNoOps() + { + var h = new DerivationHarness(); + + h.Ask("Weather in Palo Alto?"); + var staleLlmRequest = h.PendingLlmRequestId(); + + h.Segment("Checking [WEATHER(\"Palo Alto\", @w)"); + var toolRequest = h.LastToolInvocations().Single().RequestId; + + h.ToolResult(toolRequest, "\"Sunny\""); + var stateAfter = h.State.ToJson(); + + // Duplicate tool result: no-op. + var dup = h.Apply(new ToolSucceeded(toolRequest, ["\"Sunny\""])); + Assert.IsEmpty(dup); + Assert.AreEqual(stateAfter, h.State.ToJson()); + + // Stale LLM completion for a long-gone request: no-op. + var stale = h.Apply(new LlmCompleted(staleLlmRequest, "zombie text", true)); + Assert.IsEmpty(stale); + Assert.AreEqual(stateAfter, h.State.ToJson()); + } + + // ================================================================ scenario 7: budget exhaustion + + [TestMethod] + public void RepeatedToolFailures_RestartThenExhaustBudget_FailCleanly() + { + var h = new DerivationHarness(); + + h.Ask("Weather in Palo Alto?"); + + for (var attempt = 0; attempt < 40 && h.State.Phase is not FailedPhase; attempt++) + { + h.Segment($"Attempt {attempt} [WEATHER(\"Palo Alto\", @w{attempt})"); + + if (h.State.Phase is not AwaitingTools) + { + continue; // a restart re-asked without a tool call this round + } + + var requestId = h.LastToolInvocations().Single().RequestId; + h.Apply(new ToolFailed(requestId, "connection refused")); + } + + // Exhausting the choice-point stack with budget to spare RESTARTS the derivation + // (tree-of-thought from the root, teachings carried in the notes)… + Assert.IsTrue( + h.Traces.OfType().Any(t => t.Reason.StartsWith("fresh start", StringComparison.Ordinal)), + "expected at least one tree-of-thought restart before giving up"); + + // …and only when the request budget is truly spent does it fail, cleanly. + Assert.IsInstanceOfType(h.State.Phase); + Assert.IsTrue(h.AllEffects.OfType().Any()); + Assert.IsTrue(h.Traces.OfType().Any()); + } + + // ================================================================ scenario 8: conditional checklist (pure branches) + + [TestMethod] + public void Conditional_FirstTrueBranchRuns_OthersCrossedOut() + { + var h = new DerivationHarness(); + + h.Ask(new QuestionEnvelope( + "Erik has btc BTC. Decide whether he can buy msft MSFT.", + InitialBindings: new Dictionary { ["btcTotal"] = "900", ["msftTotal"] = "1000" }.ToImmutableDictionary(), + ExpectedOutputs: ["btcLeft"], + LearnRuleOnSuccess: false)); + + h.Segment("Now, let's compare the two values:\n- If [@btcTotal >= @msftTotal"); + h.Segment(", then Erik can buy and the remainder is [@btcLeft is @btcTotal - @msftTotal"); + h.Segment(".\n- If [@btcTotal < @msftTotal"); + h.Segment(", then Erik keeps his BTC, so [@btcLeft = @btcTotal"); + h.Segment(".\nThat settles it.", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.AreEqual("900", h.State.Sigma["btcLeft"]); + + var guards = h.Traces.OfType().Where(g => g.Branch >= 0).ToList(); + Assert.HasCount(2, guards); + Assert.IsFalse(guards[0].Result); + Assert.IsTrue(guards[1].Result); + Assert.AreEqual(1, h.Traces.OfType().Single().Branch); + Assert.AreEqual(0, h.Traces.OfType().Single().Branch); + Assert.EndsWith("@btcLeft = 900", h.AnswerText()); + } + + // ================================================================ scenario 8b: comprehension end-to-end + + [TestMethod] + public void Comprehension_Customers_ExecutesThroughTheKernel() + { + var h = new DerivationHarness(); + + h.Ask(new QuestionEnvelope( + "How many customers live in Palo Alto?", + InitialBindings: new Dictionary + { + ["customers"] = """[{"city":"Palo Alto"},{"city":"Seattle"},{"city":"Palo Alto"}]""", + }.ToImmutableDictionary(), + ExpectedOutputs: ["total"], + LearnRuleOnSuccess: false)); + + h.Segment("Consider each customer [@c = { ... \"city\": @city ... }"); + h.Segment(" from [@customers"); + h.Segment(":\n- Retain only customers [@c"); + h.Segment(" that live in Palo Alto [@city = \"Palo Alto\""); + h.Segment(".\n- Subsequently, increment [@total"); + h.Segment(" by one for each retained customer [@c"); + h.Segment(".\nThat's the count.", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.AreEqual("2", h.State.Sigma["total"]); + + var query = h.Traces.OfType().Single(); + Assert.AreEqual(3, query.RowsIn); + Assert.AreEqual(2, query.RowsOut); + Assert.EndsWith("@total = 2", h.AnswerText()); + } + + // ================================================================ scenario 8c: contracts + + [TestMethod] + public void PreConditionViolation_IsUserError_FailsWithoutRetry() + { + var h = new DerivationHarness(); + + h.Ask(new QuestionEnvelope( + "Profit for a free crate of apples?", + InitialBindings: new Dictionary { ["B"] = "0", ["S"] = "5" }.ToImmutableDictionary(), + ExpectedOutputs: [], + LearnRuleOnSuccess: false, + Pre: [new ContractClauseText("@B > 0", "the buying price must be positive — Alice paid something for the apples")])); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.IsEmpty(h.AllEffects.OfType().ToList(), "a pre-condition violation must not consume any LLM budget"); + Assert.IsTrue(h.Traces.OfType().Single(c => c.IsPre) is { Passed: false }); + Assert.Contains("buying price must be positive", ((FailedPhase)h.State.Phase).Reason); + } + + [TestMethod] + public void PostConditionViolation_Backtracks_ThenAlternativeSatisfiesIt() + { + var h = new DerivationHarness(); + + h.Ask(new QuestionEnvelope( + "Split 10 into two positive parts, the first larger.", + InitialBindings: new Dictionary { ["n"] = "10" }.ToImmutableDictionary(), + ExpectedOutputs: ["a", "b"], + LearnRuleOnSuccess: false, + Post: [new ContractClauseText("@a > @b", "the first part must be the larger one")])); + + // First attempt violates the post-condition (a=3 < b=7) → backtrack at completion. + h.Segment("Let's take [@a is 3"); + h.Segment(" and [@b is @n - @a"); + h.Segment(". Done.", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase, "post-violation must backtrack, not complete"); + Assert.IsTrue(h.Traces.OfType().Any(c => !c.IsPre && !c.Passed)); + Assert.IsTrue(h.Traces.OfType().Any(t => t.Reason.Contains("post-condition"))); + + // The alternative derivation satisfies the contract. + h.Segment("Let's take [@bigger is 7"); + h.Segment(" and [@smaller is @n - @bigger"); + h.Segment(". Setting [@a is @bigger"); + h.Segment(" and [@b is @smaller"); + h.Segment(". Done.", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.IsTrue(h.Traces.OfType().Any(c => !c.IsPre && c.Passed)); + Assert.Contains("@a = 7", h.AnswerText()); + } + + // ================================================================ scenario 8d: virtual-memory paging + + [TestMethod] + public void RecalledContextAndRules_ShapeThePrompt() + { + var context = DerivationHarness.DefaultContext() with + { + Rules = + [ + new Universalis.Core.Ir.RuleDefinition( + new Universalis.Core.Ir.RuleSignature("relevantRule", [], "the one that matters"), + "relevant", Universalis.Core.Ir.UniversalisProgram.Empty), + new Universalis.Core.Ir.RuleDefinition( + new Universalis.Core.Ir.RuleSignature("noiseRule", [], "irrelevant filler"), + "noise", Universalis.Core.Ir.UniversalisProgram.Empty), + ], + }; + + var h = new DerivationHarness(context); + + h.Ask(QuestionEnvelope.ForText("What does the handbook say about checkpoints?") with + { + Context = [new RecalledChunk("handbook", "Checkpoints persist standing queries so they survive process death.")], + RecalledRules = ["relevantRule"], + }); + + var prompt = h.LastPrompt().SystemPrompt; + + Assert.Contains("CONTEXT", prompt); + Assert.Contains("Checkpoints persist standing queries", prompt); + Assert.Contains("relevantRule", prompt); + Assert.DoesNotContain("noiseRule", prompt, "unrecalled rules are paged OUT of the context"); + } + + // ================================================================ scenario 9: replay determinism + + [TestMethod] + public void ReplayingTheSameEvents_YieldsByteIdenticalStatesAndEffects() + { + static (string FinalState, string Effects) Run() + { + var h = new DerivationHarness(); + + h.Ask("What is the weather between Mountain View and Menlo Park?"); + h.Segment("The city is Palo Alto. Checking [WEATHER(\"Palo Alto\", @w)"); + h.ToolResult(h.LastToolInvocations().Single().RequestId, "\"Sunny and 80°F\""); + h.Segment(". The weather is [@w"); + h.Segment(".", stoppedAtHedge: false); + + var effects = string.Join("\n", h.AllEffects.Select(e => e.ToString())); + return (h.State.ToJson(), effects); + } + + var first = Run(); + var second = Run(); + + Assert.AreEqual(first.FinalState, second.FinalState); + Assert.AreEqual(first.Effects, second.Effects); + } + + // ================================================================ scenario 10: mid-derivation state round-trip + + [TestMethod] + public void StateSerializes_AndResumes_MidDerivation() + { + var h = new DerivationHarness(); + + h.Ask("Weather in Palo Alto?"); + h.Segment("Checking [WEATHER(\"Palo Alto\", @w)"); + Assert.IsInstanceOfType(h.State.Phase); + + // Simulate the substrate checkpoint/recover cycle: state → JSON → state. + var json = h.State.ToJson(); + var restored = DerivationState.FromJson(json); + Assert.AreEqual(json, restored.ToJson()); + + // The restored state still knows its pending request and continues correctly. + var step = new DerivationStep(); + var requestId = restored.Pending!.RequestIds.Single(); + var result = step.Step(restored, new ToolSucceeded(requestId, ["\"Sunny\""]), h.Context); + + Assert.IsInstanceOfType(result.State.Phase); + Assert.AreEqual("\"Sunny\"", result.State.Sigma["w"]); + } + + // ================================================================ scenario 11: extract-as-match repair + + [TestMethod] + public void InventedExtractPredicate_RepairsToPatternMatch() + { + var h = new DerivationHarness(); + + h.Ask(new QuestionEnvelope( + "Get the close price from @stockData.", + InitialBindings: new Dictionary + { + ["stockData"] = """{"data":{"symbol":"IBM","close":"428.90"},"status":"ok"}""", + }.ToImmutableDictionary(), + ExpectedOutputs: [], + LearnRuleOnSuccess: false)); + + // Observed live: the model invents an extraction predicate with unmistakable intent. + h.Segment(" Extracting the price [EXTRACT(@stockData, { \"close\": @price })"); + + Assert.IsTrue(h.Traces.OfType().Any(r => r.What == "extract-as-match")); + Assert.AreEqual("\"428.90\"", h.State.Sigma["price"], "the pattern match must bind through the repair"); + } + + // ================================================================ scenario 12: post-answer noise is skipped + + [TestMethod] + public void HedgeFailure_AfterOutputsBound_IsSkippedNotUnwound() + { + var h = new DerivationHarness(); + + h.Ask(new QuestionEnvelope( + "Erik keeps his BTC; show it as @btcLeft.", + InitialBindings: new Dictionary { ["btc"] = "0.05" }.ToImmutableDictionary(), + ExpectedOutputs: ["btcLeft"], + LearnRuleOnSuccess: false)); + + h.Segment(" Keeping it [@btcLeft = @btc"); + Assert.AreEqual("0.05", h.State.Sigma["btcLeft"], "the declared output is bound — the answer exists"); + + // Observed live: a junk MATH re-call after the answer existed unwound everything. + h.Segment(". Double-checking [MATH(\"250\", @btcLeft)"); + + Assert.IsTrue(h.Traces.OfType().Any(r => r.What == "post-answer-noise"), + "a failing hedge after the outputs are bound must be skipped, not backtracked"); + Assert.AreEqual("0.05", h.State.Sigma["btcLeft"], "the bound state must survive the noise"); + + h.Segment(".", stoppedAtHedge: false); + Assert.IsInstanceOfType(h.State.Phase); + Assert.Contains("@btcLeft = 0.05", h.AnswerText()); + } + + // ================================================================ scenario 13: expected outputs gate completion + + [TestMethod] + public void Completion_WithUnboundExpectedOutput_BacktracksInsteadOfFinishing() + { + var h = new DerivationHarness(); + + h.Ask(new QuestionEnvelope( + "Compute the answer as @result.", + InitialBindings: ImmutableDictionary.Empty, + ExpectedOutputs: ["result"], + LearnRuleOnSuccess: false)); + + // The model executes SOMETHING (so the essay guard passes) and stops without ever + // binding the declared output — observed live: prose narrated success, nothing computed. + h.Segment(" Scratch work [@scratch is 1"); + h.Segment(". All done.", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase, "an unbound declared output must backtrack, not complete"); + Assert.IsTrue(h.State.EngineNotes.Any(n => n.Contains("@result", StringComparison.Ordinal)), + $"the steering must name the missing output; notes: {string.Join(" | ", h.State.EngineNotes)}"); + + // The retry computes it — completion now passes and the assembler appends the value. + h.Segment(" Computing [@result is 6 * 7"); + h.Segment(".", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.Contains("@result = 42", h.AnswerText()); + } + + // ================================================================ scenario 12: phantom guard vs steering quotes + + [TestMethod] + public void PhantomGuard_IgnoresVariablesQuotedFromEngineSteering() + { + var h = new DerivationHarness(); + + h.Ask(new QuestionEnvelope( + "Double the number in @x.", + InitialBindings: new Dictionary { ["x"] = "5" }.ToImmutableDictionary(), + ExpectedOutputs: [], + LearnRuleOnSuccess: false)); + + // A failing display plants a steering Aside whose teaching quotes engine vocabulary + // (⟨@newVar is …⟩) — meta-variables, not computed values. + h.Segment(" The result is [@y"); + Assert.IsInstanceOfType(h.State.Phase, "unbound display must backtrack and re-ask"); + + // The retry ECHOES the teaching's variables in prose, then computes properly. The + // phantom guard must treat the echoes as quotation, not as uncomputed value claims. + h.Segment(" As taught, @newVar is bound by a computation, so let me compute [@doubled is @x * 2"); + h.Segment(". The answer is [@doubled"); + h.Segment(".", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase, + $"echoed teaching tokens must not veto the answer; phase: {h.State.Phase}"); + Assert.Contains("10", h.AnswerText()); + } + + // ================================================================ scenario 12: identical-failure escalation + + [TestMethod] + public void RepeatedIdenticalFailure_EscalatesInsteadOfLoopingAtAttemptOne() + { + var h = new DerivationHarness(); + + h.Ask(new QuestionEnvelope( + "Convert everything in @files to PDF.", + InitialBindings: new Dictionary { ["files"] = """["a","b"]""" }.ToImmutableDictionary(), + ExpectedOutputs: [], + LearnRuleOnSuccess: false)); + + // The pathological live loop: narrate (the display executes and plants a FRESH choice + // point), then fail on the same unbound variable — over and over. Pre-escalation every + // retry ran at attempt 1 and the loop rode the full 32-request budget to exhaustion. + for (var i = 0; i < 3 && h.State.Phase is Synthesizing; i++) + { + h.Segment(" The files are [@files"); + + if (h.State.Phase is not Synthesizing) + { + break; + } + + h.Segment(". The PDFs are [@pdfs"); + } + + // The second identical failure escalates the steering note… + Assert.IsTrue(h.State.EngineNotes.Any(n => n.Contains("happened 2 times", StringComparison.Ordinal)), + $"expected an escalated note, got: {string.Join(" | ", h.State.EngineNotes)}"); + + // …and the third exhausts the local choice point and rewinds DEEPER. + Assert.IsGreaterThan(0, h.State.Budget.BacktrackDepthUsed, "identical repeats must force a deeper rewind"); + Assert.IsLessThan(10, h.State.LlmRequestCount, "the loop must be cut early, not ridden to budget exhaustion"); + } + + // ================================================================ scenario 13: budget on success paths + + [TestMethod] + public void EndlessNarrationLoop_ExhaustsTheRequestBudget_FailsCleanly() + { + // Review finding: the budget was enforced only in the backtracking machinery — a + // never-failing loop of inline-executed display hedges re-issued requests unbounded. + var h = new DerivationHarness(); + + h.Ask("Loop forever?"); + h.Segment("Bind once [@a is 1"); + + for (var i = 0; i < RetryBudget.Default.MaxLlmRequests + 4 && h.State.Phase is Synthesizing; i++) + { + h.Segment(" still here [@a"); + } + + Assert.IsInstanceOfType(h.State.Phase, "a never-failing narration loop must hit the budget"); + Assert.Contains("budget", ((FailedPhase)h.State.Phase).Reason); + Assert.IsTrue(h.State.LlmRequestCount <= RetryBudget.Default.MaxLlmRequests, "the cap must actually bind"); + } + + // ================================================================ scenario 14: duplicate-call identity + + [TestMethod] + public void RepeatedCall_WithShiftedArguments_IsNotMisreadAsDuplicate() + { + // The InArgsKey separator is load-bearing: MATH(12, 3) and MATH(1, 23) concatenate to + // the same "123" without one, and the genuinely NEW call would be skipped as narration + // with the stale output surviving into the answer. + var context = DerivationHarness.DefaultContext() with + { + Tools = + [ + new ToolBinding( + new PredicateSignature("MATH", [ + new PredicateParam("a", ParamMode.In), + new PredicateParam("b", ParamMode.In), + new PredicateParam("sum", ParamMode.Out), + ], "adds two numbers"), + "automind://tools/math", + IsIdempotent: true, + "adds two numbers"), + ], + }; + + var h = new DerivationHarness(context); + + h.Ask("Sums?"); + h.Segment("First [MATH(12, 3, @sum)"); + h.ToolResult(h.LastToolInvocations().Single().RequestId, "15"); + h.Segment(" then [MATH(1, 23, @sum)"); + + Assert.IsNotEmpty(h.LastToolInvocations(), "a different argument tuple is a NEW call, never duplicate narration"); + } + + // ================================================================ scenario 15: If-sentence guard + + [TestMethod] + public void IfSentence_MutatingHedge_TeachesChecklistInsteadOfExecuting() + { + var h = new DerivationHarness(); + + h.Ask(new QuestionEnvelope( + "Sam has $120 and a game costs $200. How much remains?", + InitialBindings: ImmutableDictionary.Empty, + ExpectedOutputs: [], + LearnRuleOnSuccess: false)); + + h.Segment("Sam has [@cash is 120"); + h.Segment(" dollars and the game costs [@price is 200"); + + // Observed live: a sentence-form conditional ("If Sam has enough money, he … [hedge]") + // executed its then-arm hedge UNCONDITIONALLY, and the poisoned @remaining made every + // honest checklist rewrite fail its assertion until the backtracking depth ran out. + h.Segment(" dollars. If Sam has enough money, he buys it and keeps [@remaining is @cash - @price"); + + Assert.IsFalse(h.State.Sigma.ContainsKey("remaining"), + "a hedge guarded by an 'If …' sentence must never execute — the engine cannot evaluate a prose condition"); + Assert.IsTrue(h.State.EngineNotes.Any(n => n.Contains("checklist", StringComparison.Ordinal)), + $"expected the checklist teach, got: {string.Join(" | ", h.State.EngineNotes)}"); + } +} diff --git a/tests/Automind.Kernel.Tests/ModeBTests.cs b/tests/Automind.Kernel.Tests/ModeBTests.cs new file mode 100644 index 0000000..eddb846 --- /dev/null +++ b/tests/Automind.Kernel.Tests/ModeBTests.cs @@ -0,0 +1,242 @@ +using System.Collections.Immutable; + +using Automind.Kernel.Contract; + +using Universalis.Core.Ir; + +namespace Automind.Kernel.Tests; + +/// +/// Mode B (whole-program synthesis): one structured completion carries the papers' +/// {comment|expression}[] interchange form; execution, teachings, and contracts are Mode A's; +/// backtracking degenerates to whole-program regeneration with the failure as feedback. +/// +[TestClass] +public sealed class ModeBTests +{ + private static QuestionEnvelope ModeB(string text, params string[] outputs) => new( + text, + ImmutableDictionary.Empty, + [.. outputs], + LearnRuleOnSuccess: false, + ModeB: true); + + [TestMethod] + public void WholeProgram_Arithmetic_ExecutesAndAnswers() + { + var h = new DerivationHarness(); + + h.Ask(ModeB("Alice bought apples for $10 and sold for $17. Profit percent as @profitPct?", "profitPct")); + + Assert.IsTrue(h.LastPrompt().WholeProgram, "Mode B must request one whole-program completion"); + + h.Segment(""" + {"program": [ + {"comment": "The apples cost"}, + {"expression": "@buyPrice is 10"}, + {"comment": "dollars and sold for"}, + {"expression": "@sellPrice is 17"}, + {"comment": "dollars, so the profit is"}, + {"expression": "@profit is @sellPrice - @buyPrice"}, + {"comment": "dollars. The percentage is"}, + {"expression": "@profitPct is (@profit / @buyPrice) * 100"}, + {"comment": ". Alice made"}, + {"expression": "@profitPct"}, + {"comment": "percent."} + ]} + """, stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase, string.Join(" | ", h.State.EngineNotes)); + Assert.Contains("70", h.AnswerText()); + Assert.AreEqual(1, h.State.LlmRequestCount, "a clean program needs exactly one request"); + } + + [TestMethod] + public void WholeProgram_Conditional_ReconstructsTheChecklistAndTakesTheBranch() + { + var h = new DerivationHarness(); + + h.Ask(ModeB("Sam has $120, the game costs $200 — how much remains as @left?", "left")); + + h.Segment(""" + {"program": [ + {"comment": "Sam has"}, + {"expression": "@cash is 120"}, + {"comment": "dollars and the game costs"}, + {"expression": "@price is 200"}, + {"comment": "dollars. Now decide:"}, + {"comment": "- If"}, + {"expression": "@cash >= @price"}, + {"comment": ", then Sam buys the game and keeps"}, + {"expression": "@left is @cash - @price"}, + {"comment": "dollars."}, + {"comment": "- Otherwise, Sam skips the game and keeps"}, + {"expression": "@left = @cash"}, + {"comment": "dollars."}, + {"comment": "Sam ends up with"}, + {"expression": "@left"}, + {"comment": "dollars."} + ]} + """, stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase, string.Join(" | ", h.State.EngineNotes)); + Assert.IsTrue(h.State.ProgramSoFar.OfType().Any(), + "the flat interchange form must reconstruct a real conditional block through the recognizer"); + Assert.AreEqual("120", h.State.Sigma["left"], "the Otherwise branch binds @left = @cash"); + Assert.Contains("120", h.AnswerText()); + } + + [TestMethod] + public void WholeProgram_ToolCall_SuspendsDurablyAndResumesTheWalk() + { + var h = new DerivationHarness(); + + h.Ask(ModeB("What is the weather in Palo Alto?")); + + h.Segment(""" + {"program": [ + {"comment": "Let's find the weather"}, + {"expression": "WEATHER(\"Palo Alto\", @weatherPaloAlto)"}, + {"comment": ". The current weather is"}, + {"expression": "@weatherPaloAlto"}, + {"comment": ". That answers the question."} + ]} + """, stoppedAtHedge: false); + + // The walk suspends at the tool boundary — durable, exactly like Mode A. + Assert.IsInstanceOfType(h.State.Phase); + Assert.IsFalse(h.State.ModeBQueue.IsDefaultOrEmpty, "the pending walk must survive in state"); + + var invocation = h.LastToolInvocations().Single(); + h.ToolResult(invocation.RequestId, "\"Sunny and 80°F\""); + + Assert.IsInstanceOfType(h.State.Phase, string.Join(" | ", h.State.EngineNotes)); + Assert.Contains("Sunny and 80°F", h.AnswerText()); + } + + [TestMethod] + public void WholeProgram_BadJson_RegeneratesWithTheTeachingNote() + { + var h = new DerivationHarness(); + + h.Ask(ModeB("Anything.", "x")); + + h.Segment("Here is my program: profit = 70%!", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase, "a rejected program regenerates, not fails"); + Assert.AreEqual(2, h.State.LlmRequestCount); + Assert.IsTrue(h.State.EngineNotes.Any(n => n.Contains("rejected", StringComparison.Ordinal)), + $"the failure must ride into the next prompt; got: {string.Join(" | ", h.State.EngineNotes)}"); + + h.Segment(""" + {"program": [ + {"comment": "Bind"}, + {"expression": "@x is 42"}, + {"comment": "and show"}, + {"expression": "@x"}, + {"comment": "."} + ]} + """, stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase, string.Join(" | ", h.State.EngineNotes)); + Assert.Contains("42", h.AnswerText()); + } + + [TestMethod] + public void WholeProgram_EvalFailure_RegeneratesFromScratch() + { + var h = new DerivationHarness(); + + h.Ask(ModeB("Compute @x.", "x")); + + // References a variable no hedge ever bound → eval failure mid-walk → regenerate. + h.Segment(""" + {"program": [ + {"comment": "Compute"}, + {"expression": "@x is @nothing + 1"}, + {"comment": "done."} + ]} + """, stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.IsEmpty(h.State.ProgramSoFar, "regeneration must reset the failed attempt's program"); + Assert.IsEmpty(h.State.Sigma, "regeneration must reset σ to the initial bindings"); + + h.Segment(""" + {"program": [ + {"comment": "Bind"}, + {"expression": "@x is 7"}, + {"comment": "and show"}, + {"expression": "@x"}, + {"comment": "."} + ]} + """, stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase, string.Join(" | ", h.State.EngineNotes)); + Assert.Contains("7", h.AnswerText()); + } + + [TestMethod] + public void WholeProgram_RepeatedIdenticalFailure_FailsFast() + { + // Review finding: regeneration was bounded only by the 32-request budget — hours of + // full-length generations against an immovable wall. The SAME failure repeating + // MaxAttemptsPerChoicePoint times must kill the derivation early. + var h = new DerivationHarness(); + + h.Ask(ModeB("Anything.", "x")); + + h.Segment("garbage, not json", stoppedAtHedge: false); + Assert.IsInstanceOfType(h.State.Phase); + + h.Segment("garbage, not json", stoppedAtHedge: false); + Assert.IsInstanceOfType(h.State.Phase); + + h.Segment("garbage, not json", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase, "the third identical failure must fail fast"); + Assert.Contains("repeated", ((FailedPhase)h.State.Phase).Reason); + } + + [TestMethod] + public void WholeProgram_Truncated_TeachesBrevityAndGrowsTheCap() + { + // Review finding: a generation cut at the token cap was indistinguishable from + // malformed JSON, and regeneration retried into the identical ceiling with a + // misleading "return valid JSON" teach. + var h = new DerivationHarness(); + + h.Ask(ModeB("Anything.", "x")); + Assert.AreEqual(1024, h.LastPrompt().MaxSegmentTokens); + + h.Segment("""{"program": [{"comment": "unfinished""", stoppedAtHedge: false, truncated: true); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.IsTrue(h.State.EngineNotes.Any(n => n.Contains("cut off", StringComparison.Ordinal)), + $"the teach must name truncation, not JSON validity; got: {string.Join(" | ", h.State.EngineNotes)}"); + Assert.IsGreaterThan(1024, h.LastPrompt().MaxSegmentTokens, "the cap must grow after truncation"); + } + + [TestMethod] + public void WholeProgram_MissingDeclaredOutput_RegeneratesViaTheCompletionContract() + { + var h = new DerivationHarness(); + + h.Ask(ModeB("Compute @total.", "total")); + + // A well-formed program that never binds the declared output. + h.Segment(""" + {"program": [ + {"comment": "Bind"}, + {"expression": "@other is 5"}, + {"comment": "and show"}, + {"expression": "@other"}, + {"comment": "."} + ]} + """, stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase, "the completion contract applies to Mode B too"); + Assert.IsTrue(h.State.EngineNotes.Any(n => n.Contains("total", StringComparison.Ordinal)), + $"the missing output must be named in the feedback; got: {string.Join(" | ", h.State.EngineNotes)}"); + } +} diff --git a/tests/Automind.Kernel.Tests/RuleTests.cs b/tests/Automind.Kernel.Tests/RuleTests.cs new file mode 100644 index 0000000..07c31ee --- /dev/null +++ b/tests/Automind.Kernel.Tests/RuleTests.cs @@ -0,0 +1,385 @@ +using System.Collections.Immutable; +using System.Text.Json; + +using Automind.Kernel.Contract; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; +using Universalis.Core.Parsing; + +namespace Automind.Kernel.Tests; + +[TestClass] +public sealed class RuleTests +{ + private static ImmutableArray Body(params string[] hedges) => + [.. hedges.Select(h => + { + var parsed = HedgeParser.Parse(h); + Assert.IsTrue(parsed.Success, parsed.Error); + return (ProgramItem)new HedgeItem(parsed.Statement!, h); + })]; + + private static RuleDefinition ProfitRule => new( + new RuleSignature("PROFITPCT", [ + new RuleParam("buy", ParamMode.In), + new RuleParam("sell", ParamMode.In), + new RuleParam("pct", ParamMode.Out), + ], "profit percentage from buy/sell prices"), + "profit percentage", + new UniversalisProgram(Body("@d is @sell - @buy", "@pct is (@d / @buy) * 100"), [], [])); + + private static RuleDefinition CityWeatherRule => new( + new RuleSignature("CITYWEATHER", [ + new RuleParam("place", ParamMode.In), + new RuleParam("w", ParamMode.Out), + ], "weather via the WEATHER tool"), + "weather lookup", + new UniversalisProgram(Body("WEATHER(@place, @w)"), [], [])); + + private static StepContext WithRules(params RuleDefinition[] rules) => + DerivationHarness.DefaultContext() with { Rules = [.. rules] }; + + [TestMethod] + public void StoredRule_PureBody_RunsSynchronously_ZeroLlmCallsInside() + { + var h = new DerivationHarness(WithRules(ProfitRule)); + + h.Ask("What's the profit percentage for buy 12 sell 18?"); + Assert.Contains("PROFITPCT", h.LastPrompt().SystemPrompt, "learned rules are advertised to the model"); + + h.Segment("Using the stored rule [PROFITPCT(12, 18, @p)"); + + // The rule body ran inline: bindings landed, generation resumed with the bracket closed. + Assert.IsInstanceOfType(h.State.Phase); + Assert.AreEqual("50", h.State.Sigma["p"]); + Assert.IsTrue(h.Traces.OfType().Any(t => t.Name == "PROFITPCT")); + Assert.IsEmpty(h.LastToolInvocations()); + Assert.IsEmpty(h.State.Frames, "frames unwound after the rule completed"); + + h.Segment(". The answer is [@p"); + h.Segment(" percent.", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.Contains("The answer is 50 percent.", h.AnswerText()); + Assert.AreEqual(3, h.AllEffects.OfType().Count(), "LLM only drives the OUTER derivation"); + } + + [TestMethod] + public void StoredRule_WithToolCall_SuspendsAndResumes_WithoutLlm() + { + var h = new DerivationHarness(WithRules(CityWeatherRule)); + + h.Ask("Weather in Palo Alto via the stored rule?"); + var llmCallsBefore = h.AllEffects.OfType().Count(); + + h.Segment("Let me reuse what I learned [CITYWEATHER(\"Palo Alto\", @answer)"); + + // The rule suspended on its INTERNAL tool call — mid-rule, checkpointable. + Assert.IsInstanceOfType(h.State.Phase); + Assert.HasCount(1, h.State.Frames, "the rule frame persists across the suspension"); + var invocation = h.LastToolInvocations().Single(); + Assert.AreEqual("""{"city":"Palo Alto"}""", invocation.ArgsJson); + + // Mid-rule state must round-trip (the kill/recover story extends INTO rules). + var restored = DerivationState.FromJson(h.State.ToJson()); + Assert.HasCount(1, restored.Frames); + Assert.AreEqual("CITYWEATHER", restored.Frames[0].RuleName); + + h.ToolResult(invocation.RequestId, "\"Sunny and 80°F\""); + + // Tool result resumed the RULE (not the LLM), the rule completed, outputs copied back. + Assert.IsInstanceOfType(h.State.Phase); + Assert.AreEqual("\"Sunny and 80°F\"", h.State.Sigma["answer"]); + Assert.IsEmpty(h.State.Frames); + Assert.AreEqual(llmCallsBefore + 1, h.AllEffects.OfType().Count(), + "exactly one LLM resume after the rule finished — zero LLM calls inside the rule"); + + h.Segment(". It is [@answer"); + h.Segment(".", stoppedAtHedge: false); + Assert.Contains("Sunny and 80°F", h.AnswerText()); + } + + /// Paper 1's composed WEATHER rule: three chained tool suspensions, zero LLM inside. + [TestMethod] + public void ComposedWeatherRule_ChainsThreeTools_WithPatternExtraction() + { + var geoChain = new RuleDefinition( + new RuleSignature("FORECAST", [ + new RuleParam("city", ParamMode.In), + new RuleParam("weather", ParamMode.Out), + ], "weather via geo-code, point lookup, forecast fetch"), + "NWS chain", + new UniversalisProgram(Body( + "GEO_CODE(@city, @lat, @lon)", + "WEATHER_GOV(@lat, @lon, { ... \"forecast\": @url ... })", + "HTTP_GET(@url, { ... \"detailedForecast\": @weather ... })"), [], [])); + + var context = DerivationHarness.DefaultContext() with + { + Tools = + [ + new ToolBinding(new PredicateSignature("GEO_CODE", [ + new PredicateParam("city", ParamMode.In), + new PredicateParam("lat", ParamMode.Out), + new PredicateParam("lon", ParamMode.Out), + ], "coordinates"), "automind://tools/geo_code", true, "coordinates of a city"), + new ToolBinding(new PredicateSignature("WEATHER_GOV", [ + new PredicateParam("lat", ParamMode.In), + new PredicateParam("lon", ParamMode.In), + new PredicateParam("response", ParamMode.Out), + ], "NWS point"), "automind://tools/weather_gov", true, "NWS point metadata"), + new ToolBinding(new PredicateSignature("HTTP_GET", [ + new PredicateParam("url", ParamMode.In), + new PredicateParam("response", ParamMode.Out), + ], "fetch"), "automind://tools/http_get", true, "HTTP GET"), + ], + Rules = [geoChain], + }; + + var h = new DerivationHarness(context); + h.Ask("What's the forecast for Palo Alto?"); + var llmCallsBefore = h.AllEffects.OfType().Count(); + + h.Segment("Using the stored chain [FORECAST(\"Palo Alto\", @report)"); + + // Suspension 1: GEO_CODE (two outputs, keyed object result). + Assert.IsInstanceOfType(h.State.Phase); + var geo = h.LastToolInvocations().Single(); + Assert.AreEqual("automind://tools/geo_code", geo.ToolUri); + h.ToolResult(geo.RequestId, """{"lat":37.44,"lon":-122.14}"""); + + // Suspension 2: WEATHER_GOV — its out-pattern digs "forecast" from nested properties. + var gov = h.LastToolInvocations().Single(); + Assert.AreEqual("automind://tools/weather_gov", gov.ToolUri); + h.ToolResult(gov.RequestId, """{"properties":{"gridId":"MTR","forecast":"https://api.weather.gov/x/forecast"}}"""); + + // Suspension 3: HTTP_GET with the extracted URL; deep pattern finds detailedForecast. + var http = h.LastToolInvocations().Single(); + Assert.AreEqual("automind://tools/http_get", http.ToolUri); + Assert.Contains("api.weather.gov/x/forecast", http.ArgsJson); + h.ToolResult(http.RequestId, """{"properties":{"periods":[{"name":"Today","detailedForecast":"Sunny, high near 80."}]}}"""); + + // The chain completed: the rule's output copied back, ZERO LLM calls in between. + Assert.IsInstanceOfType(h.State.Phase); + Assert.AreEqual("\"Sunny, high near 80.\"", h.State.Sigma["report"]); + Assert.IsEmpty(h.State.Frames); + Assert.AreEqual(llmCallsBefore + 1, h.AllEffects.OfType().Count(), + "three tool suspensions, one LLM resume at the end"); + + h.Segment(". The forecast is [@report"); + h.Segment(".", stoppedAtHedge: false); + Assert.Contains("Sunny, high near 80.", h.AnswerText()); + } + + /// + /// The live weather-rule defect: a rule body must be isolated from the caller's σ. The + /// model called GEO_CODE directly (binding top-level @lat/@lon) and THEN invoked the rule, + /// whose body also uses @lat/@lon — dynamic-scope layering made the body's outputs look + /// like rebinds, and since rule execution is deterministic every retry failed identically. + /// + [TestMethod] + public void RuleBody_IsIsolatedFromCallerBindings() + { + var geoChain = new RuleDefinition( + new RuleSignature("FORECAST", [ + new RuleParam("city", ParamMode.In), + new RuleParam("weather", ParamMode.Out), + ], "weather via geo-code, point lookup, forecast fetch"), + "NWS chain", + new UniversalisProgram(Body( + "GEO_CODE(@city, @lat, @lon)", + "WEATHER_GOV(@lat, @lon, { ... \"forecast\": @url ... })", + "HTTP_GET(@url, { ... \"detailedForecast\": @weather ... })"), [], [])); + + var context = DerivationHarness.DefaultContext() with + { + Tools = + [ + new ToolBinding(new PredicateSignature("GEO_CODE", [ + new PredicateParam("city", ParamMode.In), + new PredicateParam("lat", ParamMode.Out), + new PredicateParam("lon", ParamMode.Out), + ], "coordinates"), "automind://tools/geo_code", true, "coordinates of a city"), + new ToolBinding(new PredicateSignature("WEATHER_GOV", [ + new PredicateParam("lat", ParamMode.In), + new PredicateParam("lon", ParamMode.In), + new PredicateParam("response", ParamMode.Out), + ], "NWS point"), "automind://tools/weather_gov", true, "NWS point metadata"), + new ToolBinding(new PredicateSignature("HTTP_GET", [ + new PredicateParam("url", ParamMode.In), + new PredicateParam("response", ParamMode.Out), + ], "fetch"), "automind://tools/http_get", true, "HTTP GET"), + ], + Rules = [geoChain], + }; + + var h = new DerivationHarness(context); + h.Ask("What's the forecast for Palo Alto?"); + + // The model first calls the primitive DIRECTLY, binding @lat/@lon in the caller's σ. + h.Segment("First the coordinates [GEO_CODE(\"Palo Alto\", @lat, @lon)"); + h.ToolResult(h.LastToolInvocations().Single().RequestId, """{"lat":37.44,"lon":-122.14}"""); + Assert.IsTrue(h.State.Sigma.ContainsKey("lat"), "the caller now has @lat bound"); + + // Then it invokes the stored rule whose BODY also uses @lat/@lon internally. + h.Segment(". Now use the stored chain [FORECAST(\"Palo Alto\", @report)"); + + var geo = h.LastToolInvocations().Single(); + Assert.AreEqual("automind://tools/geo_code", geo.ToolUri, "the rule body must run, not collide"); + h.ToolResult(geo.RequestId, """{"lat":37.44,"lon":-122.14}"""); + + var gov = h.LastToolInvocations().Single(); + h.ToolResult(gov.RequestId, """{"properties":{"forecast":"https://api.weather.gov/x/forecast"}}"""); + + var http = h.LastToolInvocations().Single(); + h.ToolResult(http.RequestId, """{"properties":{"periods":[{"name":"Today","detailedForecast":"Sunny, high near 80."}]}}"""); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.AreEqual("\"Sunny, high near 80.\"", h.State.Sigma["report"]); + Assert.IsEmpty(h.State.Frames); + } + + [TestMethod] + public void RuleShadowsToolOfTheSameName() + { + // The papers' self-learning story: a stored rule named WEATHER supersedes the primitive. + var shadow = new RuleDefinition( + new RuleSignature("WEATHER", [ + new RuleParam("city", ParamMode.In), + new RuleParam("weather", ParamMode.Out), + ], "rule-based weather"), + "shadowing rule", + new UniversalisProgram(Body("@weather = \"From the rule, not the tool\""), [], [])); + + var h = new DerivationHarness(DerivationHarness.DefaultContext() with { Rules = [shadow] }); + + h.Ask("Weather in Palo Alto?"); + h.Segment("Checking [WEATHER(\"Palo Alto\", @w)"); + + Assert.IsEmpty(h.LastToolInvocations(), "the rule shadows the primitive tool"); + Assert.IsTrue(h.Traces.OfType().Any(t => t.Name == "WEATHER")); + Assert.AreEqual("\"From the rule, not the tool\"", h.State.Sigma["w"]); + } + + [TestMethod] + public void DuplicateRuleNames_TheLastDefinitionWins() + { + // Review finding: dispatch used FirstOrDefault while the signature catalog is last-wins + // and the host promises "seeds come last so they shadow" — a stale stored body executed + // against the shadowing seed's signature. + var stale = new RuleDefinition( + new RuleSignature("PICK", [new RuleParam("x", ParamMode.Out)], "stale stored rule"), + "stale", + new UniversalisProgram(Body("@x = \"stale\""), [], [])); + + var seed = new RuleDefinition( + new RuleSignature("PICK", [new RuleParam("x", ParamMode.Out)], "shadowing seed rule"), + "seed", + new UniversalisProgram(Body("@x = \"seed\""), [], [])); + + var h = new DerivationHarness(DerivationHarness.DefaultContext() with { Rules = [stale, seed] }); + + h.Ask("Pick?"); + h.Segment("Choose [PICK(@x)"); + + Assert.AreEqual("\"seed\"", h.State.Sigma["x"], "the last-added rule shadows — seeds come last"); + } + + [TestMethod] + public void RuleCallInsideConditionalBranch_TeachesInsteadOfVanishing() + { + // Review finding: the branch-body dispatch had no NeedRule (or default) arm — a rule + // call in a taken branch fell out of the switch with no execution, no trace, no error. + var rule = new RuleDefinition( + new RuleSignature("LOOKUP", [ + new RuleParam("city", ParamMode.In), + new RuleParam("weather", ParamMode.Out), + ], "weather lookup rule"), + "lookup", + new UniversalisProgram(Body("@weather = \"Sunny\""), [], [])); + + var h = new DerivationHarness(DerivationHarness.DefaultContext() with { Rules = [rule] }); + + h.Ask("Decide?"); + h.Segment("We have [@cash is 120"); + h.Segment(" and decide:\n- If [@cash >= 100"); + h.Segment(", then look it up [LOOKUP(\"Palo Alto\", @w)"); + h.Segment("\nDone with [@cash"); + + Assert.IsTrue( + h.State.EngineNotes.Any(n => n.Contains("rule calls inside conditional branches", StringComparison.Ordinal)), + $"the drop must become a teaching backtrack; got: {string.Join(" | ", h.State.EngineNotes)}"); + Assert.IsFalse(h.Traces.OfType().Any(), "the rule must not half-run inside the branch"); + } + + [TestMethod] + public void LearnRule_ZeroParameterDerivation_LikeTheCli_DoesNotWedge() + { + // The CLI `ask --learn NAME` path: no initial bindings, no expected outputs — a + // zero-parameter memoized derivation with a tool call and a display. + var h = new DerivationHarness(); + + h.Ask(QuestionEnvelope.ForText("What is the current weather in Palo Alto?") with + { + LearnRuleOnSuccess = true, + RuleName = "paloAltoWeather", + }); + + h.Segment("Let's check the weather [WEATHER(\"Palo Alto\", @weatherPaloAlto)"); + h.ToolResult(h.LastToolInvocations().Single().RequestId, "\"Sunny and 80°F\""); + h.Segment(". The current weather in Palo Alto is [@weatherPaloAlto"); + h.Segment(". That answers the question.", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase); + Assert.IsTrue(h.AllEffects.OfType().Any(), "the answer must always be emitted"); + Assert.IsTrue(h.AllEffects.OfType().Any() || h.Traces.OfType().Any(), + "learning either succeeds or is skipped visibly — never wedges the derivation"); + } + + [TestMethod] + public void LearnRuleOnSuccess_EmitsDefineRule_WithIrAndBonsai() + { + var h = new DerivationHarness(); + + h.Ask(new QuestionEnvelope( + "Profit percentage from B and S?", + InitialBindings: new Dictionary { ["B"] = "10", ["S"] = "17" }.ToImmutableDictionary(), + ExpectedOutputs: ["P"], + LearnRuleOnSuccess: true, + RuleName: "profitpct")); + + h.Segment("The profit is [@D is @S - @B"); + h.Segment(" and the percentage is [@P is (@D / @B) * 100"); + h.Segment(". Done.", stoppedAtHedge: false); + + Assert.IsInstanceOfType(h.State.Phase); + + var defineRule = h.AllEffects.OfType().Single(); + Assert.AreEqual("profitpct", defineRule.Name); + Assert.IsTrue(h.Traces.OfType().Any(t => t.Name == "profitpct")); + + var payload = JsonDocument.Parse(defineRule.BonsaiJson); + var ir = payload.RootElement.GetProperty("ir").GetString()!; + var bonsai = payload.RootElement.GetProperty("bonsai").GetString()!; + + // The IR is a well-formed rule with the mechanically derived signature. + var rule = IrJson.DeserializeRule(ir); + Assert.AreEqual("profitpct", rule.Signature.Name); + CollectionAssert.AreEqual( + new[] { ("B", ParamMode.In), ("S", ParamMode.In), ("P", ParamMode.Out) }, + rule.Signature.Params.Select(p => (p.Name, p.Mode)).ToArray()); + + // The Bonsai document is the papers' intentional representation, URI-addressed. + Assert.Contains("universalis://rule/v1", bonsai); + Assert.Contains("universalis://is/sub", bonsai); + + // THE self-learning loop closes: the learned rule answers the next question directly. + var h2 = new DerivationHarness(DerivationHarness.DefaultContext() with { Rules = [rule] }); + h2.Ask("Profit percentage for 12 and 18?"); + h2.Segment("Reusing the learned rule [profitpct(12, 18, @result)"); + + Assert.AreEqual("50", h2.State.Sigma["result"]); + Assert.IsTrue(h2.Traces.OfType().Any(t => t.Name == "profitpct")); + } +} diff --git a/tests/Automind.Mcp.Tests/Automind.Mcp.Tests.csproj b/tests/Automind.Mcp.Tests/Automind.Mcp.Tests.csproj new file mode 100644 index 0000000..347b9e4 --- /dev/null +++ b/tests/Automind.Mcp.Tests/Automind.Mcp.Tests.csproj @@ -0,0 +1,7 @@ + + + + + + + diff --git a/tests/Automind.Mcp.Tests/LoopbackBridgeTests.cs b/tests/Automind.Mcp.Tests/LoopbackBridgeTests.cs new file mode 100644 index 0000000..104f115 --- /dev/null +++ b/tests/Automind.Mcp.Tests/LoopbackBridgeTests.cs @@ -0,0 +1,114 @@ +using System.IO.Pipelines; + +using Automind.Mcp; + +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol; +using ModelContextProtocol.Server; + +namespace Automind.Mcp.Tests; + +/// +/// The bridge against a REAL in-process MCP server (stream transports over pipe pairs — no +/// subprocess, fast-suite friendly): list, map, invoke as ITool, and observe both result +/// mapping paths plus the error→exception path the driver turns into a backtrack. +/// +[TestClass] +public sealed class LoopbackBridgeTests +{ + private static (McpServer Server, Task Run, StreamClientTransport ClientTransport) StartLoopbackServer() + { + var clientToServer = new Pipe(); + var serverToClient = new Pipe(); + + var options = new McpServerOptions + { + ServerInfo = new Implementation { Name = "loopback", Version = "1.0.0" }, + ToolCollection = + [ + McpServerTool.Create( + (string text) => new string([.. text.Reverse()]), + new McpServerToolCreateOptions { Name = "reverse", Description = "Reverses a string.", ReadOnly = true }), + McpServerTool.Create( + (string text) => text.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length, + new McpServerToolCreateOptions { Name = "word_count", Description = "Counts words.", ReadOnly = true }), + McpServerTool.Create( + string (string text) => throw new InvalidOperationException("deliberate failure"), + new McpServerToolCreateOptions { Name = "always_fails", Description = "Always errors." }), + ], + }; + + var server = McpServer.Create( + new StreamServerTransport(clientToServer.Reader.AsStream(), serverToClient.Writer.AsStream(), "loopback"), + options); + + var run = server.RunAsync(CancellationToken.None); + + var clientTransport = new StreamClientTransport( + serverInput: clientToServer.Writer.AsStream(), + serverOutput: serverToClient.Reader.AsStream()); + + return (server, run, clientTransport); + } + + [TestMethod] + [Timeout(30_000)] + public async Task Bridge_ListsMapsAndInvokes_BothResultShapes() + { + var (server, run, transport) = StartLoopbackServer(); + await using var _ = server; + + await using var client = await McpClient.CreateAsync(transport); + var tools = await client.ListToolsAsync(); + + var reverse = tools.Single(t => t.Name == "reverse"); + var signature = McpPredicateMapper.ToSignature(reverse.Name, reverse.Description, reverse.JsonSchema); + + Assert.AreEqual("REVERSE", signature.Name); + Assert.AreEqual("text", signature.InParams.Single().Name); + Assert.IsTrue(McpPredicateMapper.IsIdempotent(reverse.ProtocolTool.Annotations), "ReadOnly=true must map to idempotent"); + + var bridged = new McpBridgedTool(reverse, "loopback", signature, isIdempotent: true); + + // Prose result → string-encoded JSON. + var reversed = await bridged.InvokeAsync("""{"text":"reaqtor"}""", CancellationToken.None); + Assert.AreEqual("\"rotqaer\"", reversed.Single()); + + // Numeric result → raw JSON passthrough. + var wordCount = tools.Single(t => t.Name == "word_count"); + var counted = await new McpBridgedTool( + wordCount, "loopback", + McpPredicateMapper.ToSignature(wordCount.Name, wordCount.Description, wordCount.JsonSchema), + isIdempotent: true) + .InvokeAsync("""{"text":"the neural computer runs programs"}""", CancellationToken.None); + Assert.AreEqual("5", counted.Single()); + } + + [TestMethod] + [Timeout(30_000)] + public async Task Bridge_ErrorResult_ThrowsForTheBacktrackPath() + { + var (server, run, transport) = StartLoopbackServer(); + await using var _ = server; + + await using var client = await McpClient.CreateAsync(transport); + var tools = await client.ListToolsAsync(); + + var failing = tools.Single(t => t.Name == "always_fails"); + var bridged = new McpBridgedTool( + failing, "loopback", + McpPredicateMapper.ToSignature(failing.Name, failing.Description, failing.JsonSchema), + isIdempotent: false); + + await Assert.ThrowsExactlyAsync( + () => bridged.InvokeAsync("""{"text":"x"}""", CancellationToken.None)); + } + + [TestMethod] + public void SplitCommandLine_HonorsQuotes() + { + CollectionAssert.AreEqual( + new[] { "dotnet", @"C:\some path\Server.dll", "--flag" }, + McpToolBridge.SplitCommandLine("dotnet \"C:\\some path\\Server.dll\" --flag")); + } +} diff --git a/tests/Automind.Mcp.Tests/McpPredicateMapperTests.cs b/tests/Automind.Mcp.Tests/McpPredicateMapperTests.cs new file mode 100644 index 0000000..764cec1 --- /dev/null +++ b/tests/Automind.Mcp.Tests/McpPredicateMapperTests.cs @@ -0,0 +1,136 @@ +using System.Text.Json; + +using Automind.Mcp; + +using ModelContextProtocol.Protocol; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; + +namespace Automind.Mcp.Tests; + +[TestClass] +public sealed class McpPredicateMapperTests +{ + [TestMethod] + public void PredicateName_UppercasesAndSanitizes() + { + Assert.AreEqual("SEARCH_FILES", McpPredicateMapper.ToPredicateName("search_files")); + Assert.AreEqual("WORD_COUNT", McpPredicateMapper.ToPredicateName("word-count")); + Assert.AreEqual("MCP_1TOOL", McpPredicateMapper.ToPredicateName("1tool")); + } + + [TestMethod] + public void Signature_TakesRequiredPropertiesInSchemaOrder_ParamNamesVerbatim() + { + // Parameter names must stay verbatim: the engine keys call arguments by parameter name, + // and those keys must match the MCP input schema exactly. + var schema = JsonDocument.Parse(""" + { + "type": "object", + "properties": { + "path": { "type": "string" }, + "recursive": { "type": "boolean" }, + "pattern": { "type": "string" } + }, + "required": ["path", "pattern"] + } + """).RootElement; + + var signature = McpPredicateMapper.ToSignature("search_files", "Searches files.", schema); + + Assert.AreEqual("SEARCH_FILES", signature.Name); + Assert.AreEqual(3, signature.Params.Length, "two required in-params plus the out-param"); + Assert.AreEqual(("path", ParamMode.In), (signature.Params[0].Name, signature.Params[0].Mode)); + Assert.AreEqual(("pattern", ParamMode.In), (signature.Params[1].Name, signature.Params[1].Mode), "optional 'recursive' is dropped, order is schema order"); + Assert.AreEqual(("result", ParamMode.Out), (signature.Params[2].Name, signature.Params[2].Mode)); + } + + [TestMethod] + public void Signature_OutParamDodgesACollidingPropertyName() + { + var schema = JsonDocument.Parse("""{"type":"object","properties":{"result":{"type":"string"}},"required":["result"]}""").RootElement; + + var signature = McpPredicateMapper.ToSignature("echo", null, schema); + + Assert.AreEqual("outValue", signature.Params[^1].Name); + } + + [TestMethod] + public void Signature_ArrayTypedParameter_AcceptsTheWholeCollection() + { + // Review finding: without AcceptsCollection the engine zip-lifts a list argument into + // one invocation per element, sending scalars where the server's schema requires arrays. + var schema = JsonDocument.Parse(""" + { + "type": "object", + "properties": { + "paths": { "type": "array", "items": { "type": "string" } }, + "tags": { "type": ["array", "null"] }, + "name": { "type": "string" } + }, + "required": ["paths", "tags", "name"] + } + """).RootElement; + + var signature = McpPredicateMapper.ToSignature("touch", null, schema); + + Assert.IsTrue(signature.Params[0].AcceptsCollection, "\"type\":\"array\" takes the whole list"); + Assert.IsTrue(signature.Params[1].AcceptsCollection, "type unions containing \"array\" count too"); + Assert.IsFalse(signature.Params[2].AcceptsCollection); + } + + [TestMethod] + public void UnmappedSchemaComposition_IsDetected() + { + // Review finding: allOf/$ref schemas silently mapped to zero In-parameters; the bridge + // now skips such tools loudly instead of advertising a wrong arity. + Assert.IsTrue(McpPredicateMapper.HasUnmappedComposition( + JsonDocument.Parse("""{"allOf":[{"type":"object"}]}""").RootElement)); + Assert.IsTrue(McpPredicateMapper.HasUnmappedComposition( + JsonDocument.Parse("""{"$ref":"#/$defs/args"}""").RootElement)); + Assert.IsTrue(McpPredicateMapper.HasUnmappedComposition( + JsonDocument.Parse("""{"type":"object","required":["ghost"],"properties":{"real":{"type":"string"}}}""").RootElement), + "a required name with no matching property is composition the mapper cannot see"); + Assert.IsFalse(McpPredicateMapper.HasUnmappedComposition( + JsonDocument.Parse("""{"type":"object","properties":{"x":{"type":"string"}},"required":["x"]}""").RootElement)); + } + + [TestMethod] + public void Idempotency_OnlyFromServerAnnotations() + { + Assert.IsFalse(McpPredicateMapper.IsIdempotent(null), "unknown side effects must not double-fire on recovery"); + Assert.IsFalse(McpPredicateMapper.IsIdempotent(new ToolAnnotations())); + Assert.IsTrue(McpPredicateMapper.IsIdempotent(new ToolAnnotations { ReadOnlyHint = true })); + Assert.IsTrue(McpPredicateMapper.IsIdempotent(new ToolAnnotations { IdempotentHint = true })); + } + + [TestMethod] + public void MapResult_PrefersStructuredContent() + { + var result = new CallToolResult + { + StructuredContent = JsonDocument.Parse("""{"close": 428.9}""").RootElement, + Content = [new TextContentBlock { Text = "ignored" }], + }; + + Assert.Contains("428.9", McpPredicateMapper.MapResult(result)); + } + + [TestMethod] + public void MapResult_JsonTextPassesThroughRaw_ProseGetsEncoded() + { + // JSON text stays raw so pattern destructuring works on it downstream. + var json = new CallToolResult { Content = [new TextContentBlock { Text = """{"lat": 37.4}""" }] }; + Assert.AreEqual("""{"lat": 37.4}""", McpPredicateMapper.MapResult(json)); + + var number = new CallToolResult { Content = [new TextContentBlock { Text = "5" }] }; + Assert.AreEqual("5", McpPredicateMapper.MapResult(number)); + + var prose = new CallToolResult { Content = [new TextContentBlock { Text = "rotqaer" }] }; + Assert.AreEqual("\"rotqaer\"", McpPredicateMapper.MapResult(prose)); + + var empty = new CallToolResult { Content = [] }; + Assert.AreEqual("null", McpPredicateMapper.MapResult(empty)); + } +} diff --git a/tests/Automind.Reaqtor.Tests/Automind.Reaqtor.Tests.csproj b/tests/Automind.Reaqtor.Tests/Automind.Reaqtor.Tests.csproj new file mode 100644 index 0000000..edace19 --- /dev/null +++ b/tests/Automind.Reaqtor.Tests/Automind.Reaqtor.Tests.csproj @@ -0,0 +1,7 @@ + + + + + + + diff --git a/tests/Automind.Reaqtor.Tests/EngineSmokeTests.cs b/tests/Automind.Reaqtor.Tests/EngineSmokeTests.cs new file mode 100644 index 0000000..97ba023 --- /dev/null +++ b/tests/Automind.Reaqtor.Tests/EngineSmokeTests.cs @@ -0,0 +1,106 @@ +using System.Collections.Concurrent; + +using Automind.Reaqtor.Client; +using Automind.Reaqtor.Engine; +using Automind.Reaqtor.IO; + +using Reaqtive.Scheduler; + +using Reaqtor; +using Reaqtor.Shebang.Service; + +namespace Automind.Reaqtor.Tests; + +[TestClass] +public sealed class EngineSmokeTests +{ + /// + /// P0 acceptance: create an engine, define the Automind artifact catalog, run a standing + /// query (timer → egress topic), checkpoint, tear the engine down, recover a brand-new + /// engine instance from the same store, and observe the standing query resume on its own. + /// + [TestMethod] + [Timeout(60_000)] + public async Task Engine_Create_Checkpoint_Recover_TimerFlowsToEgress() + { + using var scheduler = PhysicalScheduler.Create(); + + var store = new InMemoryKeyValueStore(); + var iemgr = new AutomindIngressEgressManager(); + + var received = new ConcurrentQueue<(long SequenceId, DateTimeOffset Item)>(); + using var connection = iemgr.GetOrCreateSubject("smoke") + .Subscribe(new DelegateObserver<(long, DateTimeOffset)>(v => received.Enqueue(v))); + + // --- Phase 1: fresh engine, standing query, checkpoint. --- + + var engine1 = await AutomindEngineFactory.CreateNewAsync(store, scheduler, ingressEgressManager: iemgr); + try + { + var ctx = AutomindClientContext.For(engine1); + + await ctx.Heartbeat(TimeSpan.FromMilliseconds(25)).SubscribeAsync( + ctx.Egress("smoke"), + new Uri("automind://test/subscriptions/smoke"), + state: null, + CancellationToken.None); + + await WaitUntilAsync(() => received.Count >= 3, TimeSpan.FromSeconds(20), "events before checkpoint"); + + await engine1.CheckpointAsync(); + await engine1.UnloadAsync(); + } + finally + { + engine1.Dispose(); + } + + // Let in-flight ticks quiesce so the post-recovery growth assertion is meaningful. + await Task.Delay(200); + var countAfterUnload = received.Count; + + // --- Phase 2: recover a NEW engine over the same store; the query resumes itself. --- + + var engine2 = await AutomindEngineFactory.RecoverAsync(store, scheduler, ingressEgressManager: iemgr); + try + { + await WaitUntilAsync( + () => received.Count > countAfterUnload, + TimeSpan.FromSeconds(20), + "events after recovery (standing query should resume without any client call)"); + } + finally + { + await engine2.UnloadAsync(); + engine2.Dispose(); + } + } + + private static async Task WaitUntilAsync(Func condition, TimeSpan timeout, string what) + { + var deadline = DateTimeOffset.UtcNow + timeout; + + while (!condition()) + { + if (DateTimeOffset.UtcNow > deadline) + { + Assert.Fail($"Timed out waiting for: {what}"); + } + + await Task.Delay(50); + } + } + + private sealed class DelegateObserver : IObserver + { + private readonly Action _onNext; + + public DelegateObserver(Action onNext) => _onNext = onNext; + + public void OnCompleted() { } + + public void OnError(Exception error) { } + + public void OnNext(T value) => _onNext(value); + } +} diff --git a/tests/Automind.Reaqtor.Tests/Fakes.cs b/tests/Automind.Reaqtor.Tests/Fakes.cs new file mode 100644 index 0000000..da38633 --- /dev/null +++ b/tests/Automind.Reaqtor.Tests/Fakes.cs @@ -0,0 +1,92 @@ +using System.Collections.Concurrent; + +using Automind.Kernel.Prompting; +using Automind.Reaqtor.Llm; +using Automind.Tools; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; + +namespace Automind.Reaqtor.Tests; + +/// +/// Scripted LLM bridge. Each entry answers one request in order; a frozen entry parks the call on +/// a gate (simulating an in-flight generation) until released — or until cancellation, which is +/// exactly what an abandoned engine does to its in-flight work. +/// +public sealed class FakeLlmService : ILlmService +{ + private readonly ConcurrentQueue>> _script = new(); + + public int Calls => _callCount; + + private int _callCount; + + public ConcurrentQueue Prompts { get; } = new(); + + public FakeLlmService Segment(string text, bool stoppedAtHedge = true) + { + _script.Enqueue((_, _) => Task.FromResult(new LlmSegmentResult(text, stoppedAtHedge))); + return this; + } + + /// Parks the next call until the returned gate is released (or the call is cancelled). + public TaskCompletionSource Freeze() + { + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _script.Enqueue((_, ct) => gate.Task.WaitAsync(ct)); + return gate; + } + + public Task CompleteAsync(PromptState prompt, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _callCount); + Prompts.Enqueue(prompt); + + if (!_script.TryDequeue(out var next)) + { + throw new InvalidOperationException("FakeLlmService script exhausted."); + } + + return next(prompt, cancellationToken); + } +} + +public static class FakeTools +{ + public static PredicateSignature WeatherSignature { get; } = new("WEATHER", [ + new PredicateParam("city", ParamMode.In), + new PredicateParam("weather", ParamMode.Out), + ], "current weather for a city"); + + /// A weather tool answering instantly with a fixed value. + public static DelegateTool Weather(string result = "\"Sunny and 80°F\"", Action? onInvoke = null) => new( + WeatherSignature, + "current weather conditions for a city", + isIdempotent: true, + (_, _) => + { + onInvoke?.Invoke(); + return Task.FromResult>([result]); + }); + + /// A weather tool that parks on a gate until released (or cancelled). + public static (DelegateTool Tool, TaskCompletionSource Gate, Func InvocationCount) FrozenWeather() + { + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var count = 0; + + var tool = new DelegateTool( + WeatherSignature, + "current weather conditions for a city", + isIdempotent: true, + async (_, ct) => + { + Interlocked.Increment(ref count); + var result = await gate.Task.WaitAsync(ct).ConfigureAwait(false); + return [result]; + }); + + return (tool, gate, () => count); + } +} diff --git a/tests/Automind.Reaqtor.Tests/FileStoreTests.cs b/tests/Automind.Reaqtor.Tests/FileStoreTests.cs new file mode 100644 index 0000000..9ba5cfb --- /dev/null +++ b/tests/Automind.Reaqtor.Tests/FileStoreTests.cs @@ -0,0 +1,157 @@ +using System.Text; + +using Automind.Reaqtor.Store; + +namespace Automind.Reaqtor.Tests; + +[TestClass] +public sealed class FileStoreTests +{ + private static string NewDirectory() => + Path.Combine(Path.GetTempPath(), "automind-tests", Guid.NewGuid().ToString("N")); + + private static byte[] Bytes(string s) => Encoding.UTF8.GetBytes(s); + + [TestMethod] + public async Task TransactionCommit_PersistsAcrossReopen() + { + var dir = NewDirectory(); + + var store = FileQueryEngineStateStore.Open(dir); + + using (var tx = store.CreateTransaction()) + { + store.GetTable("t").Enter(tx).Add("k", Bytes("hello")); + await tx.CommitAsync(CancellationToken.None); + } + + var reopened = FileQueryEngineStateStore.Open(dir); + + using var tx2 = reopened.CreateTransaction(); + Assert.AreEqual("hello", Encoding.UTF8.GetString(reopened.GetTable("t").Enter(tx2)["k"])); + } + + [TestMethod] + public async Task CheckpointWriter_PersistsAcrossReopen() + { + var dir = NewDirectory(); + + var store = FileQueryEngineStateStore.Open(dir); + var writer = store.GetWriter(); + + await using (var item = writer.GetItemWriter("subscriptions", "sub-1")) + { + item.Write(Bytes("state-blob")); + } + + await writer.CommitAsync(CancellationToken.None, progress: null!); + + var reopened = FileQueryEngineStateStore.Open(dir); + Assert.IsTrue(reopened.GetReader().TryGetItemReader("subscriptions", "sub-1", out var stream)); + + using var reader = new StreamReader(stream); + Assert.AreEqual("state-blob", reader.ReadToEnd()); + } + + [TestMethod] + public async Task EmptyCommit_SkipsTheSnapshotRewrite() + { + // Telemetry-driven: the 5 s checkpoint timer commits an EMPTY writer while the engine + // idles on a long LLM call, and each commit was a full snapshot rewrite (observed live: + // 322 rewrites / 12.4 MB cumulative for a ~40 KB store). + var dir = NewDirectory(); + var store = FileQueryEngineStateStore.Open(dir); + + using (var tx = store.CreateTransaction()) + { + store.GetTable("t").Enter(tx).Add("k", Bytes("v")); + await tx.CommitAsync(CancellationToken.None); + } + + var snapshot = new FileInfo(Path.Combine(dir, "store.xml")); + var written = snapshot.LastWriteTimeUtc; + + await Task.Delay(30); // a rewrite must be distinguishable by timestamp + + await store.GetWriter().CommitAsync(CancellationToken.None, progress: null!); // zero edits + + snapshot.Refresh(); + Assert.AreEqual(written, snapshot.LastWriteTimeUtc, "an empty commit must not rewrite the snapshot"); + + // A real change still persists. + using (var tx = store.CreateTransaction()) + { + store.GetTable("t").Enter(tx).Add("k2", Bytes("v2")); + await tx.CommitAsync(CancellationToken.None); + } + + snapshot.Refresh(); + Assert.AreNotEqual(written, snapshot.LastWriteTimeUtc, "a real change must rewrite the snapshot"); + } + + [TestMethod] + public async Task CorruptSnapshot_FallsBackToBackup() + { + var dir = NewDirectory(); + + var store = FileQueryEngineStateStore.Open(dir); + + using (var tx = store.CreateTransaction()) + { + store.GetTable("t").Enter(tx).Add("first", Bytes("1")); + await tx.CommitAsync(CancellationToken.None); // snapshot #1 + } + + using (var tx = store.CreateTransaction()) + { + store.GetTable("t").Enter(tx).Add("second", Bytes("2")); + await tx.CommitAsync(CancellationToken.None); // snapshot #2, #1 becomes .bak + } + + File.WriteAllText(Path.Combine(dir, "store.xml"), " +/// The P3 acceptance matrix: kill the process (dispose without unload or checkpoint, reopen +/// everything from disk) at every interesting phase and assert the derivation resumes and +/// completes with exactly one answer in the recovered run. +/// +[TestClass] +public sealed class KillRecoverTests +{ + private static PhysicalScheduler s_scheduler = null!; + + [ClassInitialize] + public static void ClassInitialize(TestContext _) => s_scheduler = PhysicalScheduler.Create(); + + [ClassCleanup] + public static void ClassCleanup() => s_scheduler.Dispose(); + + private static string NewDirectory() => + Path.Combine(Path.GetTempPath(), "automind-tests", Guid.NewGuid().ToString("N")); + + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30); + + private const string Segment1 = "The city is Palo Alto. Checking [WEATHER(\"Palo Alto\", @w)"; + private const string Segment2 = ". The weather is [@w"; + private const string Segment3 = "."; + + [TestMethod] + [Timeout(60_000)] + public async Task HappyPath_ThroughTheEngine_AnswerAndTracesFlow() + { + await using var h = new SubstrateHarness(s_scheduler, NewDirectory()); + h.Llm = new FakeLlmService().Segment(Segment1).Segment(Segment2).Segment(Segment3, stoppedAtHedge: false); + h.Tools.Add(FakeTools.Weather()); + + await h.StartFreshAsync(); + await h.AskAsync("d1", "What's the weather in Palo Alto?"); + + await h.WaitUntilAsync(() => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0, Timeout, "the answer"); + + var answer = h.OutputsOfKind(DerivationOutput.AnswerKind).Single(); + Assert.AreEqual("The city is Palo Alto. Checking. The weather is Sunny and 80°F.", answer.PayloadJson); + Assert.IsTrue(h.OutputsOfKind(DerivationOutput.TraceKind).Count > 0, "traces should flow to egress"); + } + + [TestMethod] + [Timeout(60_000)] + public async Task Kill_WhileAwaitingLlm_RecoversAndReissuesSameRequest() + { + var dir = NewDirectory(); + string pendingRequestBefore; + + // --- Run 1: the LLM call is in flight (frozen) when the process dies. --- + { + await using var h = new SubstrateHarness(s_scheduler, dir); + var llm = new FakeLlmService(); + llm.Freeze(); // first generation parks forever + h.Llm = llm; + h.Tools.Add(FakeTools.Weather()); + + await h.StartFreshAsync(); + await h.AskAsync("d1", "What's the weather in Palo Alto?"); + await h.WaitUntilAsync(() => llm.Calls == 1, Timeout, "the frozen LLM call to start"); + + await h.CheckpointAsync(); // per-step checkpoint: state says Synthesizing + pending RequestLlm + await h.KillAsync(); + + pendingRequestBefore = llm.Prompts.Single().ToJson(); + } + + // --- Run 2: fresh process; recovery re-issues the SAME request; the script completes. --- + { + await using var h = new SubstrateHarness(s_scheduler, dir); + var llm = new FakeLlmService().Segment(Segment1).Segment(Segment2).Segment(Segment3, stoppedAtHedge: false); + h.Llm = llm; + h.Tools.Add(FakeTools.Weather()); + + await h.RecoverAsync(); + + await h.WaitUntilAsync(() => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0, Timeout, "the answer after recovery"); + + Assert.HasCount(1, h.OutputsOfKind(DerivationOutput.AnswerKind), "exactly one answer in the recovered run"); + Assert.Contains("Sunny and 80°F", h.OutputsOfKind(DerivationOutput.AnswerKind)[0].PayloadJson); + Assert.AreEqual(3, llm.Calls, "recovery re-issues the pending generation, then the two follow-ups"); + + // Determinism: the re-issued request is byte-identical to the one that was in flight. + Assert.AreEqual(pendingRequestBefore, llm.Prompts.First().ToJson()); + } + } + + [TestMethod] + [Timeout(60_000)] + public async Task Kill_WhileAwaitingTool_RecoversAndReinvokesIdempotentTool() + { + var dir = NewDirectory(); + + // --- Run 1: the tool call is in flight (frozen) when the process dies. --- + { + await using var h = new SubstrateHarness(s_scheduler, dir); + h.Llm = new FakeLlmService().Segment(Segment1); + var (tool, _, invocations) = FakeTools.FrozenWeather(); + h.Tools.Add(tool); + + await h.StartFreshAsync(); + await h.AskAsync("d1", "What's the weather in Palo Alto?"); + await h.WaitUntilAsync(() => invocations() == 1, Timeout, "the frozen tool call to start"); + + await h.CheckpointAsync(); // state: AwaitingTools + pending InvokeTool + await h.KillAsync(); + } + + // --- Run 2: recovery re-invokes the idempotent tool; derivation completes. --- + { + await using var h = new SubstrateHarness(s_scheduler, dir); + h.Llm = new FakeLlmService().Segment(Segment2).Segment(Segment3, stoppedAtHedge: false); + var invoked = 0; + h.Tools.Add(FakeTools.Weather("\"Recovered and 60°F\"", () => Interlocked.Increment(ref invoked))); + + await h.RecoverAsync(); + + await h.WaitUntilAsync(() => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0, Timeout, "the answer after recovery"); + + Assert.HasCount(1, h.OutputsOfKind(DerivationOutput.AnswerKind)); + Assert.Contains("Recovered and 60°F", h.OutputsOfKind(DerivationOutput.AnswerKind)[0].PayloadJson); + Assert.AreEqual(1, invoked, "exactly one re-invocation of the pending tool call"); + } + } + + private static readonly Universalis.Core.Evaluation.PredicateSignature SendSignature = new("SEND", [ + new Universalis.Core.Evaluation.PredicateParam("message", Universalis.Core.Ir.ParamMode.In), + new Universalis.Core.Evaluation.PredicateParam("receipt", Universalis.Core.Ir.ParamMode.Out), + ], "sends a message (side-effecting)"); + + [TestMethod] + [Timeout(60_000)] + public async Task Kill_WhileAwaitingNonIdempotentTool_DoesNotDoubleFire() + { + var dir = NewDirectory(); + + // --- Run 1: a SIDE-EFFECTING (non-idempotent) tool call is in flight at the kill. --- + { + await using var h = new SubstrateHarness(s_scheduler, dir); + h.Llm = new FakeLlmService().Segment("Sending it [SEND(\"hello\", @receipt)"); + + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = 0; + + h.Tools.Add(new Automind.Tools.DelegateTool(SendSignature, "sends a message", isIdempotent: false, + async (_, ct) => + { + Interlocked.Increment(ref started); + return [await gate.Task.WaitAsync(ct)]; + })); + + await h.StartFreshAsync(); + await h.AskAsync("d1", "Send hello."); + await h.WaitUntilAsync(() => started == 1, Timeout, "the frozen send to start"); + + await h.CheckpointAsync(); // state: AwaitingTools + pending InvokeTool (non-idempotent) + await h.KillAsync(); + } + + // --- Run 2: recovery must NOT re-fire the send (review finding: the IsIdempotent flag + // was never consulted); it synthesizes ToolFailed, backtracks, and finishes without it. --- + { + await using var h = new SubstrateHarness(s_scheduler, dir); + h.Llm = new FakeLlmService() + .Segment("Could not resend safely. Noting [@status = \"send interrupted\"") + .Segment(". Done.", stoppedAtHedge: false); + + var refired = 0; + + h.Tools.Add(new Automind.Tools.DelegateTool(SendSignature, "sends a message", isIdempotent: false, + (_, _) => + { + Interlocked.Increment(ref refired); + return Task.FromResult>(["\"receipt\""]); + })); + + await h.RecoverAsync(); + await h.WaitUntilAsync(() => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0, Timeout, "the answer after recovery"); + + Assert.AreEqual(0, refired, "a non-idempotent tool must never re-fire on recovery — its side effect may already have happened"); + Assert.Contains("NOT re-issued", h.Transcript()); + } + } + + [TestMethod] + [Timeout(60_000)] + public async Task Kill_MidLiftedBatch_RecoversTheRemainingInvocations() + { + var dir = NewDirectory(); + var signature = new Universalis.Core.Evaluation.PredicateSignature("CONVERT", [ + new Universalis.Core.Evaluation.PredicateParam("src", Universalis.Core.Ir.ParamMode.In), + new Universalis.Core.Evaluation.PredicateParam("dst", Universalis.Core.Ir.ParamMode.Out), + ], "convert one file"); + + // --- Run 1: a lifted 3-call batch; ONE invocation succeeds, the others freeze. --- + { + await using var h = new SubstrateHarness(s_scheduler, dir); + h.Llm = new FakeLlmService().Segment("Converting them all [CONVERT(@files, @out)"); + + var succeeded = 0; + var frozen = 0; + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + h.Tools.Add(new Automind.Tools.DelegateTool(signature, "converts one file", isIdempotent: true, + async (argsJson, ct) => + { + if (argsJson.Contains("\"a\"", StringComparison.Ordinal)) + { + Interlocked.Increment(ref succeeded); + return ["\"a.pdf\""]; + } + + Interlocked.Increment(ref frozen); + return [await gate.Task.WaitAsync(ct)]; + })); + + await h.StartFreshAsync(); + await h.AskEnvelopeAsync("batch", new Automind.Kernel.Contract.QuestionEnvelope( + "Convert everything in @files.", + new Dictionary { ["files"] = """["a","b","c"]""" }.ToImmutableDictionary(), + [], false)); + + await h.WaitUntilAsync(() => succeeded == 1 && frozen == 2, Timeout, "one success, two frozen"); + await Task.Delay(750); // let the partial ToolSucceeded step process and mark state dirty + + await h.CheckpointAsync(); // state: AwaitingTools with TWO outstanding invocations + await h.KillAsync(); + } + + // --- Run 2: recovery must re-issue exactly the REMAINING invocations and complete. --- + { + await using var h = new SubstrateHarness(s_scheduler, dir); + h.Llm = new FakeLlmService().Segment(". All converted.", stoppedAtHedge: false); + + var reinvoked = new System.Collections.Concurrent.ConcurrentBag(); + + h.Tools.Add(new Automind.Tools.DelegateTool(signature, "converts one file", isIdempotent: true, + (argsJson, _) => + { + reinvoked.Add(argsJson); + var name = System.Text.Json.JsonDocument.Parse(argsJson).RootElement.GetProperty("src").GetString()!; + return Task.FromResult>([$"\"{name}.pdf\""]); + })); + + await h.RecoverAsync(); + + await h.WaitUntilAsync(() => !reinvoked.IsEmpty, TimeSpan.FromSeconds(15), "any re-issued batch invocation to reach the tool"); + await h.WaitUntilAsync(() => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0, Timeout, "the answer after batch recovery"); + + Assert.HasCount(1, h.OutputsOfKind(DerivationOutput.AnswerKind)); + Assert.IsTrue(reinvoked.Count is 2 or 3, + $"recovery re-issues the outstanding batch invocations (got {reinvoked.Count})"); + } + } + + [TestMethod] + [Timeout(60_000)] + public async Task Kill_AfterCompletion_ButBeforeCheckpoint_ReplaysFromLastCheckpoint() + { + var dir = NewDirectory(); + + // --- Run 1: completes fully, but nothing after the bootstrap checkpoint was persisted. --- + { + await using var h = new SubstrateHarness(s_scheduler, dir); + h.Llm = new FakeLlmService().Segment(Segment1).Segment(Segment2).Segment(Segment3, stoppedAtHedge: false); + h.Tools.Add(FakeTools.Weather()); + + await h.StartFreshAsync(); + await h.AskAsync("d1", "What's the weather in Palo Alto?"); + await h.WaitUntilAsync(() => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0, Timeout, "the first-run answer"); + + await h.KillAsync(); // no checkpoint since bootstrap: the derivation state is lost, the SUBSCRIPTION is not (WAL) + } + + // --- Run 2: the subscription recovers with no operator state → the derivation replays whole. --- + { + await using var h = new SubstrateHarness(s_scheduler, dir); + h.Llm = new FakeLlmService().Segment(Segment1).Segment(Segment2).Segment(Segment3, stoppedAtHedge: false); + h.Tools.Add(FakeTools.Weather()); + + await h.RecoverAsync(); + + await h.WaitUntilAsync(() => h.OutputsOfKind(DerivationOutput.AnswerKind).Count > 0, Timeout, "the replayed answer"); + + Assert.HasCount(1, h.OutputsOfKind(DerivationOutput.AnswerKind), "at-least-once: the whole derivation replays, once"); + } + } + + [TestMethod] + [Timeout(60_000)] + public async Task Checkpoint_CompletesQuickly_WhileLlmCallIsFrozenInFlight() + { + await using var h = new SubstrateHarness(s_scheduler, NewDirectory()); + var llm = new FakeLlmService(); + llm.Freeze(); + h.Llm = llm; + h.Tools.Add(FakeTools.Weather()); + + await h.StartFreshAsync(); + await h.AskAsync("d1", "What's the weather?"); + await h.WaitUntilAsync(() => llm.Calls == 1, Timeout, "the frozen LLM call to start"); + + var stopwatch = Stopwatch.StartNew(); + await h.CheckpointAsync(); + stopwatch.Stop(); + + Assert.IsLessThan(250, stopwatch.ElapsedMilliseconds, + $"checkpoint must not wait on in-flight LLM work (took {stopwatch.ElapsedMilliseconds} ms)"); + } +} diff --git a/tests/Automind.Reaqtor.Tests/SubstrateHarness.cs b/tests/Automind.Reaqtor.Tests/SubstrateHarness.cs new file mode 100644 index 0000000..a8f2383 --- /dev/null +++ b/tests/Automind.Reaqtor.Tests/SubstrateHarness.cs @@ -0,0 +1,176 @@ +using System.Collections.Concurrent; + +using Automind.Kernel; +using Automind.Kernel.Contract; +using Automind.Reaqtor.Catalog; +using Automind.Reaqtor.Client; +using Automind.Reaqtor.Engine; +using Automind.Reaqtor.IO; +using Automind.Reaqtor.Llm; +using Automind.Reaqtor.Reactive; +using Automind.Reaqtor.Store; +using Automind.Tools; + +using Reaqtive.Scheduler; + +using Reaqtor.Shebang.Service; + +namespace Automind.Reaqtor.Tests; + +/// +/// Drives a full engine over a durable file store. "Kill" fidelity: +/// disposes the engine WITHOUT unloading or checkpointing (in-flight work is cancelled, exactly +/// as process death abandons it); reopens the store directory with a +/// completely fresh object graph, pre-creates topics from the conversation catalog (the recovery +/// -order requirement), and recovers a new engine — everything not persisted is gone. +/// +public sealed class SubstrateHarness : IAsyncDisposable +{ + private readonly PhysicalScheduler _scheduler; + private readonly string _directory; + + private SimplerCheckpointingQueryEngine? _engine; + private FileQueryEngineStateStore _store = null!; + private AutomindIngressEgressManager _iemgr = null!; + + public ILlmService Llm { get; set; } = new FakeLlmService(); + + public ToolRegistry Tools { get; } = new(); + + public ConcurrentQueue<(long WireSeq, DerivationOutput Output)> Outputs { get; } = new(); + + public ConversationCatalog Conversations { get; private set; } = null!; + + public SubstrateHarness(PhysicalScheduler scheduler, string directory) + { + _scheduler = scheduler; + _directory = directory; + } + + private AutomindServices BuildServices() => new( + new DerivationStep(), + new ToolRegistryStepContextProvider(Tools), + Llm, + Tools); + + public async Task StartFreshAsync() + { + _store = FileQueryEngineStateStore.Open(_directory); + Conversations = new ConversationCatalog(_store); + _iemgr = new AutomindIngressEgressManager(); + + _engine = await AutomindEngineFactory.CreateNewAsync( + _store, _scheduler, BuildServices().ToDictionary(), _iemgr); + } + + /// Reopens everything from disk — the process-restart path. + public async Task RecoverAsync() + { + _store = FileQueryEngineStateStore.Open(_directory); + Conversations = new ConversationCatalog(_store); + _iemgr = new AutomindIngressEgressManager(); + + // Recovery-order requirement: topics must exist (and collectors be attached) BEFORE the + // engine recovers, because recovered egress observers resolve topics inside SetContext. + foreach (var record in Conversations.All()) + { + SubscribeCollector(record.Topic); + } + + _engine = await AutomindEngineFactory.RecoverAsync( + _store, _scheduler, BuildServices().ToDictionary(), _iemgr); + } + + /// Process death: no unload, no checkpoint; in-flight work is abandoned. + public Task KillAsync() + { + _engine?.Dispose(); + _engine = null; + return Task.CompletedTask; + } + + public Task CheckpointAsync() => _engine!.CheckpointAsync(); + + public Task AskAsync(string derivationId, string question) => + AskEnvelopeAsync(derivationId, QuestionEnvelope.ForText(question)); + + public async Task AskEnvelopeAsync(string derivationId, QuestionEnvelope envelope) + { + var topic = $"automind/out/{derivationId}"; + SubscribeCollector(topic); + + await Conversations.UpsertAsync(new ConversationRecord( + derivationId, topic, envelope.Text, ConversationRecord.PendingStatus)); + + var ctx = AutomindClientContext.For(_engine!); + + await ctx.Derivation(derivationId, envelope.ToJson()).SubscribeAsync( + ctx.Egress(topic), + new Uri($"automind://derivations/{derivationId}"), + state: null, + CancellationToken.None); + + return topic; + } + + private void SubscribeCollector(string topic) => + _iemgr.GetOrCreateSubject(topic) + .Subscribe(new DelegateObserver<(long, DerivationOutput)>(v => Outputs.Enqueue(v))); + + // ---------------------------------------------------------------- assertion helpers + + public List OutputsOfKind(string kind) => + [.. Outputs.Select(o => o.Output).Where(o => o.Kind == kind)]; + + /// Human-readable trace dump for live-test diagnostics. + public string Transcript() + { + var lines = Outputs + .OrderBy(o => o.Output.Seq) + .Select(o => $"[{o.Output.Seq:D3}] {o.Output.Kind}: {o.Output.PayloadJson}"); + + return string.Join("\n", lines); + } + + public async Task WaitUntilAsync(Func condition, TimeSpan timeout, string what) + { + var deadline = DateTimeOffset.UtcNow + timeout; + + while (!condition()) + { + if (DateTimeOffset.UtcNow > deadline) + { + var seen = string.Join(", ", Outputs.Select(o => o.Output.ToString())); + Assert.Fail($"Timed out waiting for: {what}. Outputs seen: [{seen}]"); + } + + await Task.Delay(25); + } + } + + public async ValueTask DisposeAsync() + { + if (_engine is not null) + { + await _engine.UnloadAsync(); + _engine.Dispose(); + } + } + + private sealed class DelegateObserver : IObserver + { + private readonly Action _onNext; + + public DelegateObserver(Action onNext) => _onNext = onNext; + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void OnNext(T value) => _onNext(value); + } +} diff --git a/tests/Automind.Reaqtor.Tests/ThinkFilterTests.cs b/tests/Automind.Reaqtor.Tests/ThinkFilterTests.cs new file mode 100644 index 0000000..682b07d --- /dev/null +++ b/tests/Automind.Reaqtor.Tests/ThinkFilterTests.cs @@ -0,0 +1,81 @@ +using Automind.Reaqtor.Llm; + +namespace Automind.Reaqtor.Tests; + +[TestClass] +public sealed class ThinkFilterTests +{ + [TestMethod] + public void PlainText_PassesThroughUnchanged() + { + var filter = new ThinkFilter(); + Assert.AreEqual("The weather is [WEATHER(\"Palo Alto\", @w)", filter.Push("The weather is [WEATHER(\"Palo Alto\", @w)")); + Assert.AreEqual("", filter.Flush()); + } + + [TestMethod] + public void LeadingThinkBlock_IsSuppressed_EvenSplitAcrossChunks() + { + var filter = new ThinkFilter(); + + // qwen3-style: thought first, then the answer — tags split at awkward chunk boundaries. + var output = filter.Push("let me compute [@x is 5] first…The answer [@x is 5"); + + Assert.AreEqual("The answer [@x is 5", output, + "hedges inside the chain of thought must never reach the scanner"); + Assert.AreEqual("", filter.Flush()); + } + + [TestMethod] + public void ConsecutiveLeadingBlocks_AllSuppressed() + { + var filter = new ThinkFilter(); + + // Inter-block whitespace passes through (it never ends the leading region). + Assert.AreEqual(" answer", filter.Push("a b answer")); + } + + [TestMethod] + public void MidAnswerThinkTag_IsLiteralProse_NeverSwallowsTheRest() + { + // Review finding: an unmatched mid-answer '' echo (granite parroting a prompt + // token) used to flip the filter into permanent suppression and silently discard every + // subsequent hedge. Once the answer has begun, the tag is ordinary text. + var filter = new ThinkFilter(); + var output = filter.Push("The doc says starts a block. Now [@x is 5"); + + Assert.AreEqual("The doc says starts a block. Now [@x is 5", output); + Assert.AreEqual("", filter.Flush()); + + // Even a well-formed mid-answer block stays literal — thinking only leads. + var filter2 = new ThinkFilter(); + Assert.AreEqual("one btwo", filter2.Push("aone btwo")); + } + + [TestMethod] + public void ComparisonAngleBracket_PassesThrough() + { + var filter = new ThinkFilter(); + Assert.AreEqual("- If [@a < @b], then", filter.Push("- If [@a < @b], then")); + + // A '<' at the end of the stream after content is literal immediately (no holdback). + var filter2 = new ThinkFilter(); + Assert.AreEqual("x this never closes body = + [ + new Comment("The profit is "), + Hedge("@d is @sell - @buy"), + new Comment(" and the percentage is "), + Hedge("@pct is (@d / @buy) * 100"), + new Comment("."), + ]; + + return new RuleDefinition( + new RuleSignature("PROFITPCT", [ + new RuleParam("buy", ParamMode.In), + new RuleParam("sell", ParamMode.In), + new RuleParam("pct", ParamMode.Out), + ], "profit percentage"), + "profit percentage", + new UniversalisProgram(body, [], [])); + } + + private static HedgeItem Hedge(string content) + { + var parsed = HedgeParser.Parse(content); + Assert.IsTrue(parsed.Success, parsed.Error); + return new HedgeItem(parsed.Statement!, content); + } + + [TestMethod] + public void Compile_EncodesOperationsAsUriNamedParameters() + { + var tree = BonsaiCompiler.Compile(ApplesRule()); + var text = tree.ToString(); + + Assert.Contains("universalis://rule/v1", text); + Assert.Contains("universalis://is/sub", text); + Assert.Contains("universalis://is/div", text); + Assert.Contains("universalis://is/mul", text); + Assert.Contains("universalis://result", text); + } + + [TestMethod] + public void BonsaiJson_RoundTrips_StructurallyIdentical() + { + var tree = BonsaiCompiler.Compile(ApplesRule()); + + var json = BonsaiSerialization.ToBonsaiJson(tree); + Assert.Contains("universalis://is/sub", json); + + var back = BonsaiSerialization.FromBonsaiJson(json); + var jsonAgain = BonsaiSerialization.ToBonsaiJson(back); + + Assert.AreEqual(json, jsonAgain, "serialize ∘ deserialize must be identity on the wire form"); + } + + [TestMethod] + public void Compile_ToolCall_ExtractsOutputsFromResult() + { + var rule = new RuleDefinition( + new RuleSignature("CITYWEATHER", [ + new RuleParam("place", ParamMode.In), + new RuleParam("w", ParamMode.Out), + ], "weather lookup"), + "weather", + new UniversalisProgram([Hedge("WEATHER(@place, @w)")], [], [])); + + var json = BonsaiSerialization.ToBonsaiJson(BonsaiCompiler.Compile(rule)); + + Assert.Contains("tool://WEATHER", json); + Assert.Contains("universalis://match/extract", json); + } + + [TestMethod] + public void Compile_Conditional_BecomesConditionNodes() + { + ImmutableArray body = + [ + Hedge("@cash is 250"), + Hedge("@price is 180"), + new ConditionalBlock([ + new ConditionalBranch( + HedgeParser.Parse("@cash >= @price").Statement, + "- If ", + [Hedge("@left is @cash - @price")]), + new ConditionalBranch(null, "- Otherwise, ", [Hedge("@left = @cash")]), + ]), + ]; + + var rule = new RuleDefinition( + new RuleSignature("DECIDE", [new RuleParam("left", ParamMode.Out)], "budget decision"), + "decision", + new UniversalisProgram(body, [], [])); + + var json = BonsaiSerialization.ToBonsaiJson(BonsaiCompiler.Compile(rule)); + + Assert.Contains("universalis://cmp/ge", json); + } +} diff --git a/tests/Universalis.Core.Tests/ComprehensionTests.cs b/tests/Universalis.Core.Tests/ComprehensionTests.cs new file mode 100644 index 0000000..bcf5f14 --- /dev/null +++ b/tests/Universalis.Core.Tests/ComprehensionTests.cs @@ -0,0 +1,247 @@ +using System.Text.Json.Nodes; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; +using Universalis.Core.Parsing; + +namespace Universalis.Core.Tests; + +[TestClass] +public sealed class ComprehensionTests +{ + private static HedgeItem Hedge(string content) + { + var parsed = HedgeParser.Parse(content); + Assert.IsTrue(parsed.Success, parsed.Error); + return new HedgeItem(parsed.Statement!, content); + } + + private static ComprehensionDraft Draft(LiterateRecognizer recognizer, params (string Prose, string? Hedge)[] chunks) + { + var events = new List(); + + foreach (var (prose, hedge) in chunks) + { + events.AddRange(recognizer.Advance(prose, hedge is null ? null : Hedge(hedge))); + } + + events.AddRange(recognizer.Finish()); + return events.OfType().Single().Draft; + } + + /// Paper 2's customers example: filter + count. + [TestMethod] + public void Customers_FilterAndCount_FromPaper() + { + var draft = Draft(new LiterateRecognizer(), + ("Consider each customer ", "@c = { ... \"city\": @city ... }"), + (" from ", "@customers"), + (":\n- Retain only customers ", "@c"), + (" that live in Palo Alto ", "@city = \"Palo Alto\""), + (".\n- Subsequently, increment ", "@total"), + (" by one for each retained customer ", "@c"), + (".", null)); + + var (block, error) = ComprehensionCompiler.Compile(draft); + Assert.IsNotNull(block, error); + + var env = EvalEnv.FromSigma(new Dictionary + { + ["customers"] = """ + [{"name":"Ada","city":"Palo Alto"},{"name":"Grace","city":"Seattle"}, + {"name":"Alan","city":"Palo Alto"},{"name":"Edsger","city":"Austin"}] + """, + }); + + var outcome = QueryPipeline.Execute(block!, env, out var rowsIn, out var rowsOut); + + var bound = (Bound)outcome; + Assert.AreEqual("2", bound.Bindings.Single(b => b.Var == "total").Json); + Assert.AreEqual(4, rowsIn); + Assert.AreEqual(2, rowsOut); + } + + /// Paper 2's World Cup players example: group, aggregate, collect, HAVING — nested results. + [TestMethod] + public void Players_GroupAggregateCollectHaving_FromPaper() + { + var draft = Draft(new LiterateRecognizer(), + ("Consider each player ", "@P = {\"position\": @Position, \"stats\": @Stats, \"games\": @Games, \"age\": @Age}"), + (" from ", "@Players"), + (".\nGroup each player ", "@P"), + (" by their position ", "@Position"), + (". This organizes players into groups based on their playing positions.\n" + + "For each group of players in the same position:\n- Determine the average stats ", "@Stats"), + (" of these players as ", "{ \"averageStats\": @AverageStats }"), + (". This gives an overall measure.\n- Find the minimum number of games ", "@Games"), + (" played by any player in this group as ", "{ \"minGames\": @MinimumGames }"), + (".\n- Collect all players ", "@P"), + (" in this group as ", "{ \"players\": @PlayerSet }"), + (".\nKeep only groups where ", "@AverageStats > 100"), + (" and ", "@MinimumGames > 3"), + (".", null)); + + var (block, error) = ComprehensionCompiler.Compile(draft); + Assert.IsNotNull(block, error); + + Assert.IsInstanceOfType(block!.Ops[0], "grouping must come first"); + Assert.HasCount(2, block.Ops.OfType().ToList(), "two HAVING clauses"); + + var env = EvalEnv.FromSigma(new Dictionary + { + // NB: the paper's item pattern is CLOSED (no "...") — rows must have exactly its keys. + ["Players"] = """ + [{"position":"Forward","stats":120,"games":10,"age":25}, + {"position":"Forward","stats":110,"games":8,"age":27}, + {"position":"Defender","stats":90,"games":12,"age":24}, + {"position":"Goalie","stats":150,"games":2,"age":30}] + """, + }); + + var outcome = QueryPipeline.Execute(block, env, out var rowsIn, out var rowsOut); + + if (outcome is EvalFailure failure) + { + Assert.Fail($"pipeline failed: {failure.Code}: {failure.Message} ({failure.Hint})"); + } + + var bound = (Bound)outcome; + var result = JsonNode.Parse(bound.Bindings.Single(b => b.Var == "queryResult").Json)!.AsArray(); + + // Forward: avg 115 > 100, min games 8 > 3 → kept. Defender: avg 90 → out. Goalie: min games 2 → out. + Assert.HasCount(1, result); + var forwards = result[0]!.AsObject(); + Assert.AreEqual("Forward", forwards["Position"]!.GetValue()); + Assert.AreEqual(115, forwards["averageStats"]!.GetValue()); + Assert.AreEqual(8, forwards["minGames"]!.GetValue()); + Assert.HasCount(2, forwards["players"]!.AsArray(), "nested collect — the anti-SQL superpower"); + Assert.AreEqual(4, rowsIn); + Assert.AreEqual(1, rowsOut); + } + + /// + /// Replays a live granite trajectory (2026-07-16 customers count) verbatim: filter and count + /// fused into the Retain bullet, a counter-initialization bullet, an all-prose loop header, + /// and an If bullet re-narrating the registered ops as a walkthrough. All tolerated; the + /// query still computes 2. + /// + [TestMethod] + public void Customers_LoopWalkthroughNarration_CompilesToFilterAndCount() + { + var draft = Draft(new LiterateRecognizer(), + ("Consider each customer ", "@c = { ... \"city\": @city ... }"), + (" from ", "@customers"), + (":\n- Retain only customers ", "@c"), + (" where ", "@city = \"Palo Alto\""), + (" and increment ", "@total"), + (" by one for each.\n- Initially, ", "@total = 0"), + ("\n- For each customer ", "@c"), + (" in ", "@customers"), + ("\n - If ", "@city = \"Palo Alto\""), + (" then increment ", "@total"), + (" by one.", null)); + + var (block, error) = ComprehensionCompiler.Compile(draft); + Assert.IsNotNull(block, error); + + Assert.HasCount(1, block!.Ops.OfType().ToList(), "the walkthrough must not double the filter"); + Assert.HasCount(1, block.Ops.OfType().ToList(), "the walkthrough must not double the count"); + + var env = EvalEnv.FromSigma(new Dictionary + { + ["customers"] = """[{"city":"Palo Alto"},{"city":"Seattle"},{"city":"Palo Alto"},{"city":"Austin"}]""", + }); + + var outcome = QueryPipeline.Execute(block, env, out _, out var rowsOut); + + var bound = (Bound)outcome; + Assert.AreEqual("2", bound.Bindings.Single(b => b.Var == "total").Json); + Assert.AreEqual(2, rowsOut); + } + + /// + /// The negated walkthrough variant: "- If [@city != "Palo Alto"], then skip …" after the + /// filter is registered is the same filter stated from the other side. + /// + [TestMethod] + public void NegatedSkipNarration_AfterFilter_IsTolerated() + { + var draft = Draft(new LiterateRecognizer(), + ("Consider each customer ", "@c = { ... \"city\": @city ... }"), + (" from ", "@customers"), + (":\n- Retain only those living in Palo Alto ", "@city = \"Palo Alto\""), + (".\n- Subsequently, increment ", "@total"), + (" by one.\n- If ", "@city != \"Palo Alto\""), + (", then skip the customer ", "@c"), + (" and continue.", null)); + + var (block, error) = ComprehensionCompiler.Compile(draft); + Assert.IsNotNull(block, error); + Assert.HasCount(1, block!.Ops.OfType().ToList()); + } + + /// + /// Observed live: with no row pattern, the filter [@city = "Palo Alto"] BOUND free @city on + /// every row — an always-true filter that counted 4 of 4 and answered wrongly. A free + /// variable bound to a ground literal in filter position must fail with the pattern teach. + /// + [TestMethod] + public void FilterOnNeverBoundField_FailsWithDestructuringTeach() + { + var draft = Draft(new LiterateRecognizer(), + ("Consider each customer ", "@c"), + (" from ", "@customers"), + (":\n- Retain only customers where ", "@city = \"Palo Alto\""), + (".\n- Subsequently, increment ", "@total"), + (" by one.", null)); + + var (block, error) = ComprehensionCompiler.Compile(draft); + Assert.IsNotNull(block, error); + + var env = EvalEnv.FromSigma(new Dictionary + { + ["customers"] = """[{"city":"Palo Alto"},{"city":"Seattle"}]""", + }); + + var outcome = QueryPipeline.Execute(block!, env, out _, out _); + + var failure = (EvalFailure)outcome; + Assert.Contains("never bound", failure.Message); + Assert.Contains("pattern", failure.Hint!); + } + + [TestMethod] + public void IfBullet_TeachesFilterThenCountSplit_NeverCompilesUnconditionalCount() + { + // Observed live (customers count): "- If [@city = "Palo Alto"], then increment [@total] + // by one." Its prose contains "increment", so without the guard it compiles as an + // UNCONDITIONAL CountIntoOp — counting all rows, silently wrong. + var draft = Draft(new LiterateRecognizer(), + ("Consider each customer ", "@c = { ... \"city\": @city ... }"), + (" from ", "@customers"), + (":\n- If ", "@city = \"Palo Alto\""), + (", then increment ", "@total"), + (" by one.", null)); + + var (block, error) = ComprehensionCompiler.Compile(draft); + + Assert.IsNull(block, "an If bullet must never compile"); + Assert.Contains("Retain only", error!); + Assert.Contains("@city = \"Palo Alto\"", error!, "the teach must quote the model's own condition"); + } + + [TestMethod] + public void UnintelligibleBulletWithHedges_CompilesToTeachingError() + { + var draft = Draft(new LiterateRecognizer(), + ("Consider each item ", "@x"), + (" from ", "@items"), + (":\n- Do something mysterious with ", "@x = 5"), + (".", null)); + + var (block, error) = ComprehensionCompiler.Compile(draft); + + Assert.IsNull(block); + Assert.Contains("Retain only", error!); + } +} diff --git a/tests/Universalis.Core.Tests/EvaluatorTests.cs b/tests/Universalis.Core.Tests/EvaluatorTests.cs new file mode 100644 index 0000000..8377452 --- /dev/null +++ b/tests/Universalis.Core.Tests/EvaluatorTests.cs @@ -0,0 +1,249 @@ +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; +using Universalis.Core.Parsing; + +namespace Universalis.Core.Tests; + +[TestClass] +public sealed class EvaluatorTests +{ + private static Statement Stmt(string hedge) + { + var result = HedgeParser.Parse(hedge); + Assert.IsTrue(result.Success, result.Error); + return result.Statement!; + } + + private static EvalEnv Env(params (string Name, string Json)[] bindings) => + EvalEnv.FromSigma(bindings.ToDictionary(b => b.Name, b => b.Json)); + + private static EvalOutcome Eval(string hedge, EvalEnv env, ISignatureCatalog? catalog = null) => + Evaluator.Evaluate(Stmt(hedge), env, catalog ?? SignatureCatalog.Empty); + + // ---------------------------------------------------------------- is / numeric tower + + [TestMethod] + public void Is_ApplesProfit_DecimalExact() + { + var bound = (Bound)Eval("@P is (@D/@B)*100", Env(("D", "7"), ("B", "10"))); + + Assert.AreEqual("70", bound.Bindings.Single(b => b.Var == "P").Json); + } + + [TestMethod] + public void Is_CoercesNumericJsonString_PaperBtcExample() + { + // The STOCK API returns "close" as a JSON string; the paper multiplies it directly. + var bound = (Bound)Eval("@total is @price * @count", Env(("price", "\"181.58000\""), ("count", "2"))); + + Assert.AreEqual("363.16", bound.Bindings[0].Json); + } + + [TestMethod] + public void Is_NonNumericString_TypeError() + { + // Paper 1: [@Z is @X + "hello"] has no derivation. Strings can't appear literally in + // arith, so the equivalent is a variable bound to a non-numeric string. + var failure = (EvalFailure)Eval("@Z is @X + @hello", Env(("X", "1"), ("hello", "\"hello\""))); + + Assert.AreEqual(EvalFailureCodes.Type, failure.Code); + } + + [TestMethod] + public void Is_DivideByZero_Fails() + { + var failure = (EvalFailure)Eval("@r is @a / @b", Env(("a", "1"), ("b", "0"))); + + Assert.AreEqual(EvalFailureCodes.DivideByZero, failure.Code); + } + + [TestMethod] + public void Is_RebindingBoundVariable_Fails() + { + var failure = (EvalFailure)Eval("@x is 1 + 1", Env(("x", "5"))); + + Assert.AreEqual(EvalFailureCodes.Rebind, failure.Code); + } + + [TestMethod] + public void Is_UnboundOperand_Fails() + { + var failure = (EvalFailure)Eval("@y is @nope + 1", Env()); + + Assert.AreEqual(EvalFailureCodes.Unbound, failure.Code); + } + + // ---------------------------------------------------------------- comparisons + + [TestMethod] + public void Comparison_NumericStrings_Compare() + { + var result = (GuardResult)Eval("@a >= @b", Env(("a", "\"10.5\""), ("b", "10"))); + + Assert.IsTrue(result.Value); + } + + [TestMethod] + public void Comparison_EqualityIsNumericAware() + { + Assert.IsTrue(((GuardResult)Eval("@a == @b", Env(("a", "3"), ("b", "3.0")))).Value); + Assert.IsFalse(((GuardResult)Eval("@a == @b", Env(("a", "\"x\""), ("b", "\"y\"")))).Value); + } + + // ---------------------------------------------------------------- bind (=) + + [TestMethod] + public void Bind_FreeVar_Binds() + { + var bound = (Bound)Eval("@btc_left = @btc_total", Env(("btc_total", "1.5"))); + + Assert.AreEqual("btc_left", bound.Bindings[0].Var); + Assert.AreEqual("1.5", bound.Bindings[0].Json); + } + + [TestMethod] + public void Bind_GroundBothSides_ActsAsTest() + { + Assert.IsTrue(((GuardResult)Eval("@city = \"Palo Alto\"", Env(("city", "\"Palo Alto\"")))).Value); + Assert.IsFalse(((GuardResult)Eval("@city = \"Seattle\"", Env(("city", "\"Palo Alto\"")))).Value); + } + + [TestMethod] + public void Bind_FreeRightSide_ModeError() + { + var failure = (EvalFailure)Eval("@a = @nowhere", Env()); + + Assert.AreEqual(EvalFailureCodes.Mode, failure.Code); + } + + [TestMethod] + public void Bind_GroundPattern_Constructs() + { + var bound = (Bound)Eval("@point = { \"x\": @a, \"y\": 2 }", Env(("a", "1"))); + + Assert.AreEqual("""{"x":1,"y":2}""", bound.Bindings[0].Json); + } + + // ---------------------------------------------------------------- display + + [TestMethod] + public void Display_FormatsStringUnquoted() + { + var display = (DisplayValue)Eval("@w", Env(("w", "\"Sunny and 80°F\""))); + + Assert.AreEqual("Sunny and 80°F", display.Formatted); + Assert.AreEqual("\"Sunny and 80°F\"", display.Json); + } + + // ---------------------------------------------------------------- calls + + private static readonly SignatureCatalog Weather = new([ + new PredicateSignature("WEATHER", [ + new PredicateParam("city", ParamMode.In), + new PredicateParam("weather", ParamMode.Out), + ], "current weather for a city"), + ]); + + [TestMethod] + public void Call_ProducesNeedTool_WithJsonPayload() + { + var need = (NeedTool)Eval("WEATHER(\"Palo Alto\", @w)", Env(), Weather); + + Assert.AreEqual("WEATHER", need.Signature.Name); + Assert.AreEqual(1, need.Plan.Count); + Assert.AreEqual("""{"city":"Palo Alto"}""", need.Plan.InvocationArgsJson[0]); + Assert.AreEqual("w", ((VarTerm)need.OutArgs[0]).Name); + } + + [TestMethod] + public void Call_NamedArguments_MapOntoSignaturePositions() + { + // Fully named, out of order. + var need = (NeedTool)Eval("WEATHER(weather: @w, city: \"Palo Alto\")", Env(), Weather); + Assert.AreEqual("""{"city":"Palo Alto"}""", need.Plan.InvocationArgsJson[0]); + Assert.AreEqual("w", ((VarTerm)need.OutArgs[0]).Name); + + // Mixed: named input + positional output. + var mixed = (NeedTool)Eval("WEATHER(city: \"Seattle\", @x)", Env(), Weather); + Assert.AreEqual("""{"city":"Seattle"}""", mixed.Plan.InvocationArgsJson[0]); + + // A name that isn't a parameter teaches the signature. + var failure = (EvalFailure)Eval("WEATHER(town: \"Palo Alto\", @w)", Env(), Weather); + Assert.AreEqual(EvalFailureCodes.Arity, failure.Code); + Assert.Contains("town", failure.Message); + } + + [TestMethod] + public void Call_UnknownPredicate_HintsKnownNames() + { + var failure = (EvalFailure)Eval("NOPE(@x)", Env(), Weather); + + Assert.AreEqual(EvalFailureCodes.UnknownPredicate, failure.Code); + Assert.Contains("WEATHER", failure.Hint!); + } + + [TestMethod] + public void Call_LiteralInOutPosition_FailsWithRepairableCode() + { + // Paper 1's discard-and-replace: the model hallucinated a value where an output belongs. + var failure = (EvalFailure)Eval("WEATHER(\"Palo Alto\", \"Sunny and 10000°F\")", Env(), Weather); + + Assert.AreEqual(EvalFailureCodes.LiteralInOutPosition, failure.Code); + } + + [TestMethod] + public void Call_WrongArity_HintsSignature() + { + var failure = (EvalFailure)Eval("WEATHER(\"Palo Alto\")", Env(), Weather); + + Assert.AreEqual(EvalFailureCodes.Arity, failure.Code); + Assert.Contains("weather: out", failure.Hint!); + } + + [TestMethod] + public void BindToolResults_SingleOut_BindsBareValue() + { + var need = (NeedTool)Eval("WEATHER(\"Palo Alto\", @w)", Env(), Weather); + + var bound = (Bound)Evaluator.BindToolResults(need.Signature, need.OutArgs, ["\"Sunny and 80°F\""], Env()); + + Assert.AreEqual("w", bound.Bindings[0].Var); + Assert.AreEqual("\"Sunny and 80°F\"", bound.Bindings[0].Json); + } + + [TestMethod] + public void BindToolResults_MultiOut_KeyedObject() + { + var catalog = new SignatureCatalog([ + new PredicateSignature("GEO_CODE", [ + new PredicateParam("city", ParamMode.In), + new PredicateParam("lat", ParamMode.Out), + new PredicateParam("lon", ParamMode.Out), + ], "coordinates of a city"), + ]); + + var need = (NeedTool)Eval("GEO_CODE(\"Palo Alto\", @lat, @lon)", Env(), catalog); + var bound = (Bound)Evaluator.BindToolResults( + need.Signature, need.OutArgs, ["""{"lat":37.44,"lon":-122.14}"""], Env()); + + Assert.AreEqual("37.44", bound.Bindings.Single(b => b.Var == "lat").Json); + Assert.AreEqual("-122.14", bound.Bindings.Single(b => b.Var == "lon").Json); + } + + [TestMethod] + public void BindToolResults_PatternOut_ExtractsFields() + { + var catalog = new SignatureCatalog([ + new PredicateSignature("STOCK", [ + new PredicateParam("symbol", ParamMode.In), + new PredicateParam("data", ParamMode.Out), + ], "stock quote"), + ]); + + var need = (NeedTool)Eval("STOCK(\"IBM\", { ... \"close\": @closePrice ... })", Env(), catalog); + var bound = (Bound)Evaluator.BindToolResults( + need.Signature, need.OutArgs, ["""{"data":[{"close":"181.58"}],"status":"ok"}"""], Env()); + + Assert.AreEqual("\"181.58\"", bound.Bindings.Single(b => b.Var == "closePrice").Json); + } +} diff --git a/tests/Universalis.Core.Tests/HedgeParserTests.cs b/tests/Universalis.Core.Tests/HedgeParserTests.cs new file mode 100644 index 0000000..2dafa1b --- /dev/null +++ b/tests/Universalis.Core.Tests/HedgeParserTests.cs @@ -0,0 +1,298 @@ +using Universalis.Core.Ir; +using Universalis.Core.Parsing; + +namespace Universalis.Core.Tests; + +[TestClass] +public sealed class HedgeParserTests +{ + private static Statement Parse(string content) + { + var result = HedgeParser.Parse(content); + Assert.IsTrue(result.Success, $"parse failed: {result.Error}"); + return result.Statement!; + } + + [TestMethod] + public void Parse_ToolCall_WithStringAndVar() + { + var call = (PredicateCall)Parse("WEATHER(\"Palo Alto\", @weatherPaloAlto)"); + + Assert.AreEqual("WEATHER", call.Name); + Assert.HasCount(2, call.Args); + Assert.AreEqual("Palo Alto", ((StrTerm)call.Args[0]).Value); + Assert.AreEqual("weatherPaloAlto", ((VarTerm)call.Args[1]).Name); + } + + [TestMethod] + public void Parse_IsBinding_ApplesProfit() + { + var isb = (IsBinding)Parse("@D is (@S-@B)"); + + Assert.AreEqual("D", isb.Var); + var bin = (ArithBinary)isb.Expr; + Assert.AreEqual(ArithOp.Sub, bin.Op); + } + + [TestMethod] + public void Parse_IsBinding_HealsPaperUnbalancedParen() + { + // The paper itself prints [@P is (@D/@B)*100)] — one surplus trailing ')'. + var result = HedgeParser.Parse("@P is (@D/@B)*100)"); + + Assert.IsTrue(result.Success, result.Error); + Assert.IsTrue(result.Healed); + var isb = (IsBinding)result.Statement!; + Assert.AreEqual("P", isb.Var); + var mul = (ArithBinary)isb.Expr; + Assert.AreEqual(ArithOp.Mul, mul.Op); + } + + [TestMethod] + public void Parse_DollarSigil_ReadsAsVariableOrAmount() + { + // Observed live: money-priming prose ("Sam has $120") makes models write the dollar + // sigil inside hedges — [$cash is 120] for @cash, and [@price is $200] for an amount. + var bind = (IsBinding)Parse("$cash is 120"); + Assert.AreEqual("cash", bind.Var); + + var amount = (IsBinding)Parse("@price is $200"); + Assert.AreEqual("price", amount.Var); + Assert.AreEqual(200m, ((ArithNum)amount.Expr).Value); + + var compare = (Comparison)Parse("$cash >= $price"); + Assert.AreEqual(CompareOp.Ge, compare.Op); + Assert.AreEqual("cash", ((VarTerm)compare.Left).Name); + } + + [TestMethod] + public void Parse_VarAsPatternKey_ReadsAsFieldName() + { + // Observed live: {@price: @msftPrice} for { "price": @msftPrice }, and the bare pun {@close}. + var call = (PredicateCall)Parse("STOCK(\"MSFT\", { ... @price: @msftPrice ... })"); + var pattern = (ObjectPatternTerm)call.Args[1]; + Assert.AreEqual("price", pattern.Fields[0].Key); + Assert.AreEqual("msftPrice", ((VarTerm)pattern.Fields[0].Value).Name); + + var pun = (PredicateCall)Parse("STOCK(\"MSFT\", { ... @close ... })"); + var punPattern = (ObjectPatternTerm)pun.Args[1]; + Assert.AreEqual("close", punPattern.Fields[0].Key); + Assert.AreEqual("close", ((VarTerm)punPattern.Fields[0].Value).Name); + } + + [TestMethod] + public void Parse_DotPathAccess_TeachesPatternDestructuring() + { + // Observed live: [@weather is @weatherData.properties.observation.condition]. + var result = HedgeParser.Parse("@weather is @weatherData.properties.observation.condition"); + + Assert.IsFalse(result.Success); + Assert.Contains("dot-path", result.Error!); + Assert.Contains("pattern", result.Error!); + } + + [TestMethod] + public void Parse_CallWrappedBinding_HealsToTheInnerStatement() + { + // Observed live (btc-decision, via telemetry): [MATH(@costPerShare is (@shares * 100 / + // @btc))] — the model wraps a computation in an invented call; the wrapper is decoration. + var math = HedgeParser.Parse("MATH(@costPerShare is (@shares * 100 / @btc))"); + Assert.IsTrue(math.Success, math.Error); + Assert.IsTrue(math.Healed); + Assert.AreEqual("costPerShare", ((IsBinding)math.Statement!).Var); + + var set = HedgeParser.Parse("SET(@x = 5)"); + Assert.IsTrue(set.Success, set.Error); + Assert.IsInstanceOfType(set.Statement); + + // Only bindings unwrap — anything else keeps its original diagnostic. + var comparison = HedgeParser.Parse("CHECK(@x > 5)"); + Assert.IsFalse(comparison.Success); + } + + [TestMethod] + public void Parse_ChecklistBulletOrImperativeInsideHedge_TeachesProseShape() + { + // Observed live (customers count): the whole bullet bracketed as ONE hedge, and bare + // imperative verbs — both previously died with dead-end token errors. + var ifBullet = HedgeParser.Parse("- If [@city = \"Palo Alto\"], then retain @c"); + Assert.IsFalse(ifBullet.Success); + Assert.Contains("prose", ifBullet.Error!); + Assert.Contains("Retain only", ifBullet.Error!); + + var imperative = HedgeParser.Parse("increment @total by 1"); + Assert.IsFalse(imperative.Success); + Assert.Contains("increment [@total] by one", imperative.Error!); + + // A real call that happens to start with a teach word still parses ('(' lookahead). + var call = (PredicateCall)Parse("SET(\"key\", @value)"); + Assert.AreEqual("SET", call.Name); + } + + [TestMethod] + public void Parse_BooleanIsAndTernary_TeachTheChecklist() + { + // Observed live: models invent boolean variables and ternaries for decisions. + var boolean = HedgeParser.Parse("@canBuy is @btc >= @cost"); + Assert.IsFalse(boolean.Success); + Assert.Contains("checklist", boolean.Error!); + + var ternary = HedgeParser.Parse("@btcLeft is @btc >= @cost ? @btc - @cost : @btc"); + Assert.IsFalse(ternary.Success); + Assert.Contains("checklist", ternary.Error!); + + // '?' inside a string argument stays legal (string-shielded tokenizer). + var question = (PredicateCall)Parse("SEARCH(\"current price of BTC?\", @result)"); + Assert.AreEqual("SEARCH", question.Name); + } + + [TestMethod] + public void Parse_OpenPattern_CommasOptionalAroundEllipsis() + { + var call = (PredicateCall)Parse("STOCK(\"IBM\", { ... \"volume\": @V ... \"close\": @P ... })"); + + var pattern = (ObjectPatternTerm)call.Args[1]; + Assert.IsTrue(pattern.IsOpen); + Assert.HasCount(2, pattern.Fields); + Assert.AreEqual("volume", pattern.Fields[0].Key); + Assert.AreEqual("close", pattern.Fields[1].Key); + } + + [TestMethod] + public void Parse_UnicodeEllipsis_Accepted() + { + var call = (PredicateCall)Parse("STOCK(\"IBM\", { … \"close\": @P … })"); + + Assert.IsTrue(((ObjectPatternTerm)call.Args[1]).IsOpen); + } + + [TestMethod] + public void Parse_NestedPattern_WeatherGovForecast() + { + var call = (PredicateCall)Parse("WEATHER_GOV(@lat, @lon, { ... \"forecast\": @url ... })"); + + Assert.AreEqual("WEATHER_GOV", call.Name); + Assert.HasCount(3, call.Args); + Assert.IsTrue(((ObjectPatternTerm)call.Args[2]).IsOpen); + } + + [TestMethod] + public void Parse_Comparison_Guard() + { + var cmp = (Comparison)Parse("@btc_total >= @msft_total"); + + Assert.AreEqual(CompareOp.Ge, cmp.Op); + } + + [TestMethod] + public void Parse_Comparison_CompoundArithSide() + { + var cmp = (Comparison)Parse("@total >= @price * @count"); + + Assert.AreEqual(CompareOp.Ge, cmp.Op); + Assert.IsInstanceOfType(cmp.Right); + } + + [TestMethod] + public void Parse_Bind_VarToVar() + { + var bind = (BindStmt)Parse("@btc_left = @btc_total"); + + Assert.AreEqual("btc_left", ((VarTerm)bind.Left).Name); + Assert.AreEqual("btc_total", ((VarTerm)bind.Right).Name); + } + + [TestMethod] + public void Parse_Bind_PatternDestructuring() + { + var bind = (BindStmt)Parse("@c = { ... \"city\": @city ... }"); + + Assert.IsInstanceOfType(bind.Right); + } + + [TestMethod] + public void Parse_Display_Var() + { + var display = (DisplayStmt)Parse("@weatherPaloAlto"); + + Assert.AreEqual("weatherPaloAlto", ((VarTerm)display.Value).Name); + } + + [TestMethod] + public void Parse_Display_Arithmetic() + { + var display = (DisplayStmt)Parse("@x + @y"); + + Assert.IsInstanceOfType(display.Value); + } + + [TestMethod] + public void Parse_ArrayPattern_LeadingEllipsis() + { + var bind = (BindStmt)Parse("@last = [..., @x]"); + + var arr = (ArrayPatternTerm)bind.Right; + Assert.AreEqual(EllipsisPosition.Leading, arr.Ellipsis); + Assert.HasCount(1, arr.Items); + } + + [TestMethod] + public void Parse_DottedToolNames_Allowed() + { + var call = (PredicateCall)Parse("math.eval(\"2+2\", @r)"); + + Assert.AreEqual("math.eval", call.Name); + } + + [TestMethod] + public void Parse_EqEq_IsComparisonNotBind() + { + var cmp = (Comparison)Parse("@city == \"Palo Alto\""); + + Assert.AreEqual(CompareOp.Eq, cmp.Op); + } + + [TestMethod] + public void Parse_Garbage_Fails() + { + Assert.IsFalse(HedgeParser.Parse("@@@!!").Success); + Assert.IsFalse(HedgeParser.Parse("F(@a").Success); + Assert.IsFalse(HedgeParser.Parse("@x is").Success); + } + + [TestMethod] + public void Parse_IsCall_RewritesToCallWithTrailingOutVar() + { + // Live-observed natural syntax: [@x is WEATHER("Palo Alto")] ≡ [WEATHER("Palo Alto", @x)]. + var call = (PredicateCall)Parse("@w is WEATHER(\"Palo Alto\")"); + + Assert.AreEqual("WEATHER", call.Name); + Assert.HasCount(2, call.Args); + Assert.AreEqual("w", ((VarTerm)call.Args[^1]).Name); + } + + [TestMethod] + public void Parse_Conjunction_TeachesOneCallPerHedge() + { + var result = HedgeParser.Parse("TODAY(@now), WEATHER(\"Palo Alto\", @w)"); + + Assert.IsFalse(result.Success); + Assert.Contains("ONE call", result.Error!); + } + + [TestMethod] + public void Parse_NamedArguments_ParseIntoNamedTerms() + { + // Observed live, repeatedly: models write name: value. Accepted; the evaluator maps + // names onto signature positions. + var call = (PredicateCall)Parse("TO_PDF(src: \"a.txt\", dst: @out)"); + + var src = (NamedTerm)call.Args[0]; + Assert.AreEqual("src", src.Name); + Assert.AreEqual("a.txt", ((StrTerm)src.Value).Value); + + var dst = (NamedTerm)call.Args[1]; + Assert.AreEqual("dst", dst.Name); + Assert.AreEqual("out", ((VarTerm)dst.Value).Name); + } +} diff --git a/tests/Universalis.Core.Tests/HedgeScannerTests.cs b/tests/Universalis.Core.Tests/HedgeScannerTests.cs new file mode 100644 index 0000000..c13c9cd --- /dev/null +++ b/tests/Universalis.Core.Tests/HedgeScannerTests.cs @@ -0,0 +1,106 @@ +using Universalis.Core.Parsing; + +namespace Universalis.Core.Tests; + +[TestClass] +public sealed class HedgeScannerTests +{ + [TestMethod] + public void Split_ProseAndHedges_Basic() + { + var segments = HedgeScanner.Split("Let's check the weather [WEATHER(\"Palo Alto\", @w)]. Done."); + + Assert.HasCount(3, segments); + Assert.AreEqual("Let's check the weather ", segments[0].Text); + Assert.IsFalse(segments[0].IsHedge); + Assert.AreEqual("WEATHER(\"Palo Alto\", @w)", segments[1].Text); + Assert.IsTrue(segments[1].IsHedge); + Assert.AreEqual(". Done.", segments[2].Text); + } + + [TestMethod] + public void Split_NestedSquareBrackets_StayInsideHedge() + { + var segments = HedgeScanner.Split("x [@a = [1, 2, 3]] y"); + + Assert.HasCount(3, segments); + Assert.AreEqual("@a = [1, 2, 3]", segments[1].Text); + Assert.IsTrue(segments[1].IsHedge); + } + + [TestMethod] + public void Split_ClosingBracketInsideString_DoesNotClose() + { + var segments = HedgeScanner.Split("[HTTP_GET(\"https://x/a]b\", @r)]"); + + Assert.HasCount(1, segments); + Assert.AreEqual("HTTP_GET(\"https://x/a]b\", @r)", segments[0].Text); + Assert.IsTrue(segments[0].IsHedge); + } + + [TestMethod] + public void Split_EscapedQuoteInsideString_Handled() + { + var segments = HedgeScanner.Split("[F(\"say \\\"hi]\\\" now\", @r)]"); + + Assert.HasCount(1, segments); + Assert.IsTrue(segments[0].IsHedge); + Assert.Contains("hi]", segments[0].Text); + } + + [TestMethod] + public void Split_TruncatedHedge_ReportedOpen() + { + var segments = HedgeScanner.Split("thinking [WEATHER(\"PA\", @w"); + + Assert.HasCount(2, segments); + Assert.IsTrue(segments[1].IsHedge); + Assert.IsTrue(segments[1].IsOpenHedge); + } + + [TestMethod] + public void Push_CutLandsExactlyBeforeClosingBracket() + { + const string Text = "go [F(@x)] rest"; + var scanner = new HedgeScanner(); + var closedAt = -1; + + for (var i = 0; i < Text.Length; i++) + { + if (scanner.Push(Text[i]) == ScanEvent.HedgeClosed) + { + closedAt = i; + } + } + + Assert.AreEqual(Text.IndexOf(']', StringComparison.Ordinal), closedAt); + } + + /// Property: feeding char-by-char must agree with batch splitting (deterministic seeds). + [TestMethod] + public void Property_IncrementalEqualsBatch() + { + var random = new Random(20260716); + var alphabet = "ab [](){}\"\\,@:.".ToCharArray(); + + for (var trial = 0; trial < 500; trial++) + { + var length = random.Next(0, 60); + var chars = new char[length]; + for (var i = 0; i < length; i++) + { + chars[i] = alphabet[random.Next(alphabet.Length)]; + } + + var text = new string(chars); + + var viaSplit = HedgeScanner.Split(text); + + // Reconstruct: prose chars + hedge contents must exactly re-form the input. + var reconstructed = string.Concat(viaSplit.Select(s => + s.IsHedge ? (s.IsOpenHedge ? "[" + s.Text : "[" + s.Text + "]") : s.Text)); + + Assert.AreEqual(text, reconstructed, $"reconstruction mismatch for input: {text}"); + } + } +} diff --git a/tests/Universalis.Core.Tests/LiftingTests.cs b/tests/Universalis.Core.Tests/LiftingTests.cs new file mode 100644 index 0000000..a987e82 --- /dev/null +++ b/tests/Universalis.Core.Tests/LiftingTests.cs @@ -0,0 +1,148 @@ +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; +using Universalis.Core.Parsing; + +namespace Universalis.Core.Tests; + +[TestClass] +public sealed class LiftingTests +{ + private static readonly SignatureCatalog Catalog = new([ + new PredicateSignature("toPdf", [ + new PredicateParam("src", ParamMode.In), + new PredicateParam("dst", ParamMode.Out), + ], "convert a single file to PDF"), + new PredicateSignature("resize", [ + new PredicateParam("file", ParamMode.In), + new PredicateParam("percent", ParamMode.In), + new PredicateParam("result", ParamMode.Out), + ], "resize one file by a percentage"), + new PredicateSignature("archive", [ + new PredicateParam("files", ParamMode.In, AcceptsCollection: true), + new PredicateParam("zip", ParamMode.Out), + ], "zip a list of files"), + ]); + + private static EvalEnv Env(params (string Name, string Json)[] bindings) => + EvalEnv.FromSigma(bindings.ToDictionary(b => b.Name, b => b.Json)); + + private static EvalOutcome Eval(string hedge, EvalEnv env) + { + var parsed = HedgeParser.Parse(hedge); + Assert.IsTrue(parsed.Success, parsed.Error); + return Evaluator.Evaluate(parsed.Statement!, env, Catalog); + } + + [TestMethod] + public void ToPdf_OverList_ZipLiftsToThreeInvocations() + { + // Paper 2: [toPdf(@files, @dst)] over a list ≡ [map(toPDF, @files, @dst)]. + var need = (NeedTool)Eval("toPdf(@files, @dst)", Env(("files", """["a.txt","b.txt","c.txt"]"""))); + + Assert.AreEqual(3, need.Plan.Count); + Assert.IsTrue(need.Plan.IsLifted); + Assert.AreEqual("""{"src":"a.txt"}""", need.Plan.InvocationArgsJson[0]); + Assert.AreEqual("""{"src":"c.txt"}""", need.Plan.InvocationArgsJson[2]); + } + + [TestMethod] + public void Lifted_ScalarBroadcasts_AcrossInvocations() + { + var need = (NeedTool)Eval("resize(@files, @pct, @out)", Env( + ("files", """["a.png","b.png"]"""), + ("pct", "50"))); + + Assert.AreEqual(2, need.Plan.Count); + Assert.AreEqual("""{"file":"a.png","percent":50}""", need.Plan.InvocationArgsJson[0]); + Assert.AreEqual("""{"file":"b.png","percent":50}""", need.Plan.InvocationArgsJson[1]); + } + + [TestMethod] + public void Lifted_TwoArrays_ZipRequiresEqualLength() + { + var failure = (EvalFailure)Eval("resize(@files, @pcts, @out)", Env( + ("files", """["a","b","c"]"""), + ("pcts", "[50, 60]"))); + + Assert.AreEqual(EvalFailureCodes.LiftLength, failure.Code); + Assert.Contains("3", failure.Message); + Assert.Contains("2", failure.Message); + } + + [TestMethod] + public void CollectionParameter_DoesNotLift() + { + var need = (NeedTool)Eval("archive(@files, @zip)", Env(("files", """["a","b","c"]"""))); + + Assert.AreEqual(1, need.Plan.Count); + Assert.IsFalse(need.Plan.IsLifted); + Assert.AreEqual("""{"files":["a","b","c"]}""", need.Plan.InvocationArgsJson[0]); + } + + [TestMethod] + public void NestedArray_LiftFails() + { + var failure = (EvalFailure)Eval("toPdf(@files, @dst)", Env(("files", """[["a"],["b"]]"""))); + + Assert.AreEqual(EvalFailureCodes.LiftNested, failure.Code); + } + + [TestMethod] + public void LiftedResults_BindStructureOfArrays() + { + var need = (NeedTool)Eval("toPdf(@files, @dst)", Env(("files", """["a.txt","b.txt"]"""))); + + var bound = (Bound)Evaluator.BindToolResults( + need.Signature, need.OutArgs, ["\"a.pdf\"", "\"b.pdf\""], Env()); + + Assert.AreEqual("""["a.pdf","b.pdf"]""", bound.Bindings.Single(b => b.Var == "dst").Json); + } + + [TestMethod] + public void LiftedResults_PatternOut_CollectsPerVariable() + { + var catalog = new SignatureCatalog([ + new PredicateSignature("probe", [ + new PredicateParam("host", ParamMode.In), + new PredicateParam("report", ParamMode.Out), + ], "probe one host"), + ]); + + var parsed = HedgeParser.Parse("""probe(@hosts, { ... "latency": @ms ... })"""); + var need = (NeedTool)Evaluator.Evaluate( + parsed.Statement!, + Env(("hosts", """["x","y"]""")), + catalog); + + var bound = (Bound)Evaluator.BindToolResults( + need.Signature, + need.OutArgs, + ["""{"latency":12,"up":true}""", """{"latency":34,"up":true}"""], + Env(("hosts", """["x","y"]"""))); + + Assert.AreEqual("[12,34]", bound.Bindings.Single(b => b.Var == "ms").Json); + } + + [TestMethod] + public void LiftedArithmetic_MapsElementwise() + { + var bound = (Bound)Eval("@doubled is @xs * 2", Env(("xs", "[1, 2, 3]"))); + + Assert.AreEqual("[2,4,6]", bound.Bindings[0].Json); + } + + [TestMethod] + public void LiftedArithmetic_ZipsTwoArrays() + { + var bound = (Bound)Eval("@sums is @xs + @ys", Env(("xs", "[1, 2]"), ("ys", "[10, 20]"))); + + Assert.AreEqual("[11,22]", bound.Bindings[0].Json); + } + + [TestMethod] + public void LiftedComparison_ElementwiseAnd() + { + Assert.IsTrue(((GuardResult)Eval("@xs > 0", Env(("xs", "[1, 2, 3]")))).Value); + Assert.IsFalse(((GuardResult)Eval("@xs > 0", Env(("xs", "[1, -2, 3]")))).Value); + } +} diff --git a/tests/Universalis.Core.Tests/LiterateRecognizerTests.cs b/tests/Universalis.Core.Tests/LiterateRecognizerTests.cs new file mode 100644 index 0000000..a5fc7ee --- /dev/null +++ b/tests/Universalis.Core.Tests/LiterateRecognizerTests.cs @@ -0,0 +1,175 @@ +using System.Collections.Immutable; +using System.Text.Json; + +using Universalis.Core.Ir; +using Universalis.Core.Parsing; + +namespace Universalis.Core.Tests; + +[TestClass] +public sealed class LiterateRecognizerTests +{ + private static HedgeItem Hedge(string content) + { + var parsed = HedgeParser.Parse(content); + Assert.IsTrue(parsed.Success, parsed.Error); + return new HedgeItem(parsed.Statement!, content); + } + + /// Streams (prose, hedge) chunks exactly as the interception protocol delivers them. + private static List Stream(LiterateRecognizer recognizer, params (string Prose, string? Hedge)[] chunks) + { + var events = new List(); + + foreach (var (prose, hedge) in chunks) + { + events.AddRange(recognizer.Advance(prose, hedge is null ? null : Hedge(hedge))); + } + + events.AddRange(recognizer.Finish()); + return events; + } + + [TestMethod] + public void TopLevel_ProseAndHedges_FlowStraightThrough() + { + var events = Stream(new LiterateRecognizer(), + ("Let's check the weather ", "WEATHER(\"Palo Alto\", @w)"), + (". The weather is ", "@w"), + (".", null)); + + Assert.HasCount(5, events); + Assert.IsInstanceOfType(events[0]); + Assert.IsInstanceOfType(events[1]); + Assert.IsInstanceOfType(events[2]); + Assert.IsInstanceOfType(events[3]); + Assert.IsInstanceOfType(events[4]); + } + + [TestMethod] + public void BtcConditional_FromPaper_RecognizedAsTwoBranchChecklist() + { + // Paper 2's BTC example, streamed in interception-protocol chunks. + var events = Stream(new LiterateRecognizer(), + ("First, we need to get today's date ", "TODAY(@today)"), + (". Next, let's find the current price of MSFT stocks ", "STOCK(\"MSFT\", @today, { ... \"close\": @msft_price ... })"), + (". The total cost of ", "@msft"), + (" MSFT stocks is ", "@msft_total is @msft_price*@msft"), + (". Then, let's find the current price of BTC ", "SEARCH(\"current price of BTC in USD\", { ... \"price\": @btc_price ... })"), + (". The total value of ", "@btc"), + (" BTC is ", "@btc_total is @btc_price*@btc"), + (". Now, let's compare the two values:\n- If ", "@btc_total >= @msft_total"), + (", then Erik can buy the MSFT stocks. The cost in BTC is ", "@btc_cost is @msft_total/@btc_price"), + (". The remaining BTC is ", "@btc_left is @btc_total - @btc_cost"), + (". Erik should buy the MSFT stocks.\n- If ", "@btc_total < @msft_total"), + (", then Erik cannot afford the MSFT stocks. He should keep the BTC, so ", "@btc_left = @btc_total"), + (".", null)); + + Assert.HasCount(7, events.OfType().ToList(), "hedges before the checklist execute immediately"); + + var conditional = events.OfType().Single().Block; + Assert.HasCount(2, conditional.Branches); + + var first = conditional.Branches[0]; + Assert.AreEqual(CompareOp.Ge, ((Comparison)first.Guard!).Op); + Assert.Contains("- If", first.GuardProse); + Assert.HasCount(2, first.Body.OfType().ToList()); + + var second = conditional.Branches[1]; + Assert.AreEqual(CompareOp.Lt, ((Comparison)second.Guard!).Op); + Assert.HasCount(1, second.Body.OfType().ToList()); + } + + [TestMethod] + public void Conditional_ClosesAtColumnZeroProse_RemainderFlowsToTop() + { + var events = Stream(new LiterateRecognizer(), + ("Decide:\n- If ", "@a >= @b"), + (", take it ", "@x is @a - @b"), + (".\nSo that settles the matter ", "@x"), + (".", null)); + + var conditional = events.OfType().Single().Block; + Assert.HasCount(1, conditional.Branches); + + // The display hedge after the block close is top-level again. + Assert.HasCount(1, events.OfType().ToList()); + Assert.IsTrue(events.OfType().Any(p => p.Text.Contains("So that settles", StringComparison.Ordinal))); + } + + [TestMethod] + public void Conditional_IndentedContinuationLines_StayInsideBranch() + { + var events = Stream(new LiterateRecognizer(), + ("Choose:\n- If ", "@a >= @b"), + (", then act.\n Even across an indented line ", "@r is @a + @b"), + (".", null)); + + var conditional = events.OfType().Single().Block; + Assert.HasCount(1, conditional.Branches[0].Body.OfType().ToList()); + } + + [TestMethod] + public void OtherwiseBullet_BecomesElseBranch() + { + var events = Stream(new LiterateRecognizer(), + ("Decide:\n- If ", "@a >= @b"), + (", keep going ", "@x is @a - @b"), + (".\n- Otherwise, stop and set ", "@x = @a"), + (".", null)); + + var conditional = events.OfType().Single().Block; + Assert.HasCount(2, conditional.Branches); + Assert.IsNotNull(conditional.Branches[0].Guard); + Assert.IsNull(conditional.Branches[1].Guard); + Assert.HasCount(1, conditional.Branches[1].Body.OfType().ToList()); + } + + [TestMethod] + public void CustomersComprehension_FromPaper_RecognizedWithBullets() + { + var events = Stream(new LiterateRecognizer(), + ("Consider each customer ", "@c = { ... \"city\": @city ... }"), + (" from ", "@customers"), + (":\n- Retain only customers ", "@c"), + (" that live in Palo Alto ", "@city = \"Palo Alto\""), + (".\n- Subsequently, increment ", "@total"), + (" by one for each retained customer ", "@c"), + (".", null)); + + var draft = events.OfType().Single().Draft; + + Assert.AreEqual("c", draft.ItemVar); + Assert.IsNotNull(draft.ItemPattern); + Assert.AreEqual("customers", draft.SourceVar); + Assert.HasCount(2, draft.Bullets); + Assert.HasCount(2, draft.Bullets[0].Hedges); + Assert.Contains("Retain only", draft.Bullets[0].Prose); + Assert.HasCount(2, draft.Bullets[1].Hedges); + Assert.Contains("increment", draft.Bullets[1].Prose); + + Assert.IsEmpty(events.OfType().ToList(), "comprehension hedges are deferred, not executed"); + } + + [TestMethod] + public void RecognizerState_SerializesMidBlock_AndResumes() + { + // Simulates a checkpoint landing in the middle of a checklist. + var first = new LiterateRecognizer(); + first.Advance("Decide:\n- If ", Hedge("@a >= @b")); + first.Advance(", take ", Hedge("@x is @a - @b")); + + var json = JsonSerializer.Serialize(first.State); + var resumed = new LiterateRecognizer(JsonSerializer.Deserialize(json)!); + + var tail = new List(); + tail.AddRange(resumed.Advance(".\n- If ", Hedge("@a < @b"))); + tail.AddRange(resumed.Advance(", instead ", Hedge("@x = @a"))); + tail.AddRange(resumed.Finish()); + + var conditional = tail.OfType().Single().Block; + Assert.HasCount(2, conditional.Branches); + Assert.HasCount(1, conditional.Branches[0].Body.OfType().ToList()); + Assert.HasCount(1, conditional.Branches[1].Body.OfType().ToList()); + } +} diff --git a/tests/Universalis.Core.Tests/PatternMatcherTests.cs b/tests/Universalis.Core.Tests/PatternMatcherTests.cs new file mode 100644 index 0000000..2adf4c2 --- /dev/null +++ b/tests/Universalis.Core.Tests/PatternMatcherTests.cs @@ -0,0 +1,139 @@ +using System.Text.Json.Nodes; + +using Universalis.Core.Evaluation; +using Universalis.Core.Ir; +using Universalis.Core.Parsing; + +namespace Universalis.Core.Tests; + +[TestClass] +public sealed class PatternMatcherTests +{ + /// The exact STOCK("IBM") blob printed in paper 2 (pattern-matching section). + private const string StockBlob = + """ + {"data":[{"symbol":"IBM","name":"International Business Machines Corp","exchange":"NYSE", + "mic_code":"XNYS","currency":"USD","datetime":"2024-04-19","timestamp":1713533400, + "open":"182.42999","high":"182.80000","low":"180.57001","close":"181.58000","volume":"3037600", + "previous_close":"181.47000","change":"0.11000","percent_change":"0.06062","average_volume":"0", + "is_market_open":false,"fifty_two_week":{"low":"180.17000","high":"183.46001", + "low_change":"1.41000","high_change":"-1.88000","low_change_percent":"0.78260", + "high_change_percent":"-1.02475","range":"180.169998 - 183.460007"}},{"ignored":true}],"status":"ok"} + """; + + private static Term Pattern(string text) + { + var result = HedgeParser.Parse("@subject = " + text); + Assert.IsTrue(result.Success, result.Error); + return ((BindStmt)result.Statement!).Right; + } + + private static readonly EvalEnv Empty = EvalEnv.FromSigma(new Dictionary()); + + [TestMethod] + public void OpenPattern_DeepSearch_BindsStockFields() + { + // Paper 2: { ... "volume": @V ... "close": @P ... "currency": @X } over the messy blob — + // all three keys live nested inside data[0], requiring depth-first descent. + var pattern = Pattern("""{ ... "volume": @V ... "close": @P ... "currency": @X }"""); + var value = JsonNode.Parse(StockBlob); + + var match = PatternMatcher.Match(pattern, value, Empty); + + Assert.IsTrue(match.Success, match.Reason); + Assert.AreEqual("\"3037600\"", match.Bindings.Single(b => b.Var == "V").Json); + Assert.AreEqual("\"181.58000\"", match.Bindings.Single(b => b.Var == "P").Json); + Assert.AreEqual("\"USD\"", match.Bindings.Single(b => b.Var == "X").Json); + } + + [TestMethod] + public void OpenPattern_SelfLevelWinsOverDescendants() + { + // "low" exists at data[0].low AND data[0].fifty_two_week.low — document order + self-first + // resolves to the shallower one encountered first in pre-order. + var pattern = Pattern("""{ ... "low": @L ... }"""); + var value = JsonNode.Parse(StockBlob); + + var match = PatternMatcher.Match(pattern, value, Empty); + + Assert.IsTrue(match.Success, match.Reason); + Assert.AreEqual("\"180.57001\"", match.Bindings.Single(b => b.Var == "L").Json); + } + + [TestMethod] + public void OpenPattern_MissingKey_HintListsAvailableKeys() + { + // The paper prints "closing" but the API key is "close" — the hint teaches the model. + var pattern = Pattern("""{ ... "closing": @P ... }"""); + var value = JsonNode.Parse(StockBlob); + + var match = PatternMatcher.Match(pattern, value, Empty); + + Assert.IsFalse(match.Success); + Assert.Contains("closing", match.Reason!); + Assert.Contains("data", match.Hint!); + } + + [TestMethod] + public void OpenPattern_WeatherGovShape_FindsNestedForecast() + { + var pattern = Pattern("""{ ... "forecast": @url ... }"""); + var value = JsonNode.Parse("""{"id":"x","properties":{"gridId":"MTR","forecast":"https://api.weather.gov/f"}}"""); + + var match = PatternMatcher.Match(pattern, value, Empty); + + Assert.IsTrue(match.Success, match.Reason); + Assert.AreEqual("\"https://api.weather.gov/f\"", match.Bindings[0].Json); + } + + [TestMethod] + public void ClosedPattern_RequiresExactKeySet() + { + var pattern = Pattern("""{ "a": @x }"""); + + Assert.IsTrue(PatternMatcher.Match(pattern, JsonNode.Parse("""{"a":1}"""), Empty).Success); + Assert.IsFalse(PatternMatcher.Match(pattern, JsonNode.Parse("""{"a":1,"b":2}"""), Empty).Success); + } + + [TestMethod] + public void NonLinearPattern_SameVarMustMatchSameValue() + { + var pattern = Pattern("""{ ... "a": @x ... "b": @x ... }"""); + + Assert.IsTrue(PatternMatcher.Match(pattern, JsonNode.Parse("""{"a":7,"b":7}"""), Empty).Success); + Assert.IsFalse(PatternMatcher.Match(pattern, JsonNode.Parse("""{"a":7,"b":8}"""), Empty).Success); + } + + [TestMethod] + public void BoundEnvVar_ActsAsEqualityTest() + { + var env = EvalEnv.FromSigma(new Dictionary { ["expected"] = "\"ok\"" }); + var pattern = Pattern("""{ ... "status": @expected ... }"""); + + Assert.IsTrue(PatternMatcher.Match(pattern, JsonNode.Parse("""{"status":"ok"}"""), env).Success); + Assert.IsFalse(PatternMatcher.Match(pattern, JsonNode.Parse("""{"status":"bad"}"""), env).Success); + } + + [TestMethod] + public void ArrayPatterns_ExactAndEllipsis() + { + Assert.IsTrue(PatternMatcher.Match(Pattern("[1, 2, 3]"), JsonNode.Parse("[1,2,3]"), Empty).Success); + Assert.IsFalse(PatternMatcher.Match(Pattern("[1, 2]"), JsonNode.Parse("[1,2,3]"), Empty).Success); + + var head = PatternMatcher.Match(Pattern("[@first, ...]"), JsonNode.Parse("[10,20,30]"), Empty); + Assert.IsTrue(head.Success); + Assert.AreEqual("10", head.Bindings[0].Json); + + var tail = PatternMatcher.Match(Pattern("[..., @last]"), JsonNode.Parse("[10,20,30]"), Empty); + Assert.IsTrue(tail.Success); + Assert.AreEqual("30", tail.Bindings[0].Json); + } + + [TestMethod] + public void LiteralFields_NumericAwareEquality() + { + Assert.IsTrue(PatternMatcher.Match(Pattern("""{ ... "n": 3 ... }"""), JsonNode.Parse("""{"n":3.0}"""), Empty).Success); + Assert.IsTrue(PatternMatcher.Match(Pattern("""{ ... "n": 3 ... }"""), JsonNode.Parse("""{"n":"3"}"""), Empty).Success); + Assert.IsFalse(PatternMatcher.Match(Pattern("""{ ... "n": 3 ... }"""), JsonNode.Parse("""{"n":"x"}"""), Empty).Success); + } +} diff --git a/tests/Universalis.Core.Tests/RoundTripTests.cs b/tests/Universalis.Core.Tests/RoundTripTests.cs new file mode 100644 index 0000000..5c8d5f9 --- /dev/null +++ b/tests/Universalis.Core.Tests/RoundTripTests.cs @@ -0,0 +1,139 @@ +using Universalis.Core.Ir; +using Universalis.Core.Parsing; +using Universalis.Core.Rendering; + +namespace Universalis.Core.Tests; + +[TestClass] +public sealed class RoundTripTests +{ + /// The apples answer from paper 2, verbatim — including its unbalanced paren. + private const string ApplesAnswer = + "The apples cost $[@B], and the selling price was $[@S], so Alice made a profit of " + + "$[@D is (@S-@B)]. The profit percentage is therefore [@P is (@D/@B)*100)]%."; + + private static void AssertSameIr(UniversalisProgram expected, UniversalisProgram actual) + { + // Records holding ImmutableArray lack structural equality; canonical JSON is the identity. + Assert.AreEqual(IrJson.Serialize(expected), IrJson.Serialize(actual)); + } + + [TestMethod] + public void Apples_Parse_Render_Reparse_IsIdentity() + { + var first = UniversalisParser.ParseProgram(ApplesAnswer); + Assert.IsTrue(first.Success, first.Error); + + var rendered = ConcreteRenderer.RenderProgram(first.Program!, RenderMode.Formulas); + Assert.AreEqual(ApplesAnswer, rendered); // ConcreteText preservation makes formulas mode lossless. + + var second = UniversalisParser.ParseProgram(rendered); + Assert.IsTrue(second.Success, second.Error); + AssertSameIr(first.Program!, second.Program!); + } + + [TestMethod] + public void Apples_IrJson_RoundTrips() + { + var parsed = UniversalisParser.ParseProgram(ApplesAnswer).Program!; + + var json = IrJson.Serialize(parsed); + var back = IrJson.DeserializeProgram(json); + + AssertSameIr(parsed, back); + } + + [TestMethod] + public void Apples_PaperShape_ExportImport_PreservesExecutableContent() + { + var parsed = UniversalisParser.ParseProgram(ApplesAnswer).Program!; + + var paperJson = PaperShape.Export(parsed); + Assert.Contains("\"expression\"", paperJson); + Assert.Contains("@D is (@S-@B)", paperJson); + + var imported = PaperShape.Import(paperJson); + Assert.IsTrue(imported.Success, imported.Error); + + // The paper shape normalizes whitespace in comments; the executable hedges must survive exactly. + var hedges = parsed.Items.OfType().Select(h => IrJson.Serialize(new UniversalisProgram([h], [], []))); + var importedHedges = imported.Program!.Items.OfType().Select(h => IrJson.Serialize(new UniversalisProgram([h], [], []))); + CollectionAssert.AreEqual(hedges.ToList(), importedHedges.ToList()); + } + + [TestMethod] + public void PaperIntentionalRepresentation_FromPaper2_Imports() + { + // The worth-of-apples-in-gold example from paper 2's "Under the Hood" section (simplified: + // the WOLFRAM free-text out-pattern is v2; we use a JSON pattern instead). + const string PaperJson = """ + [ + { "comment": "Let's first calculate the worth of the apples in dollars." }, + { "expression": "@W is @A*@X" }, + { "comment": "Convert the result to a number." }, + { "expression": "TO_DOUBLE(@G, @GD)" }, + { "comment": "Finally, convert the worth of apples from dollars to ounces of gold." }, + { "expression": "@WorthInGold is @W/@GD" } + ] + """; + + var imported = PaperShape.Import(PaperJson); + + Assert.IsTrue(imported.Success, imported.Error); + Assert.HasCount(3, imported.Program!.Items.OfType().ToList()); + } + + [TestMethod] + public void Import_NonStringValues_TeachInsteadOfThrowing() + { + // Review finding: GetValue() on {"comment": 42} threw InvalidOperationException, + // escaping as an internal error instead of the teaching result the Mode B loop is built on. + var badComment = PaperShape.Import("""[{"comment": 42}]"""); + Assert.IsNull(badComment.Program); + Assert.Contains("must be a string", badComment.Error!); + + var badExpression = PaperShape.Import("""[{"expression": 3}]"""); + Assert.IsNull(badExpression.Program); + Assert.Contains("must be a string", badExpression.Error!); + } + + [TestMethod] + public void Import_ItemWithBothKeys_YieldsCommentThenExpression() + { + // Review finding: the Mode B decoding schema permits both keys on one item (no oneOf), + // and small models pair prose with its code — the expression was silently dropped. + var imported = PaperShape.Import("""[{"comment": "The apples cost", "expression": "@buyPrice is 10"}]"""); + + Assert.IsTrue(imported.Success, imported.Error); + Assert.HasCount(2, imported.Program!.Items); + Assert.IsInstanceOfType(imported.Program.Items[0]); + Assert.IsInstanceOfType(imported.Program.Items[1]); + } + + [TestMethod] + public void ValuesMode_SubstitutesBindings_PaperExample() + { + // Paper 2: with B=10, S=17 the clause renders as [7 is (17-10)]. + var parsed = UniversalisParser.ParseProgram("[@D is (@S - @B)]").Program!; + var env = Evaluation.EvalEnv.FromSigma(new Dictionary + { + ["B"] = "10", + ["S"] = "17", + ["D"] = "7", + }); + + var rendered = ConcreteRenderer.RenderProgram(parsed, RenderMode.Values, env); + + Assert.AreEqual("[7 is (17 - 10)]", rendered); + } + + [TestMethod] + public void UnparseableBracketText_ReclassifiedAsProse() + { + var result = UniversalisParser.ParseProgram("see [the docs] for details"); + + Assert.IsTrue(result.Success, result.Error); + Assert.HasCount(1, result.Warnings); + Assert.IsInstanceOfType(result.Program!.Items[1]); + } +} diff --git a/tests/Universalis.Core.Tests/Universalis.Core.Tests.csproj b/tests/Universalis.Core.Tests/Universalis.Core.Tests.csproj new file mode 100644 index 0000000..f5c6937 --- /dev/null +++ b/tests/Universalis.Core.Tests/Universalis.Core.Tests.csproj @@ -0,0 +1,7 @@ + + + + + + + diff --git a/tools/McpSampleServer/McpSampleServer.csproj b/tools/McpSampleServer/McpSampleServer.csproj new file mode 100644 index 0000000..b9a8b80 --- /dev/null +++ b/tools/McpSampleServer/McpSampleServer.csproj @@ -0,0 +1,11 @@ + + + + Exe + + + + + + + diff --git a/tools/McpSampleServer/Program.cs b/tools/McpSampleServer/Program.cs new file mode 100644 index 0000000..a2573bb --- /dev/null +++ b/tools/McpSampleServer/Program.cs @@ -0,0 +1,32 @@ +// A deliberately tiny stdio MCP server, so the Automind MCP bridge can be tested and demoed +// end-to-end with no external installs (no node/npx): two read-only tools over stdin/stdout. + +using ModelContextProtocol.Protocol; +using ModelContextProtocol.Server; + +var options = new McpServerOptions +{ + ServerInfo = new Implementation { Name = "automind-sample", Version = "1.0.0" }, + ToolCollection = + [ + McpServerTool.Create( + (string text) => new string([.. text.Reverse()]), + new McpServerToolCreateOptions + { + Name = "reverse", + Description = "Reverses the characters of a text.", + ReadOnly = true, + }), + McpServerTool.Create( + (string text) => text.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Length, + new McpServerToolCreateOptions + { + Name = "word_count", + Description = "Counts the words in a text.", + ReadOnly = true, + }), + ], +}; + +await using var server = McpServer.Create(new StdioServerTransport("automind-sample"), options); +await server.RunAsync();