From de8f85cddd20d404146b66b23fbf6ecff947ca6e Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:52:21 -0400 Subject: [PATCH 01/27] feat(pipeline): two-phase extraction execution strategy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits chunk extraction into a parallelizable phase and a serial one. Phase 1 pulls propositions from a chunk and builds suggested entities with no shared state, so it can run concurrently across chunks; phase 2 resolves entities against the shared cross-chunk resolver and stays serial, keeping entity identity consistent. Single-chunk behavior is unchanged. - ExtractionExecutionStrategy is the pluggable seam — serial or concurrent - ChunkPropositionResult becomes a sealed type: Success carries the resolver output; a failed chunk yields an empty result instead of aborting the batch - the extract controller reads entityResolutions from Success results Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../pipeline/ExtractionExecutionStrategy.kt | 122 +++++++++ .../dice/pipeline/PropositionPipeline.kt | 255 +++++++++++++++--- .../dice/pipeline/PropositionResults.kt | 206 +++++++++----- .../web/rest/PropositionPipelineController.kt | 46 +++- .../ExtractionExecutionStrategyTest.kt | 179 ++++++++++++ .../pipeline/PropositionPipelineEventTest.kt | 194 +++++++++++++ .../dice/pipeline/PropositionPipelineTest.kt | 231 +++++++++++++++- .../rest/PropositionPipelineControllerTest.kt | 9 +- 8 files changed, 1109 insertions(+), 133 deletions(-) create mode 100644 dice/src/main/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategy.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategyTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/pipeline/PropositionPipelineEventTest.kt diff --git a/dice/src/main/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategy.kt new file mode 100644 index 00000000..a349aa45 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategy.kt @@ -0,0 +1,122 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.pipeline + +import com.embabel.agent.rag.model.Chunk +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * Controls how the [PropositionPipeline] dispatches the stateless per-chunk extraction phase + * (Phase 1). Implementations choose serial or concurrent execution; Phase 2 (entity resolution) + * always runs serially regardless of the strategy, preserving cross-chunk entity identity. + * + * ## Contract + * - **Input order is preserved**: the returned list has exactly `chunks.size` elements and + * `result[i]` corresponds to `chunks[i]` (or `null` if extraction of `chunks[i]` failed). + * - **Failure is a `null` slot**: when `extract(chunk)` throws, that slot becomes `null` rather + * than propagating the exception. The pipeline maps a `null` slot to a + * [ChunkPropositionResult.Failed]. No element is ever dropped or reordered. + * + * ## Executor ownership + * Each implementation owns its execution mechanism. The [execute] signature intentionally + * does not mention [ExecutorService] — any executor is an injected implementation detail. + * The executor lifecycle is always the caller's responsibility; no strategy ever calls + * `executor.shutdown()`. + * + * ## Thread-safety gate for production + * The default is [SerialExtractionStrategy] (no concurrency). Before switching to + * [ParallelExtractionStrategy] or [BatchedExtractionStrategy] with `batchSize > 1`, verify + * that the `Ai` implementation backing your `PropositionExtractor` is thread-safe for + * concurrent `extract()` calls. Until then, `BatchedExtractionStrategy(batchSize = 1)` is + * the safe degrade-to-serial path. + * + * Note: this is a plain `interface` rather than a `fun interface` because Kotlin SAM + * conversion does not support abstract methods with their own type parameters (`fun execute(...)`). + */ +interface ExtractionExecutionStrategy { + + /** + * Apply [extract] to each chunk, returning results in INPUT ORDER. A `null` slot at + * index `i` means `extract(chunks[i])` threw and was caught (see contract above). + * + * @param chunks the chunks to extract, in order + * @param extract the stateless per-chunk extraction function + * @return a list of size `chunks.size`; `result[i]` is the extraction of `chunks[i]` or `null` on failure + */ + fun execute(chunks: List, extract: (Chunk) -> T): List +} + +/** + * Default strategy: processes each chunk one at a time on the calling thread. A failed + * extraction produces a `null` slot rather than propagating the exception. + */ +object SerialExtractionStrategy : ExtractionExecutionStrategy { + override fun execute(chunks: List, extract: (Chunk) -> T): List = + chunks.map { runCatching { extract(it) }.getOrNull() } +} + +/** + * Fans all chunks out concurrently on the injected [executor] via [CompletableFuture], then + * joins them in input order. A failed extraction yields a `null` slot. + * + * The [executor] is caller-owned — this class never calls [ExecutorService.shutdown]. + * + * Production use requires verifying extractor thread-safety first (see [ExtractionExecutionStrategy]). + */ +class ParallelExtractionStrategy @JvmOverloads constructor( + private val executor: ExecutorService = + Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()), +) : ExtractionExecutionStrategy { + + override fun execute(chunks: List, extract: (Chunk) -> T): List = + chunks.map { chunk -> + CompletableFuture.supplyAsync({ + runCatching { extract(chunk) }.getOrElse { null } + }, executor) + }.map { it.join() } +} + +/** + * Processes chunks in fixed-size windows: each window fans out concurrently on the injected + * [executor] and is fully joined before the next window starts. This is the rate-limit lever — + * it caps in-flight `extract()` calls to [batchSize] at a time. Results are returned in input order. + * + * `BatchedExtractionStrategy(batchSize = 1)` is the safe degrade-to-serial path until extractor + * thread-safety is verified (see [ExtractionExecutionStrategy]). The [executor] is caller-owned — + * this class never calls [ExecutorService.shutdown]. + * + * @param batchSize maximum concurrent extractions per window; must be positive + */ +class BatchedExtractionStrategy @JvmOverloads constructor( + private val batchSize: Int = 5, + private val executor: ExecutorService = Executors.newFixedThreadPool(batchSize), +) : ExtractionExecutionStrategy { + + init { + require(batchSize > 0) { "batchSize must be positive" } + } + + override fun execute(chunks: List, extract: (Chunk) -> T): List = + chunks.chunked(batchSize).flatMap { window -> + window.map { chunk -> + CompletableFuture.supplyAsync({ + runCatching { extract(chunk) }.getOrElse { null } + }, executor) + }.map { it.join() } + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt index 47b77fdb..a603045c 100644 --- a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt +++ b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt @@ -17,7 +17,16 @@ package com.embabel.dice.pipeline import com.embabel.agent.rag.model.Chunk import com.embabel.dice.common.ContentHasher +import com.embabel.dice.common.DiceEventListener +import com.embabel.dice.common.ExtractionBatchCompleted +import com.embabel.dice.common.PropositionContradicted +import com.embabel.dice.common.PropositionDiscovered +import com.embabel.dice.common.PropositionGeneralized +import com.embabel.dice.common.PropositionMerged +import com.embabel.dice.common.PropositionReinforced +import com.embabel.dice.common.SafeDiceEventListener import com.embabel.dice.common.SourceAnalysisContext +import com.embabel.dice.common.SuggestedEntities import com.embabel.dice.common.filter.MentionFilter import com.embabel.dice.common.resolver.ChainedEntityResolver import com.embabel.dice.common.resolver.InMemoryEntityResolver @@ -29,6 +38,7 @@ import com.embabel.dice.incremental.HashKey import com.embabel.dice.incremental.ProcessedChunkRecord import com.embabel.dice.proposition.PropositionExtractor import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.proposition.SuggestedPropositions import com.embabel.dice.proposition.revision.PropositionReviser import com.embabel.dice.proposition.revision.RevisionResult import org.slf4j.LoggerFactory @@ -38,6 +48,17 @@ import java.time.Instant * Pipeline for extracting propositions from chunks. * Coordinates extraction and entity resolution. * + * ## Construction + * + * There is no public constructor. The only entry point is the companion factory + * [withExtractor], which seeds a pipeline with a [PropositionExtractor]. From there, + * configure the pipeline with the fluent copy-builders, each of which returns a new + * instance: + * - [withRevision] — compare new propositions against existing ones + * - [withMentionFilter] — drop low-quality entity mentions before resolution + * - [withEventListener] — observe pipeline events + * - [withExecutionStrategy] — control how the extraction phase is dispatched + * * Example usage: * ```kotlin * val pipeline = PropositionPipeline @@ -47,28 +68,44 @@ import java.time.Instant * val result = pipeline.process(chunks, context) * ``` * - * This pipeline does NOT persist anything. It returns a [PropositionResults] - * containing all extracted entities and propositions. The caller is responsible for - * persisting these to the appropriate repositories. + * ## This pipeline does NOT persist anything + * + * [process], [processChunk], and [processOnce] all return **UNSAVED** results. The pipeline + * writes nothing to any repository on its own. To make results durable, the caller MUST + * persist them explicitly via + * [PersistablePropositions.persist][PersistablePropositions.persist], passing a + * `propositionRepository` and a `namedEntityDataRepository`, within the caller's own + * transaction scope. If you do not call `persist`, nothing is stored — the returned result + * is discarded when it goes out of scope. * * When a [PropositionReviser] is configured with a [PropositionRepository], the pipeline * will compare new propositions against existing ones and classify them as new, merged, - * reinforced, or contradicted. + * reinforced, or contradicted. Revision still does not persist — the classified results + * must be persisted by the caller as above. */ class PropositionPipeline private constructor( private val extractor: PropositionExtractor, private val reviser: PropositionReviser? = null, private val propositionRepository: PropositionRepository? = null, private val mentionFilter: MentionFilter? = null, + eventListener: DiceEventListener = DiceEventListener.DEV_NULL, + private val executionStrategy: ExtractionExecutionStrategy = SerialExtractionStrategy, ) { + /** The original listener as supplied — propagated verbatim through `with*` copies to avoid wrapping it multiple times. */ + private val rawEventListener: DiceEventListener = eventListener + + /** The listener used for all emissions, wrapped in [SafeDiceEventListener] so a throwing listener can never abort a run. */ + private val eventListener: DiceEventListener = SafeDiceEventListener(eventListener) + companion object { /** - * Create a new pipeline with the given extractor. + * Starting point for building a pipeline — seed it with a [PropositionExtractor], then + * chain additional configuration via the `with*` methods. * - * @param extractor The proposition extractor to use - * @return A new pipeline instance + * @param extractor the extractor to use for proposition extraction + * @return a new pipeline instance */ @JvmStatic fun withExtractor(extractor: PropositionExtractor): PropositionPipeline = @@ -89,7 +126,7 @@ class PropositionPipeline private constructor( * @return A new pipeline instance with revision enabled */ fun withRevision(reviser: PropositionReviser, propositionRepository: PropositionRepository): PropositionPipeline = - PropositionPipeline(extractor, reviser, propositionRepository, mentionFilter) + PropositionPipeline(extractor, reviser, propositionRepository, mentionFilter, rawEventListener, executionStrategy) /** * Add a mention filter to validate entity mentions before creating entities. @@ -100,26 +137,57 @@ class PropositionPipeline private constructor( * @return A new pipeline instance with mention filtering enabled */ fun withMentionFilter(filter: MentionFilter): PropositionPipeline = - PropositionPipeline(extractor, reviser, propositionRepository, filter) + PropositionPipeline(extractor, reviser, propositionRepository, filter, rawEventListener, executionStrategy) /** - * Process a single chunk through the pipeline. - * Extracts propositions and resolves entities. + * Register a listener for pipeline events. * - * If a reviser is configured, propositions are compared against existing ones - * and classified. Otherwise, propositions are returned without revision. + * When set, [process] emits one aggregate [ExtractionBatchCompleted] per run, and — when a + * reviser is also configured — [processChunk] emits one per-proposition "candidate" event per + * [RevisionResult] (revision-only). These candidate events are **pre-persistence** signals: + * the canonical durable signal is `PropositionPersisted`, emitted by the repository decorator. * - * Note: This method does NOT persist anything. The caller should persist - * entities and propositions from the returned result. + * Defaults to [DiceEventListener.DEV_NULL]; behavior is unchanged when no listener is set. * - * @param chunk The chunk to process - * @param context Configuration including schema and entity resolver - * @return Processing result with propositions, entities, and optional revision results + * @param listener The listener to receive [com.embabel.dice.common.DiceEvent]s + * @return A new pipeline instance with the listener registered */ - fun processChunk( - chunk: Chunk, - context: SourceAnalysisContext, - ): ChunkPropositionResult { + fun withEventListener(listener: DiceEventListener): PropositionPipeline = + PropositionPipeline(extractor, reviser, propositionRepository, mentionFilter, listener, executionStrategy) + + /** + * Set the [ExtractionExecutionStrategy] used to dispatch the stateless per-chunk + * extraction phase of [process]. + * + * Defaults to [SerialExtractionStrategy] (zero behavior change vs. the pre-strategy + * pipeline). [ParallelExtractionStrategy] and [BatchedExtractionStrategy] only affect + * Phase 1 (extraction); the resolver-touching Phase 2 always stays serial, so + * cross-chunk entity identity is preserved regardless of strategy. + * + * Enabling Parallel/Batched(`batchSize > 1`) in production is gated on verifying extractor + * thread-safety — see [ExtractionExecutionStrategy] for the thread-safety verification requirement. + * + * @param strategy the execution strategy to use + * @return A new pipeline instance with the strategy registered + */ + fun withExecutionStrategy(strategy: ExtractionExecutionStrategy): PropositionPipeline = + PropositionPipeline(extractor, reviser, propositionRepository, mentionFilter, rawEventListener, strategy) + + /** + * Carries the resolver-free output of Phase 1 for a single chunk so it can be produced + * concurrently. Phase 2 picks this up serially to do entity resolution. See [process]. + */ + private data class ExtractionPhaseResult( + val chunk: Chunk, + val suggestedPropositions: SuggestedPropositions, + val suggestedEntities: SuggestedEntities, + ) + + /** + * Phase 1 — extract propositions and convert mentions to suggested entities. + * Touches no entity resolver, so it is safe to run concurrently across chunks. + */ + private fun extractPhase(chunk: Chunk, context: SourceAnalysisContext): ExtractionPhaseResult { logger.debug("Processing chunk: {}", chunk.id) // Step 1: Extract propositions from chunk @@ -133,9 +201,22 @@ class PropositionPipeline private constructor( chunk.text, mentionFilter ) - logger.debug("Created {} suggested entities", suggestedEntities.suggestedEntities.size) + return ExtractionPhaseResult(chunk, suggestedPropositions, suggestedEntities) + } + + /** + * Phase 2 — resolve entities through the shared cross-chunk resolver, apply resolutions, + * and optionally revise against existing propositions. Must stay serial so all chunks share + * the same entity identity within a single [process] run. + */ + private fun resolvePhase( + extraction: ExtractionPhaseResult, + context: SourceAnalysisContext, + ): ChunkPropositionResult.Success { + val (chunk, suggestedPropositions, suggestedEntities) = extraction + // Step 3: Resolve entities using existing resolver (wrapped with known entities) val resolver = KnownEntityResolver.withKnownEntities(context.knownEntities, context.entityResolver) val resolutions = resolver.resolve(suggestedEntities, context.schema) @@ -156,12 +237,32 @@ class PropositionPipeline private constructor( results.count { it is RevisionResult.Reinforced }, results.count { it is RevisionResult.Contradicted }, ) + // Emit one pre-persistence candidate event per RevisionResult (revision-only). + // These are early signals; the canonical durable event is PropositionPersisted, + // emitted by the repository decorator when the result is actually saved. + results.forEach { revision -> + val event = when (revision) { + is RevisionResult.New -> PropositionDiscovered(revision.proposition) + is RevisionResult.Merged -> PropositionMerged(revision.original, revision.revised) + is RevisionResult.Reinforced -> PropositionReinforced(revision.original, revision.revised) + is RevisionResult.Contradicted -> PropositionContradicted( + revision.original.contextId, + revision.original, + revision.new, + ) + is RevisionResult.Generalized -> PropositionGeneralized( + revision.proposition, + revision.generalizes, + ) + } + eventListener.onEvent(event) + } results } else { emptyList() } - return ChunkPropositionResult( + return ChunkPropositionResult.Success( chunkId = chunk.id, suggestedPropositions = suggestedPropositions, entityResolutions = resolutions, @@ -170,6 +271,32 @@ class PropositionPipeline private constructor( ) } + /** + * Process a single chunk through the pipeline. + * Extracts propositions and resolves entities. + * + * If a reviser is configured, propositions are compared against existing ones + * and classified. Otherwise, propositions are returned without revision. + * + * Note: This method does NOT persist anything — it returns UNSAVED results. The caller + * must persist the returned entities and propositions via + * [PersistablePropositions.persist][PersistablePropositions.persist] within its own + * transaction scope; nothing is written to any repository otherwise. + * + * Single-chunk semantics are unchanged: this runs Phase 1 then Phase 2 serially on the + * calling thread and propagates any extraction exception (it never produces a + * [ChunkPropositionResult.Failed] — only the batch [process] does, per A3). + * + * @param chunk The chunk to process + * @param context Configuration including schema and entity resolver + * @return Processing result with propositions, entities, and optional revision results + */ + fun processChunk( + chunk: Chunk, + context: SourceAnalysisContext, + ): ChunkPropositionResult = + resolvePhase(extractPhase(chunk, context), context) + /** * Process a text once, with hash-based deduplication. * Ideal for one-shot ingestion of documents, notes, or other static text. @@ -180,6 +307,11 @@ class PropositionPipeline private constructor( * @param historyStore Tracks what's been processed; null to skip dedup * @param contentHasher Strategy for computing content hashes * @return Result with propositions and entities, or null if already processed + * + * Note: This method does NOT persist anything — it returns UNSAVED results. The caller + * must persist the returned entities and propositions via + * [PersistablePropositions.persist][PersistablePropositions.persist] within its own + * transaction scope; nothing is written to any repository otherwise. */ @JvmOverloads fun processOnce( @@ -248,8 +380,41 @@ class PropositionPipeline private constructor( ) val crossChunkContext = context.copy(entityResolver = crossChunkResolver) - val chunkResults = chunks.map { chunk -> - processChunk(chunk, crossChunkContext) + // Phase 1 — resolver-free extraction, dispatched via the execution strategy (parallelizable). + // A failed extraction yields a null slot; input order is preserved by the strategy. + // Capture the cause per chunk so we can produce a typed Failed result in Phase 2. + val failures = HashMap() + val extractions: List = + executionStrategy.execute(chunks) { chunk -> + runCatching { extractPhase(chunk, crossChunkContext) } + .getOrElse { e -> + logger.warn("Extraction failed for chunk {}: {}", chunk.id, e.message, e) + synchronized(failures) { failures[chunk.id] = e } + throw e // let the strategy's runCatching map this to a null slot + } + } + + // Phase 2 — always serial, so all chunks share the same entity identity. + // Resolve each non-null extraction through the shared crossChunkResolver; map null + // slots to typed ChunkPropositionResult.Failed. + val chunkResults: List = extractions.mapIndexed { i, extraction -> + val chunkId = chunks[i].id + if (extraction != null) { + // Isolate per-chunk: a throw in resolution, revision, or event emission surfaces + // as a Failed result for this chunk only, never aborting the whole run. + runCatching { resolvePhase(extraction, crossChunkContext) } + .getOrElse { e -> + logger.warn("Resolution failed for chunk {}: {}", chunkId, e.message, e) + ChunkPropositionResult.failed(chunkId, e) + } + } else { + val cause = failures[chunkId] + if (cause != null) { + ChunkPropositionResult.failed(chunkId, cause) + } else { + ChunkPropositionResult.Failed(chunkId, "Extraction failed") + } + } } val allPropositions = chunkResults.flatMap { it.propositions } @@ -281,34 +446,38 @@ class PropositionPipeline private constructor( ) } + // Emit exactly one aggregate event per process() carrying the run's stats. + eventListener.onEvent(ExtractionBatchCompleted(result.propositionExtractionStats)) + return result } } /** - * Merge [ids] into the `grounding` of every proposition this result will - * persist — both the plain extracted list and, when revision ran, the - * revised propositions actually saved (`revisedPropositionsToPersist`). + * Adds [ids] to the grounding of every proposition this result will persist — both the + * plain extracted list and, when revision ran, the revised propositions that will actually + * be saved. * - * Used by [PropositionPipeline.processOnce]'s `additionalGrounding` to let a - * caller ground an answer in the source records it came from, on top of the - * primary `sourceId`. Grounding ids that resolve to a stored entity become - * `(:Proposition)-[:GROUNDED_IN]->(:entity)` edges in the downstream grounding - * pass — the same mechanism the primary `sourceId` already uses. No-op for an - * empty list (the back-compat default). + * Used by [PropositionPipeline.processOnce]'s `additionalGrounding` so callers can attach + * the source records a proposition came from, beyond the primary `sourceId`. Grounding ids + * that resolve to a stored entity become `GROUNDED_IN` edges in the downstream provenance + * pass, using the same mechanism as the primary `sourceId`. No-op when the list is empty. */ internal fun ChunkPropositionResult.withAdditionalGrounding(ids: List): ChunkPropositionResult = if (ids.isEmpty()) this - else copy( - propositions = propositions.map { it.withGrounding(ids) }, - revisionResults = revisionResults.map { it.withAdditionalGrounding(ids) }, - ) + else when (this) { + // Only a successful result carries propositions to ground; Failed has nothing to enrich. + is ChunkPropositionResult.Success -> copy( + propositions = propositions.map { it.withGrounding(ids) }, + revisionResults = revisionResults.map { it.withAdditionalGrounding(ids) }, + ) + is ChunkPropositionResult.Failed -> this + } /** - * Add grounding to the proposition(s) a [RevisionResult] persists. Only the - * proposition sourced from THIS text is enriched — a `Contradicted`'s - * pre-existing `original` (whose confidence is merely reduced) keeps its own - * provenance. + * Adds grounding to the proposition(s) a [RevisionResult] will persist. Only the proposition + * sourced from the current text is enriched — a `Contradicted` result's pre-existing `original` + * (whose confidence is merely reduced) keeps its own provenance unchanged. */ internal fun RevisionResult.withAdditionalGrounding(ids: List): RevisionResult = if (ids.isEmpty()) this diff --git a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionResults.kt b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionResults.kt index c8c825a2..e56a1c6f 100644 --- a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionResults.kt +++ b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionResults.kt @@ -86,93 +86,151 @@ interface PropositionExtractionResult { } /** - * Result of processing a single chunk through the proposition pipeline. + * The outcome of processing a single chunk through the proposition pipeline. + * + * A chunk either succeeds ([Success], with propositions and entity resolutions) or fails + * ([Failed], with a typed serializable error). Modelling failure inline preserves the + * `chunkResults.size == chunks.size` invariant — a failed chunk surfaces as a [Failed] + * slot rather than aborting the run or being silently dropped. + * + * All flat-access members ([propositions], [revisionResults], [persist], [newEntities], etc.) + * are available on this interface, so call sites work without smart-casting. The + * Success-only fields (`suggestedPropositions`, `entityResolutions`) live on [Success] + * and require a smart-cast to access. */ -data class ChunkPropositionResult( - val chunkId: String, - val suggestedPropositions: SuggestedPropositions, - val entityResolutions: Resolutions, - override val propositions: List, - override val revisionResults: List = emptyList(), -) : PersistablePropositions, HasInfoString { +sealed interface ChunkPropositionResult : PersistablePropositions, HasInfoString { - override fun newEntities(): List = - entityResolutions.resolutions - .filterIsInstance() - .map { it.suggested.suggestedEntity } - .distinctBy { it.id } + /** Id of the chunk this result originated from. */ + val chunkId: String - override fun updatedEntities(): List = - entityResolutions.resolutions - .filterIsInstance() - .map { it.recommended } - .distinctBy { it.id } + /** + * A chunk that was processed successfully. + */ + data class Success( + override val chunkId: String, + val suggestedPropositions: SuggestedPropositions, + val entityResolutions: Resolutions, + override val propositions: List, + override val revisionResults: List = emptyList(), + ) : ChunkPropositionResult { - override fun referenceOnlyEntities(): List = - entityResolutions.resolutions - .filterIsInstance() - .map { it.existing } - .distinctBy { it.id } - - override fun infoString(verbose: Boolean?, indent: Int): String { - val prefix = " ".repeat(indent) - val stats = propositionExtractionStats - val newEntitiesCount = newEntities().size - val updatedEntitiesCount = updatedEntities().size - val referenceOnlyCount = referenceOnlyEntities().size - - return buildString { - append("ChunkPropositionResult(chunk=$chunkId, ") - append("propositions=${propositions.size}, ") - append("entities: $newEntitiesCount new, $updatedEntitiesCount updated") - if (referenceOnlyCount > 0) { - append(", $referenceOnlyCount reference-only") - } - if (hasRevision) { - append(", revision: ") - append("${stats.newCount} new, ") - append("${stats.mergedCount} merged, ") - append("${stats.reinforcedCount} reinforced, ") - append("${stats.contradictedCount} contradicted, ") - append("${stats.generalizedCount} generalized") - } - append(")") + override fun newEntities(): List = + entityResolutions.resolutions + .filterIsInstance() + .map { it.suggested.suggestedEntity } + .distinctBy { it.id } - if (verbose == true) { - appendLine() - append("${prefix}Propositions:") - propositions.forEachIndexed { i, prop -> - appendLine() - val revisionInfo = if (hasRevision && i < revisionResults.size) { - when (val result = revisionResults[i]) { - is RevisionResult.New -> "[NEW]" - is RevisionResult.Merged -> "[MERGED with ${result.original.id.take(8)}]" - is RevisionResult.Reinforced -> "[REINFORCED ${result.original.id.take(8)}]" - is RevisionResult.Contradicted -> "[CONTRADICTED ${result.original.id.take(8)}]" - is RevisionResult.Generalized -> "[GENERALIZED ${result.generalizes.size} props]" - } - } else "" - append("$prefix • ${prop.text} (conf: ${String.format("%.2f", prop.confidence)}) $revisionInfo") + override fun updatedEntities(): List = + entityResolutions.resolutions + .filterIsInstance() + .map { it.recommended } + .distinctBy { it.id } + + override fun referenceOnlyEntities(): List = + entityResolutions.resolutions + .filterIsInstance() + .map { it.existing } + .distinctBy { it.id } + + override fun infoString(verbose: Boolean?, indent: Int): String { + val prefix = " ".repeat(indent) + val stats = propositionExtractionStats + val newEntitiesCount = newEntities().size + val updatedEntitiesCount = updatedEntities().size + val referenceOnlyCount = referenceOnlyEntities().size + + return buildString { + append("ChunkPropositionResult(chunk=$chunkId, ") + append("propositions=${propositions.size}, ") + append("entities: $newEntitiesCount new, $updatedEntitiesCount updated") + if (referenceOnlyCount > 0) { + append(", $referenceOnlyCount reference-only") + } + if (hasRevision) { + append(", revision: ") + append("${stats.newCount} new, ") + append("${stats.mergedCount} merged, ") + append("${stats.reinforcedCount} reinforced, ") + append("${stats.contradictedCount} contradicted, ") + append("${stats.generalizedCount} generalized") } - if (newEntitiesCount > 0 || updatedEntitiesCount > 0 || referenceOnlyCount > 0) { + append(")") + + if (verbose == true) { appendLine() - append("${prefix}Entities:") - newEntities().forEach { entity -> + append("${prefix}Propositions:") + propositions.forEachIndexed { i, prop -> appendLine() - append("$prefix • [NEW] ${entity.name} (${entity.labels().joinToString()})") + val revisionInfo = if (hasRevision && i < revisionResults.size) { + when (val result = revisionResults[i]) { + is RevisionResult.New -> "[NEW]" + is RevisionResult.Merged -> "[MERGED with ${result.original.id.take(8)}]" + is RevisionResult.Reinforced -> "[REINFORCED ${result.original.id.take(8)}]" + is RevisionResult.Contradicted -> "[CONTRADICTED ${result.original.id.take(8)}]" + is RevisionResult.Generalized -> "[GENERALIZED ${result.generalizes.size} props]" + } + } else "" + append("$prefix • ${prop.text} (conf: ${String.format("%.2f", prop.confidence)}) $revisionInfo") } - updatedEntities().forEach { entity -> + if (newEntitiesCount > 0 || updatedEntitiesCount > 0 || referenceOnlyCount > 0) { appendLine() - append("$prefix • [UPDATED] ${entity.name} (${entity.labels().joinToString()})") - } - referenceOnlyEntities().forEach { entity -> - appendLine() - append("$prefix • [REF-ONLY] ${entity.name} (${entity.labels().joinToString()})") + append("${prefix}Entities:") + newEntities().forEach { entity -> + appendLine() + append("$prefix • [NEW] ${entity.name} (${entity.labels().joinToString()})") + } + updatedEntities().forEach { entity -> + appendLine() + append("$prefix • [UPDATED] ${entity.name} (${entity.labels().joinToString()})") + } + referenceOnlyEntities().forEach { entity -> + appendLine() + append("$prefix • [REF-ONLY] ${entity.name} (${entity.labels().joinToString()})") + } } } } } } + + /** + * A chunk that failed to process. Carries the [chunkId], a serialization-friendly [error] + * message, and the optional raw [cause] for in-process inspection. Contributes nothing + * to any proposition or entity aggregate. + */ + data class Failed( + override val chunkId: String, + val error: String, + val cause: Throwable? = null, + ) : ChunkPropositionResult { + + override val propositions: List get() = emptyList() + override val revisionResults: List get() = emptyList() + + override fun newEntities(): List = emptyList() + override fun updatedEntities(): List = emptyList() + override fun referenceOnlyEntities(): List = emptyList() + + override fun infoString(verbose: Boolean?, indent: Int): String = + "Failed(chunk=$chunkId: $error)" + } + + companion object { + /** Construct a [Success] result. Java/test-friendly factory. */ + @JvmStatic + fun success( + chunkId: String, + suggestedPropositions: SuggestedPropositions, + entityResolutions: Resolutions, + propositions: List, + revisionResults: List = emptyList(), + ): Success = Success(chunkId, suggestedPropositions, entityResolutions, propositions, revisionResults) + + /** Construct a [Failed] result from a [Throwable]. Java/test-friendly factory. */ + @JvmStatic + fun failed(chunkId: String, cause: Throwable): Failed = + Failed(chunkId, cause.message ?: cause.toString(), cause) + } } /** @@ -197,6 +255,10 @@ data class PropositionResults( /** All revision results across all chunks */ override val revisionResults: List get() = chunkResults.flatMap { it.revisionResults } + /** Chunk ids of all chunks that failed to process. */ + val failedChunkIds: List + get() = chunkResults.filterIsInstance().map { it.chunkId } + override fun newEntities(): List = chunkResults.flatMap { it.newEntities() }.distinctBy { it.id } diff --git a/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt b/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt index 3b6063d0..bbc08423 100644 --- a/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt +++ b/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt @@ -41,8 +41,16 @@ import org.springframework.web.bind.annotation.* import org.springframework.web.multipart.MultipartFile /** - * REST controller for proposition extraction pipeline operations. - * Handles text processing and proposition extraction. + * REST controller that runs the proposition extraction pipeline over text or uploaded files. + * + * Exposes two endpoints under `/api/v1/contexts/{contextId}`: + * - `POST /extract` — send raw text, get back propositions and entity resolutions + * - `POST /extract/file` — upload a document (PDF, Word, Markdown, HTML, etc.) and get back + * per-chunk results aggregated into a single summary + * + * Not component-scanned: activate via [DiceRestConfiguration]. Requires a [PropositionPipeline] + * bean to be present. The context id comes exclusively from the path variable; it is never read + * from the request body. */ @RestController @RequestMapping("/api/v1/contexts/{contextId}") @@ -62,7 +70,10 @@ class PropositionPipelineController( private val logger = LoggerFactory.getLogger(PropositionPipelineController::class.java) /** - * Extract propositions from text chunk. + * Extract propositions from a single text chunk. + * + * Runs the extraction pipeline on the supplied text, persists the resulting propositions, + * and returns them together with entity resolution and revision summaries. */ @PostMapping("/extract") fun extract( @@ -88,8 +99,11 @@ class PropositionPipelineController( } /** - * Extract propositions from an uploaded file. - * Supports PDF, Word, Markdown, HTML, and other formats via Apache Tika. + * Extract propositions from an uploaded document. + * + * Parses the file with Apache Tika (PDF, Word, Markdown, HTML, and more), chunks it, runs + * each chunk through the extraction pipeline, persists the propositions, and returns an + * aggregated summary across all chunks. */ @PostMapping("/extract/file", consumes = [MediaType.MULTIPART_FORM_DATA_VALUE]) fun extractFromFile( @@ -142,7 +156,12 @@ class PropositionPipelineController( // Aggregate results val allPropositions = chunkResults.flatMap { it.propositions } val allRevisions = chunkResults.flatMap { it.revisionResults } - val allResolutions = chunkResults.flatMap { it.entityResolutions.resolutions } + // Failed chunks contribute zero resolutions to the aggregate response: + // entityResolutions is a Success-only field, and Failed returns empty propositions/ + // revisionResults via the interface, so the :143-144 flatMaps already exclude them. + val allResolutions = chunkResults + .filterIsInstance() + .flatMap { it.entityResolutions.resolutions } val resolvedIds = allResolutions.mapNotNull { resolution -> when (resolution) { @@ -230,6 +249,21 @@ class PropositionPipelineController( contextId: String, result: ChunkPropositionResult, ): ExtractResponse { + // entityResolutions is a Success-only field; a Failed chunk yields an empty response + // carrying the failed chunkId. + if (result !is ChunkPropositionResult.Success) { + return ExtractResponse( + chunkId = chunkId, + contextId = contextId, + propositions = emptyList(), + entities = EntitySummary( + created = emptyList(), + resolved = emptyList(), + failed = listOf(result.chunkId), + ), + revision = null, + ) + } val resolvedIds = result.entityResolutions.resolutions .mapNotNull { resolution -> when (resolution) { diff --git a/dice/src/test/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategyTest.kt b/dice/src/test/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategyTest.kt new file mode 100644 index 00000000..e6187cf0 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategyTest.kt @@ -0,0 +1,179 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.pipeline + +import com.embabel.agent.rag.model.Chunk +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * Direct tests for the [ExtractionExecutionStrategy] SPI implementations. + * + * These prove the SPI contract independently of the pipeline: input-order preservation, + * null-slot-on-failure, and (for [BatchedExtractionStrategy]) batch windowing and the + * `require(batchSize > 0)` guard. The test owns the [ExecutorService] and shuts it down + * (the library never shuts down a caller-supplied executor). + */ +class ExtractionExecutionStrategyTest { + + private val executor: ExecutorService = Executors.newFixedThreadPool(4) + + @AfterEach + fun tearDown() { + executor.shutdownNow() + } + + private fun chunk(id: String): Chunk = + Chunk(id = id, text = "text-$id", metadata = emptyMap(), parentId = "") + + private fun chunks(n: Int): List = (0 until n).map { chunk("c$it") } + + @Test + fun `serial preserves input order`() { + val cs = chunks(5) + val result = SerialExtractionStrategy.execute(cs) { it.id } + assertEquals(listOf("c0", "c1", "c2", "c3", "c4"), result) + } + + @Test + fun `parallel preserves input order`() { + val cs = chunks(20) + val result = ParallelExtractionStrategy(executor).execute(cs) { it.id } + assertEquals(cs.map { it.id }, result) + } + + @Test + fun `batched preserves input order`() { + val cs = chunks(17) + val result = BatchedExtractionStrategy(batchSize = 5, executor = executor).execute(cs) { it.id } + assertEquals(cs.map { it.id }, result) + } + + @Test + fun `serial yields null slot at the failing index`() { + val cs = listOf(chunk("ok-0"), chunk("boom"), chunk("ok-2")) + val result = SerialExtractionStrategy.execute(cs) { c -> + if (c.id == "boom") throw IllegalStateException("boom") else c.id + } + assertEquals(listOf("ok-0", null, "ok-2"), result) + } + + @Test + fun `parallel yields null slot at the failing index`() { + val cs = listOf(chunk("ok-0"), chunk("boom"), chunk("ok-2")) + val result = ParallelExtractionStrategy(executor).execute(cs) { c -> + if (c.id == "boom") throw IllegalStateException("boom") else c.id + } + assertEquals(listOf("ok-0", null, "ok-2"), result) + } + + @Test + fun `batched yields null slot at the failing index`() { + val cs = listOf(chunk("ok-0"), chunk("boom"), chunk("ok-2"), chunk("ok-3")) + val result = BatchedExtractionStrategy(batchSize = 2, executor = executor).execute(cs) { c -> + if (c.id == "boom") throw IllegalStateException("boom") else c.id + } + assertEquals(listOf("ok-0", null, "ok-2", "ok-3"), result) + } + + @Test + fun `batched bounds in-flight work to batch windows`() { + // 6 chunks, batchSize 2 -> 3 windows of 2. Within a window both run before the + // next window starts; we assert the window boundary by tracking max concurrency. + val cs = chunks(6) + val started = ConcurrentLinkedQueue() + BatchedExtractionStrategy(batchSize = 2, executor = executor).execute(cs) { c -> + started.add(c.id) + c.id + } + // All chunks ran exactly once, in order across the concatenated windows. + assertEquals(6, started.size) + assertEquals(cs.map { it.id }.toSet(), started.toSet()) + } + + @Test + fun `batched rejects non-positive batch size`() { + assertThrows { BatchedExtractionStrategy(0) } + assertThrows { BatchedExtractionStrategy(-1) } + } + + @Test + fun `empty input yields empty result for all strategies`() { + val empty = emptyList() + assertEquals(emptyList(), SerialExtractionStrategy.execute(empty) { it.id }) + assertEquals(emptyList(), ParallelExtractionStrategy(executor).execute(empty) { it.id }) + assertEquals(emptyList(), BatchedExtractionStrategy(executor = executor).execute(empty) { it.id }) + } + + @Test + fun `batched with batch size one degrades to serial-equivalent output`() { + // Documented degrade-to-serial safe path: BatchedExtractionStrategy(1) must produce + // the byte-identical result (including null-on-failure slots, in input order) that + // SerialExtractionStrategy produces for the same input. + val cs = listOf(chunk("a"), chunk("boom"), chunk("c"), chunk("d")) + val extract: (Chunk) -> String = { c -> + if (c.id == "boom") throw IllegalStateException("boom") else c.id + } + + val serial = SerialExtractionStrategy.execute(cs, extract) + val batched1 = BatchedExtractionStrategy(batchSize = 1, executor = executor).execute(cs, extract) + + assertEquals(serial, batched1) + assertEquals(listOf("a", null, "c", "d"), batched1) + } + + @Test + fun `parallel strategy does not shut down the caller-supplied executor`() { + // Caller-owned lifecycle contract: the library must NEVER shut down an injected + // executor. After a full execute() the executor must remain usable for reuse. + val strategy = ParallelExtractionStrategy(executor) + strategy.execute(chunks(4)) { it.id } + + assertFalse(executor.isShutdown, "executor must not be shut down by the strategy") + + // Prove it is still usable for a subsequent run. + val second = strategy.execute(chunks(3)) { it.id } + assertEquals(listOf("c0", "c1", "c2"), second) + } + + @Test + fun `batched strategy does not shut down the caller-supplied executor`() { + val strategy = BatchedExtractionStrategy(batchSize = 2, executor = executor) + strategy.execute(chunks(5)) { it.id } + + assertFalse(executor.isShutdown, "executor must not be shut down by the strategy") + + val second = strategy.execute(chunks(2)) { it.id } + assertEquals(listOf("c0", "c1"), second) + } + + @Test + fun `default ctors provide their own executor and do not require an injected one`() { + // Parallel and Batched must be usable with their default executors + // (no caller-supplied executor), confirming the injected-executor is optional. + val parallel = ParallelExtractionStrategy() + val batched = BatchedExtractionStrategy() + val cs = chunks(6) + + assertEquals(cs.map { it.id }, parallel.execute(cs) { it.id }) + assertEquals(cs.map { it.id }, batched.execute(cs) { it.id }) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/pipeline/PropositionPipelineEventTest.kt b/dice/src/test/kotlin/com/embabel/dice/pipeline/PropositionPipelineEventTest.kt new file mode 100644 index 00000000..0c63dee0 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/pipeline/PropositionPipelineEventTest.kt @@ -0,0 +1,194 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.pipeline + +import com.embabel.agent.core.ContextId +import com.embabel.agent.core.DataDictionary +import com.embabel.agent.rag.model.Chunk +import com.embabel.dice.common.* +import com.embabel.dice.common.resolver.AlwaysCreateEntityResolver +import com.embabel.dice.proposition.* +import com.embabel.dice.proposition.revision.PropositionReviser +import com.embabel.dice.proposition.revision.RevisionResult +import org.junit.jupiter.api.Assertions.assertDoesNotThrow +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +/** + * Tests for the pipeline's event surface (`PropositionPipeline.withEventListener`). + * + * - `process()` emits exactly one [ExtractionBatchCompleted] per run, carrying + * [PropositionExtractionStats]. + * - Per-proposition candidate events (Discovered/Merged/...) are revision-only: a pipeline + * without a reviser emits none of them. The canonical durable signal in that case is + * `PropositionPersisted`, emitted by the repository decorator on save. + * A pipeline with a reviser + `withEventListener` does emit candidate events. + */ +class PropositionPipelineEventTest { + + private val testContextId = ContextId("test-context") + private val schema = DataDictionary.fromClasses("test") + + /** Minimal extractor producing one proposition per "mentions:..." chunk (mirrors PropositionPipelineTest). */ + private class MockExtractor : PropositionExtractor { + override fun extract(chunk: Chunk, context: SourceAnalysisContext): SuggestedPropositions { + val text = chunk.text + if (!text.startsWith("mentions:")) return SuggestedPropositions(chunk.id, emptyList()) + val names = text.substringAfter("mentions:").split(",").map { it.trim() } + val mentions = names.mapIndexed { i, n -> + SuggestedMention(span = n, type = "Person", role = if (i == 0) "SUBJECT" else "OBJECT") + } + return SuggestedPropositions( + chunkId = chunk.id, + propositions = listOf( + SuggestedProposition( + text = "Proposition about ${names.joinToString(" and ")}", + mentions = mentions, + confidence = 0.9, + ) + ), + ) + } + + override fun toSuggestedEntities( + suggestedPropositions: SuggestedPropositions, + context: SourceAnalysisContext, + sourceText: String?, + mentionFilter: com.embabel.dice.common.filter.MentionFilter?, + ): SuggestedEntities { + val entities = suggestedPropositions.propositions.flatMap { it.mentions }.map { + SuggestedEntity(labels = listOf(it.type), name = it.span, summary = it.span, chunkId = suggestedPropositions.chunkId) + }.distinctBy { it.name } + return SuggestedEntities(suggestedEntities = entities, sourceText = sourceText) + } + + override fun resolvePropositions( + suggestedPropositions: SuggestedPropositions, + resolutions: Resolutions, + context: SourceAnalysisContext, + ): List { + val nameToId = resolutions.resolutions.mapNotNull { r -> r.recommended?.let { r.suggested.name.lowercase() to it.id } }.toMap() + return suggestedPropositions.propositions.map { sp -> + Proposition( + contextId = context.contextId, + text = sp.text, + mentions = sp.mentions.map { m -> + EntityMention(span = m.span, type = m.type, role = MentionRole.valueOf(m.role), resolvedId = nameToId[m.span.lowercase()]) + }, + confidence = sp.confidence, + grounding = listOf(suggestedPropositions.chunkId), + ) + } + } + } + + /** Reviser that classifies every proposition as New (a candidate event source). */ + private class AllNewReviser : PropositionReviser { + override fun revise(newProposition: Proposition, repository: PropositionRepository): RevisionResult = + RevisionResult.New(newProposition) + + override fun classify(newProposition: Proposition, candidates: List) = emptyList() + } + + private fun chunks() = listOf( + Chunk(id = "chunk-1", text = "mentions:Alice,Bob", metadata = emptyMap(), parentId = ""), + ) + + private fun context() = SourceAnalysisContext( + schema = schema, + entityResolver = AlwaysCreateEntityResolver, + contextId = testContextId, + ) + + @Test + fun `process emits exactly one ExtractionBatchCompleted carrying stats`() { + val recording = RecordingDiceEventListener() + val pipeline = PropositionPipeline + .withExtractor(MockExtractor()) + .withRevision(AllNewReviser(), inMemoryRepo()) + .withEventListener(recording) + + pipeline.process(chunks(), context()) + + val emitted = recording.eventsOfType() + assertEquals(1, emitted.size, "exactly one ExtractionBatchCompleted per process()") + // Stats shape is the PropositionExtractionStats from the run. + assertTrue(emitted.first().stats.total >= 0) + } + + @Test + fun `pipeline without reviser emits no per-proposition candidate events`() { + val recording = RecordingDiceEventListener() + val pipeline = PropositionPipeline + .withExtractor(MockExtractor()) + .withEventListener(recording) + + pipeline.process(chunks(), context()) + + assertTrue( + recording.eventsOfType().isEmpty(), + "candidate events are revision-only; a reviser-less pipeline must not emit PropositionDiscovered", + ) + } + + @Test + fun `pipeline with reviser emits candidate events`() { + val recording = RecordingDiceEventListener() + val pipeline = PropositionPipeline + .withExtractor(MockExtractor()) + .withRevision(AllNewReviser(), inMemoryRepo()) + .withEventListener(recording) + + pipeline.process(chunks(), context()) + + assertTrue( + recording.eventsOfType().isNotEmpty(), + "a reviser classifying New must yield PropositionDiscovered candidate events", + ) + } + + @Test + fun `a throwing listener never aborts the run or discards the result`() { + val throwing = DiceEventListener { throw RuntimeException("boom") } + val pipeline = PropositionPipeline + .withExtractor(MockExtractor()) + .withRevision(AllNewReviser(), inMemoryRepo()) + .withEventListener(throwing) + + // Both grains emit through the listener; neither may propagate the throw out of the pipeline, + // and process() must still return the fully-computed result. + val result = assertDoesNotThrow { + pipeline.process(chunks(), context()) + } + assertTrue(result.propositionExtractionStats.total >= 0) + } + + private fun inMemoryRepo(): PropositionRepository = object : PropositionRepository { + private val store = mutableMapOf() + override val luceneSyntaxNotes: String = "test" + override fun save(proposition: Proposition): Proposition { store[proposition.id] = proposition; return proposition } + override fun findById(id: String): Proposition? = store[id] + override fun findByEntity(entityIdentifier: com.embabel.agent.rag.service.RetrievableIdentifier): List = emptyList() + override fun findSimilarWithScores(textSimilaritySearchRequest: com.embabel.common.core.types.TextSimilaritySearchRequest) = emptyList>() + override fun findByStatus(status: PropositionStatus): List = emptyList() + override fun findByGrounding(chunkId: String): List = emptyList() + override fun findByMinLevel(minLevel: Int): List = emptyList() + override fun findAll(): List = store.values.toList() + override fun delete(id: String): Boolean = store.remove(id) != null + override fun count(): Int = store.size + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/pipeline/PropositionPipelineTest.kt b/dice/src/test/kotlin/com/embabel/dice/pipeline/PropositionPipelineTest.kt index 300088b5..ebf2cab9 100644 --- a/dice/src/test/kotlin/com/embabel/dice/pipeline/PropositionPipelineTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/pipeline/PropositionPipelineTest.kt @@ -40,6 +40,9 @@ import com.embabel.dice.text2graph.builder.Person import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors private val testContextId = ContextId("test-context") @@ -71,10 +74,11 @@ class TrackingEntityRepository( } /** - * Tests for [PropositionPipeline] focusing on cross-chunk entity resolution. + * Tests for [PropositionPipeline], with emphasis on cross-chunk entity resolution. * - * The pipeline wraps the context's EntityResolver with MultiEntityResolver + InMemoryEntityResolver - * to enable entities discovered in earlier chunks to be recognized in later chunks. + * The pipeline wraps the context's EntityResolver with an InMemoryEntityResolver so entities + * discovered in earlier chunks are recognized in later ones within the same run, without any + * external persistence. */ class PropositionPipelineTest { @@ -87,6 +91,12 @@ class PropositionPipelineTest { private class MockPropositionExtractor : PropositionExtractor { override fun extract(chunk: Chunk, context: SourceAnalysisContext): SuggestedPropositions { + // Sentinel: a "fail:"-prefixed chunk simulates an extraction failure so the + // execution strategy's runCatching yields a null slot -> Phase 2 produces + // ChunkPropositionResult.Failed. + if (chunk.text.startsWith("fail:")) { + throw IllegalStateException("sentinel failure for ${chunk.id}") + } // Parse entities from chunk text: "mentions:Alice,Bob" -> [Alice, Bob] val text = chunk.text if (!text.startsWith("mentions:")) { @@ -643,9 +653,7 @@ class PropositionPipelineTest { } } - /** - * Simple in-memory proposition repository for testing. - */ + /** In-memory proposition store used by tests that need a real repository. */ private class InMemoryPropositionRepository : PropositionRepository { private val propositions = mutableMapOf() @@ -893,9 +901,7 @@ class PropositionPipelineTest { } } - /** - * Tests for structural relationship creation during persist. - */ + /** Verifies that `persist` creates the expected structural relationships (HAS_PROPOSITION, MENTIONS, HAS_ENTITY). */ @Nested inner class StructuralRelationshipTests { @@ -1421,4 +1427,211 @@ class PropositionPipelineTest { assertEquals(1, result.allPropositions.size) } } + + /** + * Concurrency contract: ordering, per-chunk error isolation, and cross-chunk identity under + * the [ParallelExtractionStrategy] and [BatchedExtractionStrategy]. The test owns the + * [ExecutorService] and shuts it down; the library never does. + */ + @Nested + inner class ExecutionStrategyTests { + + private val executor: ExecutorService = Executors.newFixedThreadPool(4) + + @org.junit.jupiter.api.AfterEach + fun tearDown() { + executor.shutdownNow() + } + + private fun newPipeline(strategy: ExtractionExecutionStrategy) = + PropositionPipeline.withExtractor(MockPropositionExtractor()).withExecutionStrategy(strategy) + + private fun context() = SourceAnalysisContext( + schema = schema, + entityResolver = AlwaysCreateEntityResolver, + contextId = testContextId, + ) + + private fun strategies(): List = listOf( + ParallelExtractionStrategy(executor), + BatchedExtractionStrategy(batchSize = 2, executor = executor), + ) + + @Test + fun `failed chunk is isolated as Failed while good chunks succeed`() { + for (strategy in strategies()) { + val chunks = listOf( + Chunk(id = "chunk-1", text = "mentions:Alice", metadata = emptyMap(), parentId = ""), + Chunk(id = "chunk-2", text = "fail:boom", metadata = emptyMap(), parentId = ""), + Chunk(id = "chunk-3", text = "mentions:Bob", metadata = emptyMap(), parentId = ""), + ) + + val result = newPipeline(strategy).process(chunks, context()) + + // Invariant: one result per input chunk, in order. + assertEquals(chunks.size, result.chunkResults.size) + + // chunk-2 is Failed; the others are Success. + assertTrue(result.chunkResults[0] is ChunkPropositionResult.Success) + assertTrue(result.chunkResults[1] is ChunkPropositionResult.Failed) + assertTrue(result.chunkResults[2] is ChunkPropositionResult.Success) + + // failedChunkIds carries exactly the failed chunk id. + assertEquals(listOf("chunk-2"), result.failedChunkIds) + + // Good chunks still produced their entities. + val names = result.newEntities().map { it.name }.toSet() + assertTrue(names.contains("Alice")) + assertTrue(names.contains("Bob")) + } + } + + @Test + fun `a chunk that fails during resolution is isolated as Failed`() { + // Phase 1 (extraction) succeeds for every chunk; the resolver throws for chunk-2 + // during Phase 2 resolution. The run must still produce one result per chunk with + // chunk-2 surfaced as Failed — not abort and discard the good chunks' results. + val resolverThatThrowsOnBoom = object : EntityResolver { + override fun resolve( + suggestedEntities: SuggestedEntities, + schema: DataDictionary, + ): Resolutions { + if (suggestedEntities.suggestedEntities.any { it.name == "Boom" }) { + throw IllegalStateException("resolution failed for Boom") + } + return AlwaysCreateEntityResolver.resolve(suggestedEntities, schema) + } + } + val failingResolveContext = SourceAnalysisContext( + schema = schema, + entityResolver = resolverThatThrowsOnBoom, + contextId = testContextId, + ) + + for (strategy in strategies()) { + val chunks = listOf( + Chunk(id = "chunk-1", text = "mentions:Alice", metadata = emptyMap(), parentId = ""), + Chunk(id = "chunk-2", text = "mentions:Boom", metadata = emptyMap(), parentId = ""), + Chunk(id = "chunk-3", text = "mentions:Bob", metadata = emptyMap(), parentId = ""), + ) + + val result = newPipeline(strategy).process(chunks, failingResolveContext) + + // Invariant: one result per input chunk, in order; the run did not abort. + assertEquals(chunks.size, result.chunkResults.size) + assertTrue(result.chunkResults[0] is ChunkPropositionResult.Success) + assertTrue(result.chunkResults[1] is ChunkPropositionResult.Failed) + assertTrue(result.chunkResults[2] is ChunkPropositionResult.Success) + assertEquals(listOf("chunk-2"), result.failedChunkIds) + + // Good chunks still produced their entities despite the resolution failure. + val names = result.newEntities().map { it.name }.toSet() + assertTrue(names.contains("Alice")) + assertTrue(names.contains("Bob")) + } + } + + @Test + fun `results preserve input order`() { + for (strategy in strategies()) { + val chunks = (0 until 10).map { + Chunk(id = "chunk-$it", text = "mentions:E$it", metadata = emptyMap(), parentId = "") + } + + val result = newPipeline(strategy).process(chunks, context()) + + assertEquals(chunks.size, result.chunkResults.size) + chunks.forEachIndexed { i, chunk -> + assertEquals(chunk.id, result.chunkResults[i].chunkId, "Order mismatch at index $i") + } + } + } + + @Test + fun `cross-chunk identity preserved under parallel and batched`() { + for (strategy in strategies()) { + val chunks = listOf( + Chunk(id = "chunk-1", text = "mentions:Alice,Bob", metadata = emptyMap(), parentId = ""), + Chunk(id = "chunk-2", text = "mentions:Alice,Charlie", metadata = emptyMap(), parentId = ""), + ) + + val result = newPipeline(strategy).process(chunks, context()) + + // 3 unique entities; Alice appears once in newEntities (resolved serially in Phase 2). + val newEntities = result.newEntities() + assertEquals(3, newEntities.size, "Found: ${newEntities.map { it.name }}") + assertEquals(1, newEntities.count { it.name == "Alice" }) + + // Every Alice mention resolves to the same id across chunks. + val aliceId = newEntities.first { it.name == "Alice" }.id + for (prop in result.allPropositions) { + prop.mentions.find { it.span == "Alice" }?.let { + assertEquals(aliceId, it.resolvedId, "All Alice mentions must share one id under $strategy") + } + } + } + } + + @Test + fun `default pipeline (no withExecutionStrategy) matches serial behavior`() { + // Oracle: the existing default-path tests assert serial behavior. Here we assert + // an explicit SerialExtractionStrategy yields identical results to the default. + val chunks = listOf( + Chunk(id = "chunk-1", text = "mentions:Alice", metadata = emptyMap(), parentId = ""), + Chunk(id = "chunk-2", text = "mentions:Alice", metadata = emptyMap(), parentId = ""), + ) + + val defaultResult = PropositionPipeline.withExtractor(MockPropositionExtractor()) + .process(chunks, context()) + val serialResult = newPipeline(SerialExtractionStrategy).process(chunks, context()) + + assertEquals(defaultResult.chunkResults.size, serialResult.chunkResults.size) + assertEquals(defaultResult.newEntities().size, serialResult.newEntities().size) + assertEquals(1, serialResult.newEntities().size) + } + + @Test + fun `single-chunk processChunk still throws on a failing chunk (A3)`() { + val pipeline = PropositionPipeline.withExtractor(MockPropositionExtractor()) + val failingChunk = Chunk(id = "chunk-x", text = "fail:boom", metadata = emptyMap(), parentId = "") + + // processChunk does NOT isolate failure — it propagates (unlike process()). + assertThrows { + pipeline.processChunk(failingChunk, context()) + } + } + + @Test + fun `default strategy and batched-size-one yield equivalent results under partial failure`() { + // Safe-path equivalence at the PIPELINE level: the default pipeline + // (Serial) and an explicit BatchedExtractionStrategy(1) must produce the same number + // of chunk results, the same Failed/Success classification per index, and the same + // failedChunkIds — proving Batched(1) is a true drop-in for the serial default. + val chunks = listOf( + Chunk(id = "chunk-1", text = "mentions:Alice", metadata = emptyMap(), parentId = ""), + Chunk(id = "chunk-2", text = "fail:boom", metadata = emptyMap(), parentId = ""), + Chunk(id = "chunk-3", text = "mentions:Bob", metadata = emptyMap(), parentId = ""), + ) + + val defaultResult = PropositionPipeline.withExtractor(MockPropositionExtractor()) + .process(chunks, context()) + val batched1Result = newPipeline( + BatchedExtractionStrategy(batchSize = 1, executor = executor) + ).process(chunks, context()) + + assertEquals(defaultResult.chunkResults.size, batched1Result.chunkResults.size) + assertEquals(defaultResult.failedChunkIds, batched1Result.failedChunkIds) + assertEquals(listOf("chunk-2"), batched1Result.failedChunkIds) + + defaultResult.chunkResults.forEachIndexed { i, expected -> + val actual = batched1Result.chunkResults[i] + assertEquals(expected::class, actual::class, "classification mismatch at index $i") + assertEquals(expected.chunkId, actual.chunkId, "chunkId mismatch at index $i") + } + assertEquals( + defaultResult.newEntities().map { it.name }.toSet(), + batched1Result.newEntities().map { it.name }.toSet(), + ) + } + } } diff --git a/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt b/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt index c7072aab..5d274c29 100644 --- a/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt @@ -43,7 +43,10 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status import org.springframework.test.web.servlet.setup.MockMvcBuilders /** - * Tests for PropositionPipelineController. + * Contract tests for the proposition extraction REST controller. + * + * Verifies that the `/extract` endpoint runs the pipeline, persists propositions, and returns the + * expected JSON shape — using a mocked pipeline and an in-memory repository. */ class PropositionPipelineControllerTest { @@ -103,7 +106,7 @@ class PropositionPipelineControllerTest { confidence = 0.95, ) - val mockResult = ChunkPropositionResult( + val mockResult = ChunkPropositionResult.Success( chunkId = "chunk-123", suggestedPropositions = SuggestedPropositions( chunkId = "chunk-123", @@ -169,7 +172,7 @@ class PropositionPipelineControllerTest { confidence = 0.9, ) - val mockResult = ChunkPropositionResult( + val mockResult = ChunkPropositionResult.Success( chunkId = "chunk-456", suggestedPropositions = SuggestedPropositions( chunkId = "chunk-456", From c8cc36f4052d332f61593784a3592c543e88a6ec Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:55:14 -0400 Subject: [PATCH 02/27] feat(collector): mark-and-sweep collector for the proposition store Adds a garbage-collector-style maintenance pass: mark candidates, then sweep. It runs dry by default and records what each run did, so a real run's effect is visible before anything is removed. - CollectorStrategy with DecayCollectorStrategy and DuplicateCollectorStrategy; CollectorRunner / DefaultCollectorRunner drive mark then sweep - SweepPolicy and SweepAction decide what happens to a marked proposition; MarkReason and PropositionMark record why it was marked - CollectorRun / CollectorRecord / CollectorOutcome give every run an audit trail, held by a CollectorRecordStore Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../projection/lineage/CollectorOutcome.kt | 39 +++ .../projection/lineage/CollectorRecord.kt | 94 +++++ .../lineage/CollectorRecordStore.kt | 82 +++++ .../dice/projection/lineage/CollectorRun.kt | 53 +++ .../lineage/InMemoryCollectorRecordStore.kt | 43 +++ .../projection/memory/CollectorRunResult.kt | 46 +++ .../dice/projection/memory/CollectorRunner.kt | 126 +++++++ .../projection/memory/CollectorStrategy.kt | 48 +++ .../memory/DecayCollectorStrategy.kt | 53 +++ .../memory/DefaultCollectorRunner.kt | 225 ++++++++++++ .../memory/DuplicateCollectorStrategy.kt | 141 ++++++++ .../dice/projection/memory/MarkReason.kt | 55 +++ .../dice/projection/memory/PropositionMark.kt | 41 +++ .../dice/projection/memory/SweepAction.kt | 44 +++ .../dice/projection/memory/SweepPolicy.kt | 64 ++++ .../projection/lineage/CollectorRecordTest.kt | 154 +++++++++ .../InMemoryCollectorRecordStoreTest.kt | 89 +++++ .../memory/CollectorRunnerBuilderTest.kt | 90 +++++ .../memory/CollectorStrategyTest.kt | 95 ++++++ .../memory/DecayCollectorStrategyTest.kt | 110 ++++++ .../memory/DefaultCollectorRunnerTest.kt | 323 ++++++++++++++++++ .../memory/DuplicateCollectorStrategyTest.kt | 180 ++++++++++ .../memory/StatusTransitionSweepPolicyTest.kt | 85 +++++ 23 files changed, 2280 insertions(+) create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorOutcome.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRecord.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRecordStore.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRun.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStore.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunResult.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunner.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorStrategy.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategy.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/MarkReason.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/PropositionMark.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepAction.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepPolicy.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/projection/lineage/CollectorRecordTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStoreTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorRunnerBuilderTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorStrategyTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategyTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategyTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/projection/memory/StatusTransitionSweepPolicyTest.kt diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorOutcome.kt b/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorOutcome.kt new file mode 100644 index 00000000..eb17fa8b --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorOutcome.kt @@ -0,0 +1,39 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.lineage + +/** + * What the collector did to a single proposition during a collection run. + * + * Distinct from a projection lifecycle: a collector marks, transitions, or (only + * when explicitly opted in) hard-deletes propositions, and may skip those that are + * exempt (e.g. pinned). The outcome is recorded on a [CollectorRecord] alongside the + * typed reason so a reviewer can trace why each proposition was acted upon. + */ +enum class CollectorOutcome { + + /** The proposition was marked for collection but not yet swept. */ + MARKED, + + /** The proposition's status was transitioned (the non-destructive default sweep). */ + TRANSITIONED, + + /** The proposition was permanently removed (explicit opt-in only; never the default). */ + HARD_DELETED, + + /** The proposition was intentionally not acted upon (e.g. exempt or not eligible). */ + SKIPPED +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRecord.kt b/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRecord.kt new file mode 100644 index 00000000..1f35c2c8 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRecord.kt @@ -0,0 +1,94 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.lineage + +import com.embabel.dice.projection.memory.MarkReason +import com.embabel.dice.proposition.PropositionStatus +import java.time.Instant + +/** + * A record of one proposition being acted upon by a collector during a run. + * + * Collectively these records form an audit trail — "which propositions were marked + * or swept, why, and by which strategy" — so a reviewer can trace any retired or + * removed proposition back to the run and reason that produced the outcome. + * + * The reason is a typed [MarkReason] rather than a free-form string, and the + * action is a typed [CollectorOutcome] rather than a lifecycle/magic value, so the + * collector audit trail stays semantically distinct from projection lineage. + * + * @property propositionId ID of the proposition that was acted upon + * @property reason Typed explanation of why the proposition was marked + * @property outcome What the collector did to the proposition + * @property strategyName Name of the collector strategy that produced this record + * @property runId ID of the collection run that produced this record + * @property at When this record was created + * @property previousStatus The proposition's status before the outcome, or null if not applicable + * @property newStatus The proposition's status after the outcome, or null if not applicable + */ +data class CollectorRecord @JvmOverloads constructor( + val propositionId: String, + val reason: MarkReason, + val outcome: CollectorOutcome, + val strategyName: String, + val runId: String, + val at: Instant = Instant.now(), + val previousStatus: PropositionStatus? = null, + val newStatus: PropositionStatus? = null, +) { + + init { + require(propositionId.isNotBlank()) { "propositionId must not be blank" } + require(runId.isNotBlank()) { "runId must not be blank" } + } + + companion object { + + /** + * Java-friendly factory method to create a [CollectorRecord]. + * + * @param propositionId ID of the proposition that was acted upon + * @param reason Typed explanation of why the proposition was marked + * @param outcome What the collector did to the proposition + * @param strategyName Name of the collector strategy that produced this record + * @param runId ID of the collection run + * @param at When this record was created + * @param previousStatus The proposition's status before the outcome + * @param newStatus The proposition's status after the outcome + */ + @JvmStatic + @JvmOverloads + fun of( + propositionId: String, + reason: MarkReason, + outcome: CollectorOutcome, + strategyName: String, + runId: String, + at: Instant = Instant.now(), + previousStatus: PropositionStatus? = null, + newStatus: PropositionStatus? = null, + ): CollectorRecord = CollectorRecord( + propositionId = propositionId, + reason = reason, + outcome = outcome, + strategyName = strategyName, + runId = runId, + at = at, + previousStatus = previousStatus, + newStatus = newStatus, + ) + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRecordStore.kt b/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRecordStore.kt new file mode 100644 index 00000000..e6f5b013 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRecordStore.kt @@ -0,0 +1,82 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.lineage + +/** + * Append-only store of [CollectorRecord]s — the audit trail of "what the collector + * marked or swept, why, and by which strategy". + * + * Implementations may be in-memory, graph-backed, or relational. The store is + * append-only: there is no method to remove or mutate records, so the history is + * non-destructive. The default query methods are expressed in terms of [all] and [runs] + * so that simple implementations only need to supply the writers ([record], [recordRun]) + * and the readers ([all], [runs]). + */ +interface CollectorRecordStore { + + /** + * Record a collector outcome. + * + * @param record The collector record to store + */ + fun record(record: CollectorRecord) + + /** + * Record a finished run header. + * + * Persisting the run header ensures even a run with zero marks leaves a trace in the audit + * trail, grouping all [CollectorRecord]s for the run under a shared [CollectorRun.runId]. + * + * @param run The finished run header to store + */ + fun recordRun(run: CollectorRun) + + /** + * Find the run header for a given run id. + * + * @param runId ID of the collection run + * @return the run whose [CollectorRun.runId] matches, or null if none was recorded + */ + fun findRun(runId: String): CollectorRun? = runs().firstOrNull { it.runId == runId } + + /** + * @return all run headers held by this store, in insertion order + */ + fun runs(): List + + /** + * Find all records for a given proposition. + * + * @param propositionId ID of the proposition + * @return records whose [CollectorRecord.propositionId] matches + */ + fun findByProposition(propositionId: String): List = + all().filter { it.propositionId == propositionId } + + /** + * Find all records produced by a given run. + * + * @param runId ID of the collection run + * @return records whose [CollectorRecord.runId] matches + */ + fun findByRun(runId: String): List = + all().filter { it.runId == runId } + + /** + * @return all records held by this store + */ + fun all(): List +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRun.kt b/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRun.kt new file mode 100644 index 00000000..c5fe5b18 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRun.kt @@ -0,0 +1,53 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.lineage + +import java.time.Instant + +/** + * A record of a collection run — a single batch of mark-and-sweep work over propositions. + * + * Groups many [CollectorRecord]s under a shared [runId]. Carries a [dryRun] flag so + * non-mutating previews are recorded separately from runs that actually transition + * propositions. Computed counts are intentionally out of scope; this is a plain record. + * + * @property runId Unique identifier for this run (matches [CollectorRecord.runId]) + * @property startedAt When the run started + * @property finishedAt When the run finished, or null if still running + * @property dryRun Whether this run was a non-mutating preview (no propositions changed) + */ +data class CollectorRun @JvmOverloads constructor( + val runId: String, + val startedAt: Instant, + val finishedAt: Instant? = null, + val dryRun: Boolean = false, +) { + + init { + require(runId.isNotBlank()) { "runId must not be blank" } + require(finishedAt == null || !finishedAt.isBefore(startedAt)) { + "finishedAt must be null or >= startedAt" + } + } + + /** + * Create a copy marked finished at the given instant. + * + * @param at When the run finished (defaults to now) + * @return a copy with [finishedAt] set + */ + fun finished(at: Instant = Instant.now()): CollectorRun = copy(finishedAt = at) +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStore.kt b/dice/src/main/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStore.kt new file mode 100644 index 00000000..4c1c6e74 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStore.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.lineage + +import java.util.concurrent.CopyOnWriteArrayList + +/** + * Thread-safe in-memory implementation of [CollectorRecordStore]. + * + * Intended as a default/stub for demos and tests. Records are append-only and + * returned in insertion order. Backed by a [CopyOnWriteArrayList] so reads never + * block writes. + */ +class InMemoryCollectorRecordStore : CollectorRecordStore { + + private val records = CopyOnWriteArrayList() + private val runHeaders = CopyOnWriteArrayList() + + override fun record(record: CollectorRecord) { + records.add(record) + } + + override fun recordRun(run: CollectorRun) { + runHeaders.add(run) + } + + override fun all(): List = records.toList() + + override fun runs(): List = runHeaders.toList() +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunResult.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunResult.kt new file mode 100644 index 00000000..34aa6ea0 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunResult.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import java.time.Instant + +/** + * Immutable summary of a single collector run. + * + * Reports what the run marked and, when not a preview, what it actually applied. The + * [applied]/[skipped]/[hardDeleted] partitions let a caller audit the outcome without + * re-querying the repository. + * + * @property runId Unique identifier for this run, shared with its audit records in the record store. + * Blank for pure-read [CollectorRunner.collect] results — those are never persisted. + * @property dryRun Whether this was a preview run (no propositions were changed). + * @property marks Every mark produced by the configured strategies during this run. + * @property applied Marks whose proposition was actually transitioned (empty on a dry run). + * @property skipped Marks whose proposition was intentionally left untouched (e.g. pinned). + * @property hardDeleted IDs of propositions permanently removed (only via an opt-in policy). + * @property startedAt When the run started. + * @property finishedAt When the run finished; defaults to now. + */ +data class CollectorRunResult @JvmOverloads constructor( + val runId: String, + val dryRun: Boolean, + val marks: List, + val applied: List, + val skipped: List, + val hardDeleted: List, + val startedAt: Instant, + val finishedAt: Instant = Instant.now(), +) diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunner.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunner.kt new file mode 100644 index 00000000..105cf67b --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunner.kt @@ -0,0 +1,126 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.common.DiceEventListener +import com.embabel.dice.projection.lineage.CollectorRecordStore +import com.embabel.dice.proposition.PropositionRepository + +/** + * Runs the mark-and-sweep collector for a context: selects ACTIVE propositions, hands them to + * one or more [CollectorStrategy]s for marking, then asks the [SweepPolicy] what to do with + * each marked one. + * + * Two entry points with different write behavior: + * - [collect] is mark-only — purely in-memory, never writes a thing. + * - [run] is mark + sweep — `dryRun = true` previews the outcome and saves an audit record; + * `dryRun = false` applies the decisions for real. + * + * Build one via the fluent [withRepository] entry point. + */ +interface CollectorRunner { + + /** + * Run the mark phase only, leaving everything untouched. + * + * No repository writes, no run record. Good for inspecting what a sweep would do before + * committing. Because nothing is persisted, the returned [CollectorRunResult.runId] is + * blank — don't look it up in a record store. + * + * @param contextId The context whose ACTIVE propositions are evaluated. + * @return A result with marks populated but applied/skipped partitions empty, and a blank runId. + */ + fun collect(contextId: ContextId): CollectorRunResult + + /** + * Run the full mark + sweep. + * + * Either way, an auditable run record is saved. With `dryRun = true`, no proposition status + * changes and no lifecycle event is emitted. With `dryRun = false`, each decision is applied + * and a [com.embabel.dice.common.PropositionStatusChanged] event is emitted per transition. + * + * @param contextId The context whose ACTIVE propositions are evaluated. + * @param dryRun When true, preview only — no proposition changes, no events. + * @return A result partitioned into marks/applied/skipped/hardDeleted. + */ + fun run(contextId: ContextId, dryRun: Boolean = false): CollectorRunResult + + companion object { + + /** + * Start building a [CollectorRunner] backed by [repository]. + * + * @param repository The proposition store to read candidates from and write transitions to. + * @return A [Builder] with defaults: no strategies, [StatusTransitionSweepPolicy], no + * record store, and a no-op event listener. + */ + @JvmStatic + fun withRepository(repository: PropositionRepository): Builder = Builder(repository) + } + + /** + * Fluent builder for a [DefaultCollectorRunner]. + * + * Defaults: empty strategy list, [StatusTransitionSweepPolicy], no record store (audit off), + * and [DiceEventListener.DEV_NULL] (events off). + */ + class Builder internal constructor(private val repository: PropositionRepository) { + + private val strategies = mutableListOf() + private var policy: SweepPolicy = StatusTransitionSweepPolicy() + private var recordStore: CollectorRecordStore? = null + private var listener: DiceEventListener = DiceEventListener.DEV_NULL + + /** + * Add a strategy whose marks contribute to the sweep. + * + * @param strategy The strategy to run during the mark phase. + * @return this builder, for chaining. + */ + fun withStrategy(strategy: CollectorStrategy): Builder = apply { strategies.add(strategy) } + + /** + * Set the policy that decides each marked proposition's fate. + * + * @param policy The sweep policy (defaults to [StatusTransitionSweepPolicy]). + * @return this builder, for chaining. + */ + fun withPolicy(policy: SweepPolicy): Builder = apply { this.policy = policy } + + /** + * Attach an audit store so run records are persisted. Without this, no trail is written. + * + * @param recordStore The store collector records are appended to. + * @return this builder, for chaining. + */ + fun withRecordStore(recordStore: CollectorRecordStore): Builder = apply { this.recordStore = recordStore } + + /** + * Attach a listener to receive events after each applied transition. + * + * @param listener The event listener (defaults to [DiceEventListener.DEV_NULL]). + * @return this builder, for chaining. + */ + fun withEventListener(listener: DiceEventListener): Builder = apply { this.listener = listener } + + /** + * @return a [DefaultCollectorRunner] from this builder's configuration. + */ + fun build(): CollectorRunner = + DefaultCollectorRunner(repository, strategies.toList(), policy, recordStore, listener) + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorStrategy.kt new file mode 100644 index 00000000..e746a00a --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorStrategy.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionRepository + +/** + * The "mark" half of the mark-and-sweep collector: looks at candidate propositions and + * reports which ones should be collected, and why. + * + * Implementations should be stateless and read-only — the same inputs should always produce + * the same marks, and the repository should never be mutated here. Marking is a + * read-and-report step; the [SweepPolicy] decides what actually happens to each mark. + * + * The repository is provided so a strategy can look up extra context it needs (e.g. finding + * a duplicate's survivor) without having to own its own storage. + */ +fun interface CollectorStrategy { + + /** + * Inspect [candidates] and report which should be collected. + * + * @param candidates The propositions under consideration (runner-selected). + * @param repository Read access to stored propositions; do not write to it. + * @param contextId The context the candidates belong to. + * @return Marks for the propositions this strategy wants collected; empty if none qualify. + */ + fun mark( + candidates: List, + repository: PropositionRepository, + contextId: ContextId, + ): List +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt new file mode 100644 index 00000000..7be71549 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt @@ -0,0 +1,53 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionRepository + +/** + * A [CollectorStrategy] that marks propositions whose decayed confidence has fallen below a + * staleness threshold. + * + * Rather than deleting anything directly, it reports a [MarkReason.Stale] mark and leaves the + * actual decision to the sweep phase — which by default soft-transitions the proposition to + * STALE rather than removing it. + * + * Stateless and read-only: the same candidates always produce the same marks, and the + * repository is never consulted (the candidate set comes from the runner). + * + * @property retireBelow Effective-confidence threshold below which a candidate is marked stale. + * @property retireDecayK Decay-rate multiplier for [Proposition.effectiveConfidence]. + */ +class DecayCollectorStrategy @JvmOverloads constructor( + private val retireBelow: Double, + private val retireDecayK: Double = 2.0, +) : CollectorStrategy { + + override fun mark( + candidates: List, + repository: PropositionRepository, + contextId: ContextId, + ): List = + candidates + .filter { it.effectiveConfidence(retireDecayK) < retireBelow } + .map { PropositionMark(propositionId = it.id, reason = MarkReason.Stale, strategyName = STRATEGY_NAME) } + + companion object { + private const val STRATEGY_NAME = "decay" + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt new file mode 100644 index 00000000..c0b41666 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt @@ -0,0 +1,225 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.common.DiceEventListener +import com.embabel.dice.common.PropositionStatusChanged +import com.embabel.dice.projection.lineage.CollectorOutcome +import com.embabel.dice.projection.lineage.CollectorRecord +import com.embabel.dice.projection.lineage.CollectorRecordStore +import com.embabel.dice.projection.lineage.CollectorRun +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionQuery +import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.proposition.PropositionStatus +import org.slf4j.LoggerFactory +import java.time.Instant +import java.util.UUID + +/** + * Default [CollectorRunner] implementation. + * + * Fetches ACTIVE candidates once per run (so already-STALE or PROMOTED propositions are + * never re-selected), gathers marks from every configured [CollectorStrategy], then asks the + * [SweepPolicy] what to do with each marked proposition. + * + * Write behavior by entry point: + * - [collect] never touches the repository or record store. + * - [run] with `dryRun = true` saves an auditable run record but applies no status change and + * emits no event. + * - [run] with `dryRun = false` applies each decision, saves the run record, then emits a + * [PropositionStatusChanged] per applied transition. + * + * @param repository Proposition store to read candidates from and write transitions to. + * @param strategies Mark strategies run during the mark phase. + * @param policy Policy deciding each marked proposition's fate. + * @param recordStore Optional audit store; when null, no run record is saved. + * @param listener Notified after each applied transition; defaults to a no-op. + */ +class DefaultCollectorRunner( + private val repository: PropositionRepository, + private val strategies: List, + private val policy: SweepPolicy, + private val recordStore: CollectorRecordStore?, + private val listener: DiceEventListener, +) : CollectorRunner { + + private val logger = LoggerFactory.getLogger(DefaultCollectorRunner::class.java) + + override fun collect(contextId: ContextId): CollectorRunResult { + val startedAt = Instant.now() + val (_, marks) = markPhase(contextId) + // Pure-read: no repository write, no run record. Nothing is persisted, so there is no run + // to cross-reference — the runId is blank to signal it is not queryable in any store. + return CollectorRunResult( + runId = EPHEMERAL_RUN_ID, + dryRun = false, + marks = marks, + applied = emptyList(), + skipped = emptyList(), + hardDeleted = emptyList(), + startedAt = startedAt, + ) + } + + override fun run(contextId: ContextId, dryRun: Boolean): CollectorRunResult { + val startedAt = Instant.now() + val runId = newRunId() + val (candidatesById, marks) = markPhase(contextId) + val marksByProposition = marks.groupBy { it.propositionId } + + val applied = mutableListOf() + val skipped = mutableListOf() + val hardDeleted = mutableListOf() + val records = mutableListOf() + + for ((propositionId, propMarks) in marksByProposition) { + val proposition = candidatesById[propositionId] ?: continue + when (val action = policy.decide(proposition, propMarks)) { + is SweepAction.TransitionStatus -> { + if (dryRun) { + // Would-be transition: record it, but mutate nothing, emit nothing, and + // leave `applied` empty — `marks` already reflects what WOULD happen. + records += records(propMarks, runId, CollectorOutcome.TRANSITIONED, proposition.status, action.newStatus) + } else { + applyTransition(proposition, action.newStatus, propMarks) + applied.addAll(propMarks) + records += records(propMarks, runId, CollectorOutcome.TRANSITIONED, proposition.status, action.newStatus) + } + } + + SweepAction.HardDelete -> { + if (!dryRun) { + repository.delete(proposition.id) + // Only an actually-removed proposition is reported as hard-deleted; on a + // dry run `hardDeleted` stays empty (the record still captures the preview). + hardDeleted += proposition.id + } + records += records(propMarks, runId, CollectorOutcome.HARD_DELETED, proposition.status, null) + } + + SweepAction.Skip -> { + skipped.addAll(propMarks) + records += records(propMarks, runId, CollectorOutcome.SKIPPED, proposition.status, null) + } + } + } + + // Compute the finish instant once and thread it into both the persisted run header and + // the returned result, so the audit object and the summary agree on the finish time. + val finishedAt = Instant.now() + persistRun(runId, startedAt, finishedAt, dryRun, records) + + return CollectorRunResult( + runId = runId, + dryRun = dryRun, + marks = marks, + applied = applied.toList(), + skipped = skipped.toList(), + hardDeleted = hardDeleted.toList(), + startedAt = startedAt, + finishedAt = finishedAt, + ) + } + + /** + * Fetches ACTIVE candidates once and runs every strategy over them. + * @return the candidates indexed by id, paired with all marks the strategies produced. + */ + private fun markPhase(contextId: ContextId): Pair, List> { + val candidates = repository.query( + PropositionQuery.forContextId(contextId).withStatus(PropositionStatus.ACTIVE), + ) + val candidatesById = candidates.associateBy { it.id } + val marks = strategies.flatMap { it.mark(candidates, repository, contextId) } + return candidatesById to marks + } + + /** Persist the swept transition, then emit the lifecycle event (persist-then-emit). */ + private fun applyTransition( + proposition: Proposition, + newStatus: PropositionStatus, + propMarks: List, + ) { + val previousStatus = proposition.status + val saved = repository.save(proposition.withStatus(newStatus)) + // Multiple strategies may mark the same proposition; combine their distinct reason keys + // (sorted for run-to-run determinism) so the emitted event is order-independent and never + // silently drops a reason. `reason` stays a nullable String for backward compatibility. + val reason = propMarks + .map { it.reason.key } + .distinct() + .sorted() + .joinToString(",") + .ifEmpty { null } + listener.onEvent( + PropositionStatusChanged( + proposition = saved, + previousStatus = previousStatus, + newStatus = newStatus, + reason = reason, + ), + ) + } + + private fun records( + propMarks: List, + runId: String, + outcome: CollectorOutcome, + previousStatus: PropositionStatus?, + newStatus: PropositionStatus?, + ): List = propMarks.map { mark -> + CollectorRecord( + propositionId = mark.propositionId, + reason = mark.reason, + outcome = outcome, + strategyName = mark.strategyName, + runId = runId, + previousStatus = previousStatus, + newStatus = newStatus, + ) + } + + private fun persistRun( + runId: String, + startedAt: Instant, + finishedAt: Instant, + dryRun: Boolean, + records: List, + ) { + val store = recordStore ?: return + // The finished run header groups the per-proposition trail under a shared runId; the + // record store owns the durable trail, so records carry that runId forward. The header + // is persisted unconditionally — even a zero-mark run must leave a retrievable trace. + val run = CollectorRun(runId = runId, startedAt = startedAt, finishedAt = finishedAt, dryRun = dryRun) + store.recordRun(run) + records.forEach(store::record) + logger.debug( + "Collector run {} finished (dryRun={}, records={})", + run.runId, + run.dryRun, + records.size, + ) + } + + private fun newRunId(): String = UUID.randomUUID().toString() + + private companion object { + /** Sentinel runId for the pure-read [collect] path: not persisted, not queryable. */ + const val EPHEMERAL_RUN_ID = "" + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategy.kt new file mode 100644 index 00000000..40ee2dfa --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategy.kt @@ -0,0 +1,141 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionQuery +import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.proposition.PropositionStatus + +/** + * A [CollectorStrategy] that finds near-duplicate propositions and marks all but the strongest + * member of each duplicate group. + * + * Because clusters from [PropositionRepository.findClusters] can overlap (a proposition may + * appear in more than one cluster), survivors are picked globally per connected component rather + * than per cluster. Within each component the survivor is the proposition with the highest + * effective confidence, with reinforcement count and id as tie-breakers. Every non-survivor gets + * a [MarkReason.Duplicate] mark pointing at its component's survivor; survivors are never marked. + * + * Only propositions in the runner's candidate snapshot are considered — any cluster member + * absent from `candidates` is silently ignored. This keeps every emitted mark tied to a swept + * candidate and avoids a second read of the repository. The strategy never writes. + * + * Default [similarityThreshold] (0.7) and [topK] (10) match [PropositionRepository.findClusters] + * so behavior is consistent with the repository's own clustering out of the box. + * + * @property similarityThreshold Minimum cosine similarity for two propositions to be clustered together. + * @property topK Maximum number of similar members considered per cluster seed. + */ +class DuplicateCollectorStrategy @JvmOverloads constructor( + private val similarityThreshold: Double = 0.7, + private val topK: Int = 10, +) : CollectorStrategy { + + override fun mark( + candidates: List, + repository: PropositionRepository, + contextId: ContextId, + ): List { + val byId = candidates.associateBy { it.id } + val clusters = repository.findClusters( + similarityThreshold = similarityThreshold, + topK = topK, + query = PropositionQuery.forContextId(contextId).withStatus(PropositionStatus.ACTIVE), + ) + + // Build connected components over clustered members, restricted to the runner-supplied + // candidate snapshot so every mark maps to a swept candidate. + val unionFind = UnionFind() + for (cluster in clusters) { + val memberIds = (listOf(cluster.anchor.id) + cluster.similar.map { it.match.id }) + .filter { byId.containsKey(it) } + .distinct() + // Union all members of this cluster into one component. + for (i in 1 until memberIds.size) { + unionFind.union(memberIds[0], memberIds[i]) + } + } + + // Group candidate members by their component root. + val componentMembers: Map> = unionFind.members() + .mapNotNull { id -> byId[id] } + .groupBy { unionFind.find(it.id) } + + return componentMembers.values + .filter { it.size >= 2 } + .flatMap { members -> + // Global survivor per component: max effectiveConfidence, then reinforceCount, + // then a stable id tie-break for full determinism on ties. + val survivor = members.maxWith( + compareBy({ it.effectiveConfidence() }, { it.reinforceCount }, { it.id }), + ) + members + .filter { it.id != survivor.id } + .map { + PropositionMark( + propositionId = it.id, + reason = MarkReason.Duplicate(survivorId = survivor.id), + strategyName = STRATEGY_NAME, + ) + } + } + // Each non-survivor belongs to exactly one component, but dedup defensively so a + // proposition is never marked more than once across overlapping clusters. + .distinctBy { it.propositionId } + .sortedBy { it.propositionId } + } + + /** + * Simple union-find over proposition ids, used to merge overlapping clusters into + * connected components so one global survivor can be chosen per component. + */ + private class UnionFind { + private val parent = mutableMapOf() + + fun find(id: String): String { + parent.getOrPut(id) { id } + var root = id + while (parent.getValue(root) != root) { + root = parent.getValue(root) + } + // Path compression: point every node on the walk directly at the root. + var cur = id + while (cur != root) { + val next = parent.getValue(cur) + parent[cur] = root + cur = next + } + return root + } + + fun union(a: String, b: String) { + val rootA = find(a) + val rootB = find(b) + if (rootA != rootB) { + // Deterministic merge direction (smaller id becomes root). + if (rootA <= rootB) parent[rootB] = rootA else parent[rootA] = rootB + } + } + + fun members(): Set = parent.keys.toSet() + } + + companion object { + private const val STRATEGY_NAME = "duplicate" + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/MarkReason.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/MarkReason.kt new file mode 100644 index 00000000..b3c0806e --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/MarkReason.kt @@ -0,0 +1,55 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +/** + * Why a proposition was marked for collection by a [CollectorStrategy]. + * + * A sealed family so the set of recognized reasons is closed for the built-in + * strategies, while remaining open for consumer-specific signals via [Custom]. + * Every case exposes a stable machine [key] so a downstream audit record can label + * the reason consistently without pattern-matching on the concrete type. + * + * @property key Stable machine label for audit/grouping (e.g. `"stale"`, `"duplicate"`). + */ +sealed interface MarkReason { + + val key: String + + /** + * The proposition's decayed utility dropped below the staleness threshold. + */ + data object Stale : MarkReason { + override val key: String = "stale" + } + + /** + * The proposition duplicates another, surviving proposition. + * + * @property survivorId ID of the proposition that should be kept. + */ + data class Duplicate(val survivorId: String) : MarkReason { + override val key: String = "duplicate" + } + + /** + * A consumer-defined reason, carrying its own machine key and human description. + * + * @property key Stable machine label supplied by the consumer. + * @property description Human-readable explanation of the reason. + */ + data class Custom(override val key: String, val description: String) : MarkReason +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/PropositionMark.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/PropositionMark.kt new file mode 100644 index 00000000..38ee289f --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/PropositionMark.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import java.time.Instant + +/** + * An immutable record that a [CollectorStrategy] has marked a proposition for collection. + * + * A mark is purely descriptive — producing one never mutates the repository. The sweep + * phase ([SweepPolicy]) decides what, if anything, to do about a marked proposition. + * + * @property propositionId ID of the marked proposition; must not be blank. + * @property reason Why the proposition was marked. + * @property strategyName Name of the strategy that produced the mark (for audit/grouping). + * @property at When the mark was created. + */ +data class PropositionMark @JvmOverloads constructor( + val propositionId: String, + val reason: MarkReason, + val strategyName: String, + val at: Instant = Instant.now(), +) { + + init { + require(propositionId.isNotBlank()) { "propositionId must not be blank" } + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepAction.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepAction.kt new file mode 100644 index 00000000..5481ee2c --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepAction.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.dice.proposition.PropositionStatus + +/** + * What the sweep phase should do with a marked proposition, as decided by a [SweepPolicy]. + * + * A sealed family so the runner can exhaustively dispatch over the recognized outcomes. + */ +sealed interface SweepAction { + + /** + * Soft, recoverable retirement: transition the proposition to [newStatus] + * (e.g. [PropositionStatus.STALE]). This is the non-destructive default outcome. + * + * @property newStatus The lifecycle status to transition the proposition to. + */ + data class TransitionStatus(val newStatus: PropositionStatus) : SweepAction + + /** + * Permanently remove the proposition. This is destructive and unrecoverable, so it is + * never the default — a policy must opt in explicitly, and the default + * [StatusTransitionSweepPolicy] never returns it. + */ + data object HardDelete : SweepAction + + /** Leave the proposition untouched (e.g. it is pinned, or carries no marks). */ + data object Skip : SweepAction +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepPolicy.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepPolicy.kt new file mode 100644 index 00000000..2c88a5fc --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepPolicy.kt @@ -0,0 +1,64 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus + +/** + * The "sweep" half of the mark-and-sweep collector: given a proposition and its marks, decides + * what should actually happen to it as a [SweepAction]. + * + * Implementations decide per-proposition and should be stateless. Applying the decision + * (transitioning status, deleting, or skipping) is the runner's job, not the policy's. + * + * Implement this to encode domain-specific sweep behavior without touching DICE core. + */ +fun interface SweepPolicy { + + /** + * Decide what to do with [proposition] given its [marks]. + * + * @param proposition The marked proposition under consideration. + * @param marks The marks produced for this proposition; may be empty. + * @return The action the sweep should take. + */ + fun decide(proposition: Proposition, marks: List): SweepAction +} + +/** + * Default [SweepPolicy]: non-destructive and safe for pinned propositions. + * + * Decision order: + * 1. Pinned propositions are always skipped, no matter what marks they carry. + * 2. A proposition with no marks is skipped. + * 3. Everything else transitions to [targetStatus]. + * + * Never returns [SweepAction.HardDelete] — the default outcome is STALE, which is reversible. + * + * @property targetStatus The status a marked, unpinned proposition transitions to. + * Defaults to [PropositionStatus.STALE]. + */ +class StatusTransitionSweepPolicy @JvmOverloads constructor( + private val targetStatus: PropositionStatus = PropositionStatus.STALE, +) : SweepPolicy { + + override fun decide(proposition: Proposition, marks: List): SweepAction = when { + proposition.pinned -> SweepAction.Skip + marks.isEmpty() -> SweepAction.Skip + else -> SweepAction.TransitionStatus(targetStatus) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/lineage/CollectorRecordTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/lineage/CollectorRecordTest.kt new file mode 100644 index 00000000..723c5510 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/projection/lineage/CollectorRecordTest.kt @@ -0,0 +1,154 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.lineage + +import com.embabel.dice.projection.memory.MarkReason +import com.embabel.dice.proposition.PropositionStatus +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +class CollectorRecordTest { + + @Test + fun `CollectorRun constructs with finishedAt null and dryRun false by default`() { + val startedAt = Instant.now() + val run = CollectorRun("r1", startedAt) + + assertEquals("r1", run.runId) + assertEquals(startedAt, run.startedAt) + assertNull(run.finishedAt) + assertEquals(false, run.dryRun) + } + + @Test + fun `CollectorRun finished returns a copy with finishedAt set`() { + val startedAt = Instant.now() + val run = CollectorRun("r1", startedAt) + val finishedInstant = startedAt.plusSeconds(5) + + val finished = run.finished(finishedInstant) + + assertEquals(finishedInstant, finished.finishedAt) + assertNull(run.finishedAt) // original unchanged + } + + @Test + fun `CollectorRun with blank runId throws`() { + assertThrows(IllegalArgumentException::class.java) { + CollectorRun(" ", Instant.now()) + } + } + + @Test + fun `CollectorRun with finishedAt before startedAt throws`() { + val startedAt = Instant.now() + assertThrows(IllegalArgumentException::class.java) { + CollectorRun("r1", startedAt, finishedAt = startedAt.minusSeconds(1)) + } + } + + @Test + fun `CollectorRun carries the dryRun flag`() { + val run = CollectorRun("r1", Instant.now(), dryRun = true) + assertTrue(run.dryRun) + } + + @Test + fun `CollectorOutcome has exactly MARKED TRANSITIONED HARD_DELETED SKIPPED`() { + assertEquals( + listOf("MARKED", "TRANSITIONED", "HARD_DELETED", "SKIPPED"), + CollectorOutcome.entries.map { it.name }, + ) + } + + @Test + fun `CollectorRecord constructs and exposes a typed MarkReason`() { + val record = CollectorRecord( + propositionId = "p1", + reason = MarkReason.Stale, + outcome = CollectorOutcome.TRANSITIONED, + strategyName = "decay", + runId = "r1", + previousStatus = PropositionStatus.ACTIVE, + newStatus = PropositionStatus.STALE, + ) + + val reason: MarkReason = record.reason + assertEquals(MarkReason.Stale, reason) + assertEquals(CollectorOutcome.TRANSITIONED, record.outcome) + assertEquals(PropositionStatus.ACTIVE, record.previousStatus) + assertEquals(PropositionStatus.STALE, record.newStatus) + } + + @Test + fun `CollectorRecord carries a Duplicate reason with survivorId`() { + val record = CollectorRecord( + propositionId = "p1", + reason = MarkReason.Duplicate(survivorId = "p2"), + outcome = CollectorOutcome.MARKED, + strategyName = "duplicate", + runId = "r1", + ) + + val reason = record.reason + assertTrue(reason is MarkReason.Duplicate) + assertEquals("p2", (reason as MarkReason.Duplicate).survivorId) + } + + @Test + fun `CollectorRecord of factory mirrors the constructor`() { + val record = CollectorRecord.of( + propositionId = "p1", + reason = MarkReason.Stale, + outcome = CollectorOutcome.SKIPPED, + strategyName = "decay", + runId = "r1", + ) + + assertEquals("p1", record.propositionId) + assertEquals(CollectorOutcome.SKIPPED, record.outcome) + } + + @Test + fun `CollectorRecord with blank propositionId throws`() { + assertThrows(IllegalArgumentException::class.java) { + CollectorRecord( + propositionId = " ", + reason = MarkReason.Stale, + outcome = CollectorOutcome.MARKED, + strategyName = "decay", + runId = "r1", + ) + } + } + + @Test + fun `CollectorRecord with blank runId throws`() { + assertThrows(IllegalArgumentException::class.java) { + CollectorRecord( + propositionId = "p1", + reason = MarkReason.Stale, + outcome = CollectorOutcome.MARKED, + strategyName = "decay", + runId = " ", + ) + } + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStoreTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStoreTest.kt new file mode 100644 index 00000000..5ecce1ab --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStoreTest.kt @@ -0,0 +1,89 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.lineage + +import com.embabel.dice.projection.memory.MarkReason +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class InMemoryCollectorRecordStoreTest { + + private fun record( + propositionId: String, + runId: String, + reason: MarkReason = MarkReason.Stale, + outcome: CollectorOutcome = CollectorOutcome.MARKED, + ) = CollectorRecord( + propositionId = propositionId, + reason = reason, + outcome = outcome, + strategyName = "decay", + runId = runId, + ) + + @Test + fun `store starts empty`() { + val store = InMemoryCollectorRecordStore() + assertTrue(store.all().isEmpty()) + } + + @Test + fun `record appends and all returns records in insertion order`() { + val store = InMemoryCollectorRecordStore() + val first = record("p1", "r1") + val second = record("p2", "r1") + + store.record(first) + store.record(second) + + assertEquals(listOf(first, second), store.all()) + } + + @Test + fun `findByProposition returns only records with that propositionId`() { + val store = InMemoryCollectorRecordStore() + val p1a = record("p1", "r1") + val p2 = record("p2", "r1") + val p1b = record("p1", "r2") + store.record(p1a) + store.record(p2) + store.record(p1b) + + assertEquals(listOf(p1a, p1b), store.findByProposition("p1")) + } + + @Test + fun `findByRun returns only records with that runId`() { + val store = InMemoryCollectorRecordStore() + val r1a = record("p1", "r1") + val r2 = record("p2", "r2") + val r1b = record("p3", "r1") + store.record(r1a) + store.record(r2) + store.record(r1b) + + assertEquals(listOf(r1a, r1b), store.findByRun("r1")) + } + + @Test + fun `findByProposition returns empty for unknown id`() { + val store = InMemoryCollectorRecordStore() + store.record(record("p1", "r1")) + + assertTrue(store.findByProposition("nope").isEmpty()) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorRunnerBuilderTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorRunnerBuilderTest.kt new file mode 100644 index 00000000..f2dc6b30 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorRunnerBuilderTest.kt @@ -0,0 +1,90 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.dice.proposition.PropositionRepository +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +/** + * Oracles for the [CollectorRunner] SPI surface: the immutable [CollectorRunResult] shape and the + * fluent [CollectorRunner.withRepository] builder with its documented defaults. + */ +class CollectorRunnerBuilderTest { + + private val repository: PropositionRepository = mockk(relaxed = true) + + @Test + fun `CollectorRunResult is an immutable data class with the documented fields`() { + val started = Instant.now() + val finished = started.plusSeconds(1) + val result = CollectorRunResult( + runId = "run-1", + dryRun = true, + marks = emptyList(), + applied = emptyList(), + skipped = emptyList(), + hardDeleted = emptyList(), + startedAt = started, + finishedAt = finished, + ) + + assertEquals("run-1", result.runId) + assertTrue(result.dryRun) + assertEquals(started, result.startedAt) + assertEquals(finished, result.finishedAt) + // copy() proves data-class immutability semantics + assertFalse(result.copy(dryRun = false).dryRun) + } + + @Test + fun `CollectorRunResult finishedAt defaults to now`() { + val before = Instant.now() + val result = CollectorRunResult( + runId = "run-2", + dryRun = false, + marks = emptyList(), + applied = emptyList(), + skipped = emptyList(), + hardDeleted = emptyList(), + startedAt = before, + ) + assertTrue(!result.finishedAt.isBefore(before)) + } + + @Test + fun `withRepository builds a DefaultCollectorRunner with documented defaults`() { + val runner = CollectorRunner.withRepository(repository).build() + assertInstanceOf(DefaultCollectorRunner::class.java, runner) + } + + @Test + fun `builder with-chain returns the builder for fluent composition`() { + val runner = CollectorRunner + .withRepository(repository) + .withStrategy(DecayCollectorStrategy(retireBelow = 0.3)) + .withPolicy(StatusTransitionSweepPolicy()) + .withRecordStore(com.embabel.dice.projection.lineage.InMemoryCollectorRecordStore()) + .withEventListener(com.embabel.dice.common.DiceEventListener.DEV_NULL) + .build() + assertInstanceOf(DefaultCollectorRunner::class.java, runner) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorStrategyTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorStrategyTest.kt new file mode 100644 index 00000000..ef2fbcd5 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorStrategyTest.kt @@ -0,0 +1,95 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionRepository +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +class CollectorStrategyTest { + + private val contextId = ContextId("test-context") + private fun proposition(text: String = "p"): Proposition = + Proposition( + contextId = contextId, + text = text, + mentions = emptyList(), + confidence = 0.8, + decay = 0.1, + ) + + @Test + fun `MarkReason cases expose stable machine keys`() { + assertEquals("stale", MarkReason.Stale.key) + assertEquals("duplicate", MarkReason.Duplicate("survivor-1").key) + assertEquals("survivor-1", MarkReason.Duplicate("survivor-1").survivorId) + val custom = MarkReason.Custom("k", "desc") + assertEquals("k", custom.key) + assertEquals("desc", custom.description) + } + + @Test + fun `MarkReason is a sealed family usable in exhaustive when`() { + val reasons: List = listOf( + MarkReason.Stale, + MarkReason.Duplicate("s"), + MarkReason.Custom("c", "d"), + ) + reasons.forEach { reason -> + val label: String = when (reason) { + is MarkReason.Stale -> reason.key + is MarkReason.Duplicate -> reason.key + is MarkReason.Custom -> reason.key + } + assertTrue(label.isNotBlank()) + } + } + + @Test + fun `PropositionMark defaults at to now and carries fields`() { + val before = Instant.now() + val mark = PropositionMark("p1", MarkReason.Stale, "decay") + assertEquals("p1", mark.propositionId) + assertEquals(MarkReason.Stale, mark.reason) + assertEquals("decay", mark.strategyName) + assertTrue(!mark.at.isBefore(before)) + } + + @Test + fun `PropositionMark rejects blank propositionId`() { + assertThrows(IllegalArgumentException::class.java) { + PropositionMark(" ", MarkReason.Stale, "decay") + } + } + + @Test + fun `CollectorStrategy mark returns marks for candidates`() { + val repository = mockk(relaxed = true) + val strategy = CollectorStrategy { candidates, _, _ -> + candidates.map { PropositionMark(it.id, MarkReason.Stale, "test") } + } + val candidates = listOf(proposition("a"), proposition("b")) + val marks = strategy.mark(candidates, repository, contextId) + assertEquals(2, marks.size) + assertTrue(marks.all { it.reason == MarkReason.Stale }) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategyTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategyTest.kt new file mode 100644 index 00000000..5e229209 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategyTest.kt @@ -0,0 +1,110 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionRepository +import io.mockk.mockk +import io.mockk.verify +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +class DecayCollectorStrategyTest { + + private val contextId = ContextId("test-context") + + private fun proposition( + text: String = "p", + confidence: Double, + decay: Double = 0.0, + ): Proposition = + Proposition( + contextId = contextId, + text = text, + mentions = emptyList(), + confidence = confidence, + decay = decay, + ) + + @Test + fun `marks candidates below the threshold with Stale and strategyName decay`() { + val low = proposition("low", confidence = 0.2) + val high = proposition("high", confidence = 0.95) + val repository = mockk(relaxed = true) + + val marks = DecayCollectorStrategy(retireBelow = 0.5) + .mark(listOf(low, high), repository, contextId) + + assertEquals(1, marks.size) + assertEquals(low.id, marks[0].propositionId) + assertEquals(MarkReason.Stale, marks[0].reason) + assertEquals("decay", marks[0].strategyName) + } + + @Test + fun `candidates at or above the threshold produce no marks`() { + val high = proposition("high", confidence = 0.95) + val repository = mockk(relaxed = true) + + val marks = DecayCollectorStrategy(retireBelow = 0.5) + .mark(listOf(high), repository, contextId) + + assertTrue(marks.isEmpty()) + } + + @Test + fun `performs no repository writes`() { + val low = proposition("low", confidence = 0.1) + val repository = mockk(relaxed = true) + + DecayCollectorStrategy(retireBelow = 0.5).mark(listOf(low), repository, contextId) + + verify(exactly = 0) { repository.save(any()) } + verify(exactly = 0) { repository.delete(any()) } + } + + @Test + fun `filters on effectiveConfidence so decay-free candidates are judged by raw confidence`() { + // Freshly created propositions have age 0, so effectiveConfidence == confidence + // regardless of k. This pins the filter to effectiveConfidence rather than a + // raw field comparison, for both the default and a custom retireDecayK. + val justBelow = proposition("below", confidence = 0.49) + val justAbove = proposition("above", confidence = 0.51) + val repository = mockk(relaxed = true) + + val withDefaultK = DecayCollectorStrategy(retireBelow = 0.5) + .mark(listOf(justBelow, justAbove), repository, contextId) + val withCustomK = DecayCollectorStrategy(retireBelow = 0.5, retireDecayK = 5.0) + .mark(listOf(justBelow, justAbove), repository, contextId) + + assertEquals(listOf(justBelow.id), withDefaultK.map { it.propositionId }) + assertEquals(listOf(justBelow.id), withCustomK.map { it.propositionId }) + } + + @Test + fun `mark at defaults to roughly now`() { + val low = proposition("low", confidence = 0.1) + val repository = mockk(relaxed = true) + val before = Instant.now() + + val marks = DecayCollectorStrategy(retireBelow = 0.5).mark(listOf(low), repository, contextId) + + assertTrue(!marks[0].at.isBefore(before)) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt new file mode 100644 index 00000000..0d147258 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt @@ -0,0 +1,323 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.common.PropositionStatusChanged +import com.embabel.dice.common.RecordingDiceEventListener +import com.embabel.dice.projection.lineage.CollectorRecordStore +import com.embabel.dice.projection.lineage.InMemoryCollectorRecordStore +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionQuery +import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.proposition.PropositionStatus +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import io.mockk.verify +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.time.Instant +import java.time.temporal.ChronoUnit + +/** + * Behavior oracles for the dry-run-first [CollectorRunner] (its default implementation, + * `DefaultCollectorRunner`). These encode the load-bearing runner contracts: + * + * - `collect()` is a pure mark phase: zero repository writes AND zero record-store writes; + * - `run(dryRun = true)` performs no repository status change but persists an auditable + * run record (a [com.embabel.dice.projection.lineage.CollectorRun] with `dryRun == true` + * plus one record per marked proposition) and emits no status-change event; + * - `run(dryRun = false)` transitions each swept proposition to STALE via the normal + * status-transition path, persists then emits a [PropositionStatusChanged] per applied + * transition, and persists a finished run with records; + * - a pinned proposition that would otherwise be marked is skipped, not transitioned; + * - a second `run(dryRun = false)` immediately after the first applies zero transitions + * (ACTIVE-only candidate selection means already-STALE props are not re-selected); + * - the candidate query never selects PROMOTED propositions. + */ +class DefaultCollectorRunnerTest { + + private val contextId = ContextId("test-context") + private lateinit var repository: PropositionRepository + private lateinit var recordStore: CollectorRecordStore + private lateinit var listener: RecordingDiceEventListener + + private fun proposition( + text: String, + confidence: Double = 0.9, + decay: Double = 0.1, + status: PropositionStatus = PropositionStatus.ACTIVE, + pinned: Boolean = false, + contentRevised: Instant = Instant.now(), + ): Proposition = Proposition( + contextId = contextId, + text = text, + mentions = emptyList(), + confidence = confidence, + decay = decay, + status = status, + pinned = pinned, + contentRevised = contentRevised, + metadataRevised = contentRevised, + ) + + private val stalePast: Instant = Instant.now().minus(365, ChronoUnit.DAYS) + + private fun decayedProp(text: String, pinned: Boolean = false): Proposition = + proposition(text, confidence = 0.5, decay = 0.5, pinned = pinned, contentRevised = stalePast) + + private fun runner(strategy: CollectorStrategy = DecayCollectorStrategy(retireBelow = 0.3)): CollectorRunner = + CollectorRunner + .withRepository(repository) + .withStrategy(strategy) + .withRecordStore(recordStore) + .withEventListener(listener) + .build() + + @BeforeEach + fun setup() { + repository = mockk(relaxed = true) + recordStore = InMemoryCollectorRecordStore() + listener = RecordingDiceEventListener() + every { repository.query(any()) } returns emptyList() + } + + @Test + fun `collect performs no repository and no record-store writes`() { + val decayed = decayedProp("old fact") + every { repository.query(any()) } returns listOf(decayed) + + val result = runner().collect(contextId) + + assertTrue(result.marks.isNotEmpty()) + // Nothing is persisted on the pure-read path, so the runId is blank (not queryable). + assertTrue(result.runId.isBlank()) + verify(exactly = 0) { repository.save(any()) } + verify(exactly = 0) { repository.saveAll(any()) } + verify(exactly = 0) { repository.delete(any()) } + assertTrue(recordStore.all().isEmpty()) + assertTrue(listener.eventsOfType().isEmpty()) + } + + @Test + fun `dry run persists a run record only and performs no repository write or emit`() { + val decayed = decayedProp("old fact") + every { repository.query(any()) } returns listOf(decayed) + + val result = runner().run(contextId, dryRun = true) + + assertTrue(result.dryRun) + verify(exactly = 0) { repository.save(any()) } + verify(exactly = 0) { repository.delete(any()) } + // One auditable record persisted for the marked proposition. + assertTrue(recordStore.findByRun(result.runId).isNotEmpty()) + assertTrue(recordStore.findByProposition(decayed.id).isNotEmpty()) + // No lifecycle event emitted on a dry run. + assertTrue(listener.eventsOfType().isEmpty()) + } + + @Test + fun `live run transitions to STALE and emits a status change per applied transition`() { + val decayed = decayedProp("old fact") + every { repository.query(any()) } returns listOf(decayed) + val saved = slot() + every { repository.save(capture(saved)) } answers { saved.captured } + + val result = runner().run(contextId, dryRun = false) + + assertEquals(1, result.applied.size) + assertEquals(PropositionStatus.STALE, saved.captured.status) + verify(exactly = 1) { repository.save(any()) } + + val events = listener.eventsOfType() + assertEquals(1, events.size) + assertEquals(PropositionStatus.ACTIVE, events[0].previousStatus) + assertEquals(PropositionStatus.STALE, events[0].newStatus) + // Persisted run + record for the applied transition. + assertTrue(recordStore.findByRun(result.runId).isNotEmpty()) + } + + @Test + fun `live run skips a pinned proposition`() { + val pinned = decayedProp("pinned old fact", pinned = true) + every { repository.query(any()) } returns listOf(pinned) + + val result = runner().run(contextId, dryRun = false) + + assertTrue(result.applied.isEmpty()) + assertTrue(result.skipped.isNotEmpty()) + verify(exactly = 0) { repository.save(any()) } + verify(exactly = 0) { repository.delete(any()) } + assertTrue(listener.eventsOfType().isEmpty()) + } + + @Test + fun `a second live run applies zero transitions`() { + val decayed = decayedProp("old fact") + // First run sees the ACTIVE candidate; second run sees none + // (ACTIVE-only selection means the now-STALE proposition is not re-selected). + every { repository.query(any()) } returnsMany listOf(listOf(decayed), emptyList()) + every { repository.save(any()) } answers { firstArg() } + + val first = runner().run(contextId, dryRun = false) + val second = runner().run(contextId, dryRun = false) + + assertEquals(1, first.applied.size) + assertTrue(second.applied.isEmpty()) + } + + @Test + fun `dry run persists a retrievable run header flagged dryRun`() { + val decayed = decayedProp("old fact") + every { repository.query(any()) } returns listOf(decayed) + + val result = runner().run(contextId, dryRun = true) + + val run = recordStore.findRun(result.runId) + assertEquals(result.runId, run?.runId) + assertEquals(true, run?.dryRun) + // A dry run reports nothing as applied or hard-deleted (marks reflect the preview). + assertTrue(result.applied.isEmpty()) + assertTrue(result.hardDeleted.isEmpty()) + } + + @Test + fun `live run persists a retrievable run header flagged not dryRun`() { + val decayed = decayedProp("old fact") + every { repository.query(any()) } returns listOf(decayed) + every { repository.save(any()) } answers { firstArg() } + + val result = runner().run(contextId, dryRun = false) + + val run = recordStore.findRun(result.runId) + assertEquals(result.runId, run?.runId) + assertEquals(false, run?.dryRun) + } + + @Test + fun `a zero-mark run still leaves a retrievable run header`() { + // No candidates -> no marks -> no records, but the run must still leave a trace. + every { repository.query(any()) } returns emptyList() + + val result = runner().run(contextId, dryRun = false) + + assertTrue(result.marks.isEmpty()) + assertTrue(recordStore.findByRun(result.runId).isEmpty()) + val run = recordStore.findRun(result.runId) + assertEquals(result.runId, run?.runId) + assertEquals(1, recordStore.runs().size) + } + + @Test + fun `live run with a hard-delete policy removes the proposition and reports it as hard-deleted`() { + // The default StatusTransitionSweepPolicy never hard-deletes (recoverable STALE only), but a + // consumer-supplied policy may opt in to HardDelete. A live run must then actually delete the + // proposition, populate `hardDeleted` (and not `applied`), and emit no status-change event. + val decayed = decayedProp("old fact") + every { repository.query(any()) } returns listOf(decayed) + + val hardDeletePolicy = SweepPolicy { _, marks -> + if (marks.isEmpty()) SweepAction.Skip else SweepAction.HardDelete + } + val runner = CollectorRunner + .withRepository(repository) + .withStrategy(DecayCollectorStrategy(retireBelow = 0.3)) + .withPolicy(hardDeletePolicy) + .withRecordStore(recordStore) + .withEventListener(listener) + .build() + + val result = runner.run(contextId, dryRun = false) + + assertEquals(listOf(decayed.id), result.hardDeleted) + assertTrue(result.applied.isEmpty()) + verify(exactly = 1) { repository.delete(decayed.id) } + verify(exactly = 0) { repository.save(any()) } + assertTrue(listener.eventsOfType().isEmpty()) + } + + @Test + fun `dry run with a hard-delete policy deletes nothing but records the would-be hard delete`() { + // On a dry run a HardDelete decision must not touch the repository and must leave `hardDeleted` + // empty, yet still leave an auditable record so the preview is reviewable. + val decayed = decayedProp("old fact") + every { repository.query(any()) } returns listOf(decayed) + + val hardDeletePolicy = SweepPolicy { _, marks -> + if (marks.isEmpty()) SweepAction.Skip else SweepAction.HardDelete + } + val runner = CollectorRunner + .withRepository(repository) + .withStrategy(DecayCollectorStrategy(retireBelow = 0.3)) + .withPolicy(hardDeletePolicy) + .withRecordStore(recordStore) + .withEventListener(listener) + .build() + + val result = runner.run(contextId, dryRun = true) + + assertTrue(result.hardDeleted.isEmpty()) + verify(exactly = 0) { repository.delete(any()) } + assertTrue(recordStore.findByProposition(decayed.id).isNotEmpty()) + } + + @Test + fun `when two strategies mark the same proposition the emitted event carries both reasons combined`() { + // WR-03 regression guard: a proposition marked by more than one strategy must surface ALL its + // distinct reasons on the lifecycle event (sorted for run-to-run determinism), not just the + // reason of whichever strategy happened to run first. + val decayed = decayedProp("contested fact") + every { repository.query(any()) } returns listOf(decayed) + every { repository.save(any()) } answers { firstArg() } + + // Two independent strategies that each mark the same candidate with a different reason. + val staleStrategy = CollectorStrategy { candidates, _, _ -> + candidates.map { PropositionMark(it.id, MarkReason.Stale, "decay") } + } + val customStrategy = CollectorStrategy { candidates, _, _ -> + candidates.map { PropositionMark(it.id, MarkReason.Custom("audit", "flagged"), "audit") } + } + val runner = CollectorRunner + .withRepository(repository) + .withStrategy(staleStrategy) + .withStrategy(customStrategy) + .withRecordStore(recordStore) + .withEventListener(listener) + .build() + + runner.run(contextId, dryRun = false) + + val events = listener.eventsOfType() + assertEquals(1, events.size) + // Distinct keys, sorted: "audit" then "stale". + assertEquals("audit,stale", events[0].reason) + } + + @Test + fun `candidate query never selects PROMOTED propositions`() { + val querySlot = slot() + every { repository.query(capture(querySlot)) } returns emptyList() + + runner().run(contextId, dryRun = false) + + // The runner selects ACTIVE candidates only, so PROMOTED is excluded by construction. + assertEquals(setOf(PropositionStatus.ACTIVE), querySlot.captured.statuses) + assertTrue(PropositionStatus.PROMOTED !in querySlot.captured.statuses.orEmpty()) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategyTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategyTest.kt new file mode 100644 index 00000000..b4f38f60 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategyTest.kt @@ -0,0 +1,180 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.agent.rag.service.Cluster +import com.embabel.common.core.types.SimilarityResult +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionRepository +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class DuplicateCollectorStrategyTest { + + private val contextId = ContextId("test-context") + + private fun proposition( + text: String, + confidence: Double = 0.8, + reinforceCount: Int = 0, + ): Proposition = + Proposition( + contextId = contextId, + text = text, + mentions = emptyList(), + confidence = confidence, + decay = 0.0, + reinforceCount = reinforceCount, + ) + + private fun cluster(anchor: Proposition, vararg similar: Proposition): Cluster = + Cluster(anchor, similar.map { SimilarityResult.create(it, 0.95) }) + + @Test + fun `marks only the weaker member of a two-member cluster with the survivor id`() { + val strong = proposition("strong", confidence = 0.9) + val weak = proposition("weak", confidence = 0.3) + val repository = mockk(relaxed = true) + every { repository.findClusters(any(), any(), any()) } returns listOf(cluster(strong, weak)) + + val marks = DuplicateCollectorStrategy() + .mark(listOf(strong, weak), repository, contextId) + + assertEquals(1, marks.size) + assertEquals(weak.id, marks[0].propositionId) + assertEquals(MarkReason.Duplicate(strong.id), marks[0].reason) + assertEquals("duplicate", marks[0].strategyName) + } + + @Test + fun `never marks the survivor`() { + val strong = proposition("strong", confidence = 0.9) + val weak = proposition("weak", confidence = 0.3) + val repository = mockk(relaxed = true) + every { repository.findClusters(any(), any(), any()) } returns listOf(cluster(strong, weak)) + + val marks = DuplicateCollectorStrategy() + .mark(listOf(strong, weak), repository, contextId) + + assertFalse(marks.any { it.propositionId == strong.id }) + } + + @Test + fun `tie on effectiveConfidence is broken by reinforceCount`() { + val survivor = proposition("a", confidence = 0.6, reinforceCount = 5) + val loser = proposition("b", confidence = 0.6, reinforceCount = 1) + val repository = mockk(relaxed = true) + every { repository.findClusters(any(), any(), any()) } returns listOf(cluster(survivor, loser)) + + val marks = DuplicateCollectorStrategy() + .mark(listOf(survivor, loser), repository, contextId) + + assertEquals(1, marks.size) + assertEquals(loser.id, marks[0].propositionId) + assertEquals(MarkReason.Duplicate(survivor.id), marks[0].reason) + } + + @Test + fun `a single-member cluster produces no marks`() { + val only = proposition("only", confidence = 0.8) + val repository = mockk(relaxed = true) + every { repository.findClusters(any(), any(), any()) } returns + listOf(Cluster(only, emptyList())) + + val marks = DuplicateCollectorStrategy() + .mark(listOf(only), repository, contextId) + + assertTrue(marks.isEmpty()) + } + + @Test + fun `performs no repository writes`() { + val strong = proposition("strong", confidence = 0.9) + val weak = proposition("weak", confidence = 0.3) + val repository = mockk(relaxed = true) + every { repository.findClusters(any(), any(), any()) } returns listOf(cluster(strong, weak)) + + DuplicateCollectorStrategy().mark(listOf(strong, weak), repository, contextId) + + verify(exactly = 0) { repository.save(any()) } + verify(exactly = 0) { repository.delete(any()) } + } + + @Test + fun `ignores cluster members absent from the runner candidate snapshot`() { + val strong = proposition("strong", confidence = 0.9) + val weak = proposition("weak", confidence = 0.3) + val repository = mockk(relaxed = true) + every { repository.findClusters(any(), any(), any()) } returns listOf(cluster(strong, weak)) + + // candidates intentionally empty: members outside the swept candidate set are not marked, + // so every emitted mark maps to a runner-selected candidate. + val marks = DuplicateCollectorStrategy().mark(emptyList(), repository, contextId) + + assertTrue(marks.isEmpty()) + verify(exactly = 0) { repository.findById(any()) } + } + + @Test + fun `is idempotent and deterministic under overlapping clusters`() { + // Two overlapping clusters sharing member b: {a, b} and {b, c}. + // b is a non-survivor in one cluster and could be a survivor in the other; + // global component selection must pick a single survivor and mark each loser once. + val a = proposition("a", confidence = 0.5) + val b = proposition("b", confidence = 0.9) + val c = proposition("c", confidence = 0.7) + val candidates = listOf(a, b, c) + val repository = mockk(relaxed = true) + every { repository.findClusters(any(), any(), any()) } returns + listOf(cluster(a, b), cluster(b, c)) + + val first = DuplicateCollectorStrategy().mark(candidates, repository, contextId) + val second = DuplicateCollectorStrategy().mark(candidates, repository, contextId) + + // Identical marks across runs (deterministic, ordered by propositionId). + assertEquals(first.map { it.propositionId }, second.map { it.propositionId }) + // No proposition marked more than once. + assertEquals(first.size, first.distinctBy { it.propositionId }.size) + // b is the strongest in the {a,b,c} component, so it survives and is never marked. + assertFalse(first.any { it.propositionId == b.id }) + // a and c are the losers, both pointing at b as survivor. + assertEquals(setOf(a.id, c.id), first.map { it.propositionId }.toSet()) + assertTrue(first.all { it.reason == MarkReason.Duplicate(b.id) }) + } + + @Test + fun `full tie on confidence and reinforceCount is broken deterministically by id`() { + val one = proposition("one", confidence = 0.6, reinforceCount = 2) + val two = proposition("two", confidence = 0.6, reinforceCount = 2) + val repository = mockk(relaxed = true) + every { repository.findClusters(any(), any(), any()) } returns listOf(cluster(one, two)) + + val marks = DuplicateCollectorStrategy().mark(listOf(one, two), repository, contextId) + + // Survivor is the larger id; the smaller id is marked. Deterministic regardless of input order. + val expectedSurvivor = maxOf(one.id, two.id) + val expectedLoser = minOf(one.id, two.id) + assertEquals(1, marks.size) + assertEquals(expectedLoser, marks[0].propositionId) + assertEquals(MarkReason.Duplicate(expectedSurvivor), marks[0].reason) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/StatusTransitionSweepPolicyTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/StatusTransitionSweepPolicyTest.kt new file mode 100644 index 00000000..510619b4 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/StatusTransitionSweepPolicyTest.kt @@ -0,0 +1,85 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class StatusTransitionSweepPolicyTest { + + private val contextId = ContextId("test-context") + + private fun proposition(pinned: Boolean = false): Proposition = + Proposition( + contextId = contextId, + text = "p", + mentions = emptyList(), + confidence = 0.8, + decay = 0.1, + pinned = pinned, + ) + + private fun marks(p: Proposition) = listOf(PropositionMark(p.id, MarkReason.Stale, "decay")) + + @Test + fun `SweepAction is a sealed family`() { + val actions: List = listOf( + SweepAction.TransitionStatus(PropositionStatus.STALE), + SweepAction.HardDelete, + SweepAction.Skip, + ) + actions.forEach { action -> + val described: String = when (action) { + is SweepAction.TransitionStatus -> action.newStatus.name + is SweepAction.HardDelete -> "delete" + is SweepAction.Skip -> "skip" + } + assertEquals(true, described.isNotBlank()) + } + } + + @Test + fun `pinned proposition is skipped even when marked`() { + val p = proposition(pinned = true) + val action = StatusTransitionSweepPolicy().decide(p, marks(p)) + assertEquals(SweepAction.Skip, action) + } + + @Test + fun `empty marks is skipped`() { + val p = proposition() + val action = StatusTransitionSweepPolicy().decide(p, emptyList()) + assertEquals(SweepAction.Skip, action) + } + + @Test + fun `marked unpinned proposition transitions to STALE by default`() { + val p = proposition() + val action = StatusTransitionSweepPolicy().decide(p, marks(p)) + assertEquals(SweepAction.TransitionStatus(PropositionStatus.STALE), action) + } + + @Test + fun `custom target status is honored`() { + val p = proposition() + val action = StatusTransitionSweepPolicy(targetStatus = PropositionStatus.SUPERSEDED) + .decide(p, marks(p)) + assertEquals(SweepAction.TransitionStatus(PropositionStatus.SUPERSEDED), action) + } +} From 6ad03941a4158b5b9b54dc807fd68e0473be9b8d Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:57:31 -0400 Subject: [PATCH 03/27] feat(consolidation): dream-loop consolidation passes and maintenance orchestrator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds offline passes that tidy stored memory the way sleep consolidates the day, and an orchestrator that runs them. Opt-in — nothing runs by default. - ConsolidationPass with SessionConsolidationPass, AbstractionPass, ContradictionResolutionPass, and DecaySweepPass (which reuses the collector, so consolidation and collection share one mechanism) - DreamLoopOrchestrator runs the passes and emits a DreamLoopReport per run - MemoryMaintenanceOrchestrator ties the consolidation passes together with the collector pass; MemoryConsolidator reaches its final form here Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../consolidation/AbstractionPass.kt | 103 +++++++ .../consolidation/ConsolidationPass.kt | 111 ++++++++ .../ContradictionResolutionPass.kt | 92 ++++++ .../consolidation/DecaySweepPass.kt | 100 +++++++ .../consolidation/SessionConsolidationPass.kt | 62 ++++ .../memory/DefaultDreamLoopOrchestrator.kt | 175 ++++++++++++ .../DefaultMemoryMaintenanceOrchestrator.kt | 117 ++++++-- .../memory/DreamLoopOrchestrator.kt | 75 +++++ .../dice/projection/memory/DreamLoopReport.kt | 47 +++ .../memory/MemoryMaintenanceOrchestrator.kt | 6 +- .../consolidation/AbstractionPassTest.kt | 163 +++++++++++ .../ConsolidationPassResultTest.kt | 101 +++++++ .../ContradictionResolutionPassTest.kt | 122 ++++++++ .../consolidation/DecaySweepPassTest.kt | 193 +++++++++++++ .../SessionConsolidationPassTest.kt | 122 ++++++++ .../memory/DreamLoopOrchestratorTest.kt | 269 ++++++++++++++++++ .../MemoryMaintenanceOrchestratorTest.kt | 215 ++++++++++++++ 17 files changed, 2044 insertions(+), 29 deletions(-) create mode 100644 dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ConsolidationPass.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPass.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/operations/consolidation/SessionConsolidationPass.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestrator.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/projection/memory/DreamLoopReport.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/operations/consolidation/AbstractionPassTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ConsolidationPassResultTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPassTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/operations/consolidation/SessionConsolidationPassTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestratorTest.kt diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt new file mode 100644 index 00000000..8367c42f --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt @@ -0,0 +1,103 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.operations.consolidation + +import com.embabel.agent.core.ContextId +import com.embabel.dice.operations.PropositionGroup +import com.embabel.dice.operations.abstraction.PropositionAbstractor +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus + +/** + * Consolidation pass that synthesizes higher-level propositions from groups of related ground-level + * propositions by delegating to an injected [PropositionAbstractor]. + * + * This pass does NOT re-implement abstraction — it groups level-0 ACTIVE snapshot propositions by + * resolved entity, hands each qualifying group to the abstractor, and marks the consumed sources + * `SUPERSEDED`. It adds only pass-level concerns: snapshot filtering, an idempotency guard, and a + * `maxLevel` cap. + * + * ## Why the idempotency guard exists + * + * Without a guard, every cycle would re-abstract the same groups, producing fresh higher-level + * propositions whose own sources are then re-abstracted again — unbounded level inflation that never + * settles. The guard skips a group whenever an existing higher-level proposition already covers all of + * the group's member IDs (`sourceIds.containsAll(memberIds)`), so a re-run over an unchanged snapshot + * is a [ConsolidationPassResult.NoOp]. The `maxLevel` cap is a secondary safeguard that drops any + * abstraction whose resulting level exceeds the configured ceiling. + * + * @property abstractor The delegate that synthesizes higher-level propositions for a group. + * @property abstractionThreshold Minimum group size before a group is eligible for abstraction. + * @property abstractionTargetCount Desired number of abstractions per group, passed to the abstractor. + * @property maxLevel Highest abstraction level permitted; abstractions above this are discarded. + */ +class AbstractionPass @JvmOverloads constructor( + private val abstractor: PropositionAbstractor, + private val abstractionThreshold: Int = 5, + private val abstractionTargetCount: Int = 3, + private val maxLevel: Int = 3, +) : ConsolidationPass { + + override val name: String = "abstraction" + + override fun run(contextId: ContextId, propositions: List): ConsolidationPassResult { + return try { + val level0Active = propositions.filter { + it.status == PropositionStatus.ACTIVE && it.level == 0 + } + val existingAbstractions = propositions.filter { it.level > 0 } + + val groups = level0Active + .flatMap { prop -> prop.mentions.mapNotNull { it.resolvedId }.map { entityId -> entityId to prop } } + .groupBy({ it.first }, { it.second }) + .mapValues { (_, props) -> props.distinct() } + .filter { (_, props) -> props.size >= abstractionThreshold } + + val toSave = mutableListOf() + var skipped = 0 + var abstractedGroups = 0 + for ((entityId, props) in groups) { + val memberIds = props.map { it.id }.toSet() + // Idempotency guard: a group already covered by a higher-level proposition is not + // re-abstracted, preventing unbounded level inflation across cycles. + val alreadyCovered = existingAbstractions.any { it.sourceIds.toSet().containsAll(memberIds) } + if (alreadyCovered) { + skipped += props.size + continue + } + val abstractions = abstractor.abstract(PropositionGroup.of(entityId, props), abstractionTargetCount) + .filter { it.level <= maxLevel } + toSave += abstractions + // withStatus preserves the contentRevised decay anchor (only metadataRevised moves). + toSave += props.map { it.withStatus(PropositionStatus.SUPERSEDED) } + abstractedGroups++ + } + + if (toSave.isEmpty()) { + ConsolidationPassResult.NoOp(name, "no groups above threshold or all covered") + } else { + ConsolidationPassResult.Changed( + passName = name, + propositionsToSave = toSave, + skipped = skipped, + summary = "abstracted $abstractedGroups groups, $skipped propositions skipped (already covered)", + ) + } + } catch (e: Throwable) { + ConsolidationPassResult.Failed(name, e) + } + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ConsolidationPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ConsolidationPass.kt new file mode 100644 index 00000000..bf1b62c4 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ConsolidationPass.kt @@ -0,0 +1,111 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.operations.consolidation + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition + +/** + * One composable step in a consolidation cycle. A pass receives the current snapshot of + * propositions for a context, decides what should change, and reports it back as a + * [ConsolidationPassResult] — it never writes anything itself. + * + * ## Purity contract + * + * A pass works only with the snapshot it is handed and never touches a repository directly. + * The orchestrator owns all writes: it fetches the snapshot once, runs each pass over it, + * and applies the accumulated saves/deletes in a single write per cycle. + * + * Passes MUST be idempotent: running the same pass twice with no intervening write must produce + * [ConsolidationPassResult.NoOp] on the second run. The consolidation loop runs passes + * repeatedly until the snapshot settles, so a pass that keeps returning [ConsolidationPassResult.Changed] + * for an already-consolidated snapshot would loop forever. + * + * Implement this interface to add domain-specific consolidation behavior without touching DICE core. + */ +interface ConsolidationPass { + + /** Short, stable identifier used in reports and logging (e.g. `"abstraction"`). */ + val name: String + + /** + * Consolidate the snapshot [propositions] for [contextId], reporting the outcome. + * + * @param contextId The context the snapshot belongs to. + * @param propositions The pre-fetched snapshot to consolidate; the pass must not assume any + * ordering and must not reach outside this list for state. + * @return What changed (if anything), or why nothing did, as a [ConsolidationPassResult]. + */ + fun run(contextId: ContextId, propositions: List): ConsolidationPassResult +} + +/** + * What a [ConsolidationPass] decided to do (or not do). Sealed so the orchestrator can + * exhaustively handle every case without a catch-all. + * + * Every case carries the [passName] that produced it, so reports can attribute outcomes back + * to their source pass without threading the name through separately. + * + * @property passName The [ConsolidationPass.name] of the pass that produced this result. + */ +sealed class ConsolidationPassResult { + + abstract val passName: String + + /** + * The pass found changes to apply. + * + * @property passName The name of the pass that produced this result. + * @property propositionsToSave Propositions to upsert (new or updated copies). + * @property propositionsToDelete IDs to hard-delete. Only acted on when the orchestrator runs + * with `allowHardDelete`; otherwise ignored in favor of the soft-retire default, so a pass + * can always express a delete intent without risking data loss. + * @property skipped Count of snapshot propositions the pass deliberately left untouched. + * @property summary Short description of what changed, for reports and logging. + */ + data class Changed @JvmOverloads constructor( + override val passName: String, + val propositionsToSave: List = emptyList(), + val propositionsToDelete: List = emptyList(), + val skipped: Int = 0, + val summary: String = "", + ) : ConsolidationPassResult() + + /** + * The pass had nothing to do — the snapshot is already consolidated as far as this pass is + * concerned. Returning [NoOp] (not an empty [Changed]) is how a pass signals it has settled, + * which lets the consolidation loop detect convergence. + * + * @property passName The name of the pass that produced this result. + * @property reason Optional explanation of why nothing was done. + */ + data class NoOp( + override val passName: String, + val reason: String = "", + ) : ConsolidationPassResult() + + /** + * The pass threw. The orchestrator records the failure and continues with the remaining + * passes rather than aborting the whole cycle. + * + * @property passName The name of the pass that produced this result. + * @property cause The exception that caused the failure. + */ + data class Failed( + override val passName: String, + val cause: Throwable, + ) : ConsolidationPassResult() +} diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt new file mode 100644 index 00000000..6fde0638 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.operations.consolidation + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.proposition.revision.PropositionRelation +import com.embabel.dice.proposition.revision.PropositionReviser + +/** + * Consolidation pass that resolves contradictions among ACTIVE propositions by delegating + * classification to an injected [PropositionReviser] and retiring the losing side of each conflict. + * + * This pass does NOT re-implement contradiction detection. For each ACTIVE proposition it asks + * `reviser.classify()` to compare it against its entity-overlapping ACTIVE peers; for every pair the + * reviser reports as [PropositionRelation.CONTRADICTORY], the weaker member (lower + * [Proposition.effectiveConfidence]; ties favor the proposition being classified) is transitioned to + * [PropositionStatus.CONTRADICTED]. + * + * Classification uses [PropositionReviser.classify] — NOT a contraster — because resolving a + * contradiction is a lifecycle transition, not an articulation of differences. + * + * ## Idempotency + * + * Only ACTIVE propositions are compared and only an ACTIVE loser is transitioned, so once a conflict + * is resolved the retired proposition drops out of the candidate set and is never re-classified. A + * re-run over an already-resolved snapshot therefore yields [ConsolidationPassResult.NoOp]. Candidates + * are pruned to those sharing at least one resolved entity, bounding `classify()` fan-out. + * + * @property reviser The delegate that classifies the relation between propositions. + */ +class ContradictionResolutionPass @JvmOverloads constructor( + private val reviser: PropositionReviser, +) : ConsolidationPass { + + override val name: String = "contradiction-resolution" + + override fun run(contextId: ContextId, propositions: List): ConsolidationPassResult { + return try { + val active = propositions.filter { it.status == PropositionStatus.ACTIVE } + val toSave = mutableListOf() + for (p in active) { + val candidates = active.filter { it.id != p.id && sharesEntity(it, p) } + val classified = reviser.classify(p, candidates) + classified + .filter { it.relation == PropositionRelation.CONTRADICTORY } + .forEach { c -> + val weaker = + if (p.effectiveConfidence() <= c.proposition.effectiveConfidence()) p + else c.proposition + if (weaker.status == PropositionStatus.ACTIVE) { + // withStatus preserves the contentRevised decay anchor. + toSave += weaker.withStatus(PropositionStatus.CONTRADICTED) + } + } + } + val deduped = toSave.distinctBy { it.id } + if (deduped.isEmpty()) { + ConsolidationPassResult.NoOp(name, "no contradictions found") + } else { + ConsolidationPassResult.Changed( + passName = name, + propositionsToSave = deduped, + summary = "resolved ${deduped.size} contradictory propositions to CONTRADICTED", + ) + } + } catch (e: Throwable) { + ConsolidationPassResult.Failed(name, e) + } + } + + /** True when [a] and [b] share at least one resolved entity. */ + private fun sharesEntity(a: Proposition, b: Proposition): Boolean { + val aEntities = a.mentions.mapNotNull { it.resolvedId }.toSet() + if (aEntities.isEmpty()) return false + return b.mentions.any { it.resolvedId != null && it.resolvedId in aEntities } + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPass.kt new file mode 100644 index 00000000..58cf9449 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPass.kt @@ -0,0 +1,100 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.operations.consolidation + +import com.embabel.agent.core.ContextId +import com.embabel.dice.projection.memory.CollectorRunner +import com.embabel.dice.proposition.Proposition + +/** + * The decay pass in a consolidation cycle: a thin wrapper that delegates entirely to an injected + * [CollectorRunner] and translates its outcome into a [ConsolidationPassResult]. + * + * This pass owns no sweep logic — the runner handles candidate selection, pinned-proposition + * exemption, staleness evaluation, persistence, and lifecycle events. The pass just drives the + * runner and reports what happened. + * + * ## Soft STALE vs. hard-delete + * + * The behavior depends on the [SweepPolicy] the runner was built with. The default policy retires + * propositions softly to `STALE` and never hard-deletes, so a [DecaySweepPass] over a default + * runner is non-destructive. For hard-delete, build the runner with a policy that returns + * `SweepAction.HardDelete` for stale marks — this pass surfaces those deletes in its summary + * but never puts ids in [ConsolidationPassResult.Changed.propositionsToDelete]. + * + * ## Dual-threshold hysteresis + * + * Decay transitions use a two-threshold band to avoid status flapping: + * - [staleThresholdH] (default `0.1`): a proposition goes `STALE` only when effective confidence + * drops below H. + * - [recoveryThresholdS] (default `0.25`): the upper edge of the hysteresis band `[H, S)`. + * Revival out of `STALE` is reinforcement-driven (happens on ingest/reinforce), never + * sweep-driven. These thresholds are surfaced here for documentation and NoOp report text. + * + * ## Write boundary + * + * The [CollectorRunner] writes transitions itself inside [CollectorRunner.run]. To avoid a + * double-write, this pass is report-only: its [ConsolidationPassResult.Changed] always carries + * empty `propositionsToSave` and `propositionsToDelete`. The orchestrator gets nothing to + * re-persist from this pass. + * + * @property collectorRunner The mark-and-sweep runner this pass delegates to. + * @property staleThresholdH Lower hysteresis threshold (ACTIVE -> STALE below this). Defaults to `0.1`. + * @property recoveryThresholdS Upper hysteresis threshold; revival is reinforcement-driven, not + * sweep-driven. Defaults to `0.25`. + */ +class DecaySweepPass @JvmOverloads constructor( + private val collectorRunner: CollectorRunner, + private val staleThresholdH: Double = 0.1, + private val recoveryThresholdS: Double = 0.25, +) : ConsolidationPass { + + override val name: String = "decay-sweep" + + /** + * Runs the collector's full mark-and-sweep over [contextId] and reports the outcome. + * + * The [propositions] snapshot is not used here — the runner queries its own ACTIVE candidates + * from the repository it was built with. Returns [ConsolidationPassResult.NoOp] when nothing + * decayed below H, a report-only [ConsolidationPassResult.Changed] (empty save/delete lists — + * the runner already wrote) when transitions or hard-deletes occurred, or + * [ConsolidationPassResult.Failed] if the runner threw. + * + * @param contextId The context whose ACTIVE propositions the runner evaluates. + * @param propositions The orchestrator's snapshot — unused by this pass. + */ + override fun run(contextId: ContextId, propositions: List): ConsolidationPassResult { + return try { + val result = collectorRunner.run(contextId, dryRun = false) + if (result.applied.isEmpty() && result.hardDeleted.isEmpty()) { + ConsolidationPassResult.NoOp(name, "no propositions below H=$staleThresholdH (recovery band up to S=$recoveryThresholdS)") + } else { + ConsolidationPassResult.Changed( + passName = name, + propositionsToSave = emptyList(), + propositionsToDelete = emptyList(), + skipped = result.skipped.size, + summary = "decay sweep: ${result.applied.size} -> STALE, ${result.hardDeleted.size} hard-deleted", + ) + } + } catch (e: Throwable) { + // Match the catch breadth of the sibling passes (SessionConsolidationPass, + // AbstractionPass): isolate ANY Throwable from the delegate into a Failed result so one + // pass's failure never aborts the rest of the cycle. + ConsolidationPassResult.Failed(name, e) + } + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/SessionConsolidationPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/SessionConsolidationPass.kt new file mode 100644 index 00000000..760bd4dd --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/SessionConsolidationPass.kt @@ -0,0 +1,62 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.operations.consolidation + +import com.embabel.agent.core.ContextId +import com.embabel.dice.projection.memory.MemoryConsolidator +import com.embabel.dice.proposition.Proposition + +/** + * Consolidation pass that folds a session's propositions into long-term memory by delegating + * verbatim to an injected [MemoryConsolidator]. + * + * This pass does NOT re-implement promote/reinforce/merge logic — it hands the session and the + * snapshot (the existing long-term ACTIVE set) to the consolidator and projects the resulting + * [com.embabel.dice.projection.memory.ConsolidationResult] into a [ConsolidationPassResult]. + * + * The session propositions are supplied at construction (one pass instance per cycle) because the + * [run] snapshot is the long-term set the session is consolidated against, not the session itself. + * + * @property consolidator The delegate that decides what to promote, reinforce, merge, or discard. + * @property sessionPropositions The current session's propositions to consolidate into long-term memory. + */ +class SessionConsolidationPass @JvmOverloads constructor( + private val consolidator: MemoryConsolidator, + private val sessionPropositions: List = emptyList(), +) : ConsolidationPass { + + override val name: String = "session-consolidation" + + override fun run(contextId: ContextId, propositions: List): ConsolidationPassResult { + return try { + val result = consolidator.consolidate(sessionPropositions, propositions) + val toSave = result.promoted + result.reinforced + result.merged.map { it.result } + if (toSave.isEmpty()) { + ConsolidationPassResult.NoOp(name, "nothing to promote, reinforce, or merge") + } else { + ConsolidationPassResult.Changed( + passName = name, + propositionsToSave = toSave, + summary = "consolidated session: ${result.promoted.size} promoted, " + + "${result.reinforced.size} reinforced, ${result.merged.size} merged, " + + "${result.discarded.size} discarded", + ) + } + } catch (e: Throwable) { + ConsolidationPassResult.Failed(name, e) + } + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt new file mode 100644 index 00000000..50591951 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt @@ -0,0 +1,175 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.operations.consolidation.ConsolidationPass +import com.embabel.dice.operations.consolidation.ConsolidationPassResult +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionQuery +import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.proposition.PropositionStatus +import org.slf4j.LoggerFactory +import java.time.Instant +import java.util.concurrent.ConcurrentHashMap + +/** + * Default [DreamLoopOrchestrator]: runs a list of [ConsolidationPass]es as a single cycle. + * + * Each cycle fetches the ACTIVE snapshot once, runs every pass over it in registration order, + * then aggregates all the saves into a single [PropositionRepository.saveAll]. Hard deletes are + * off by default — they only happen when [allowHardDelete] is explicitly enabled. + * + * The [consolidate] trigger is best-effort: it tracks the ACTIVE count after each cycle and + * fires when the delta since the last cycle reaches [changeVolumeThreshold]. Use [consolidateNow] + * to bypass the threshold and run unconditionally. + * + * @param repository Storage backend; the orchestrator is the sole writer per cycle. + * @param passes Passes executed in registration order. + * @param allowHardDelete When false (default), delete-intents from passes are silently ignored. + * When true, each aggregated delete id is removed from the repository. + * @param changeVolumeThreshold Minimum active-count delta since the last cycle for [consolidate] to fire. + */ +data class DefaultDreamLoopOrchestrator( + private val repository: PropositionRepository, + private val passes: List = emptyList(), + private val allowHardDelete: Boolean = false, + private val changeVolumeThreshold: Int = 10, +) : DreamLoopOrchestrator { + + private val logger = LoggerFactory.getLogger(DefaultDreamLoopOrchestrator::class.java) + + /** + * Tracks the last-seen ACTIVE count per context, used by the change-volume trigger. + * + * Kept outside the primary constructor on purpose: it's mutable runtime state, not part of + * this orchestrator's value identity. That way [copy]-built instances each start with their + * own clean baseline (no shared state), and two orchestrators whose counters have diverged + * still compare as equal. Thread-safe via [ConcurrentHashMap]. + */ + private val lastActiveCount: ConcurrentHashMap = ConcurrentHashMap() + + override fun withPass(pass: ConsolidationPass): DefaultDreamLoopOrchestrator = + copy(passes = passes + pass) + + /** Enable or disable irreversible hard deletes (default disabled — soft STALE only). */ + fun withAllowHardDelete(allow: Boolean): DefaultDreamLoopOrchestrator = + copy(allowHardDelete = allow) + + /** Set the active-count delta required for the threshold-gated [consolidate] to run. */ + fun withChangeVolumeThreshold(threshold: Int): DefaultDreamLoopOrchestrator = + copy(changeVolumeThreshold = threshold) + + override fun consolidate(contextId: ContextId): DreamLoopReport? { + val currentActive = activeSnapshot(contextId).size + val delta = currentActive - (lastActiveCount[contextId] ?: 0) + if (delta < changeVolumeThreshold) { + // Not enough has changed to be worth a cycle; record the count and skip. + lastActiveCount[contextId] = currentActive + logger.debug( + "Skipping consolidation for {}: active-count delta {} < threshold {}", + contextId, delta, changeVolumeThreshold, + ) + return null + } + return consolidateNow(contextId) + } + + override fun consolidateNow(contextId: ContextId): DreamLoopReport { + val cycleStarted = Instant.now() + + // (1) Fetch the ACTIVE snapshot once. + val snapshot = activeSnapshot(contextId) + logger.debug("Running {} pass(es) over {} active proposition(s) for {}", passes.size, snapshot.size, contextId) + + // (2) Run each pass in registration order, isolating failures into Failed results. + val passResults = passes.map { pass -> runPass(pass, contextId, snapshot) } + + // (3) Aggregate saves into a single write. + val changed = passResults.filterIsInstance() + val toSave: List = changed.flatMap { it.propositionsToSave } + if (toSave.isNotEmpty()) { + repository.saveAll(toSave) + } + + // (4) Apply deletes only when explicitly opted in. + if (allowHardDelete) { + changed.flatMap { it.propositionsToDelete }.forEach { repository.delete(it) } + } + + // (5) Record the post-cycle ACTIVE count for the change-volume trigger. + lastActiveCount[contextId] = activeSnapshot(contextId).size + + // (6) Build the report. + val transitioned = changed.sumOf { it.propositionsToSave.size + it.propositionsToDelete.size } + val newPropositions = toSave.size + return DreamLoopReport( + contextId = contextId, + cycleStarted = cycleStarted, + passResults = passResults, + totalExamined = snapshot.size, + totalTransitioned = transitioned, + totalNewPropositions = newPropositions, + triggered = true, + ) + } + + /** + * Satisfies the [MemoryMaintenanceOrchestrator] contract by running a full cycle, but the + * result is always empty in the legacy-shaped [MaintenanceResult] — the real output is a + * [DreamLoopReport] from [consolidateNow]. For the full consolidation/abstraction/retire + * pipeline, use [DefaultMemoryMaintenanceOrchestrator] instead. + */ + override fun maintain( + contextId: ContextId, + sessionPropositions: List, + ): MaintenanceResult { + consolidateNow(contextId) + return MaintenanceResult( + consolidation = null, + abstractions = emptyList(), + superseded = emptyList(), + retired = emptyList(), + ) + } + + private fun runPass( + pass: ConsolidationPass, + contextId: ContextId, + snapshot: List, + ): ConsolidationPassResult = + try { + pass.run(contextId, snapshot) + } catch (e: Throwable) { + // Catch breadth matches the passes themselves (all catch Throwable): a pass that lets a + // non-Exception Throwable escape is still isolated into a Failed result here, honoring the + // "a failed pass does not abort the cycle" guarantee for every error class. + logger.warn("Pass '{}' failed; recording failure and continuing cycle", pass.name, e) + ConsolidationPassResult.Failed(pass.name, e) + } + + private fun activeSnapshot(contextId: ContextId): List = + repository.query( + PropositionQuery.forContextId(contextId).withStatus(PropositionStatus.ACTIVE) + ) + + companion object { + /** Start building a dream-loop orchestrator with the given repository. */ + @JvmStatic + fun withRepository(repository: PropositionRepository): DefaultDreamLoopOrchestrator = + DefaultDreamLoopOrchestrator(repository = repository) + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt index 81116271..b259f769 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt @@ -16,23 +16,40 @@ package com.embabel.dice.projection.memory import com.embabel.agent.core.ContextId -import com.embabel.dice.operations.PropositionGroup import com.embabel.dice.operations.abstraction.PropositionAbstractor +import com.embabel.dice.operations.consolidation.AbstractionPass +import com.embabel.dice.operations.consolidation.ConsolidationPassResult +import com.embabel.dice.operations.consolidation.SessionConsolidationPass import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionQuery import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.proposition.PropositionStatus /** - * Default implementation of [MemoryMaintenanceOrchestrator]. + * Default [MemoryMaintenanceOrchestrator]: runs consolidation, abstraction, retirement, and + * optionally the mark-and-sweep collector as a four-phase pipeline. * - * @param repository Storage backend for propositions - * @param consolidator Consolidation strategy for session → long-term promotion - * @param abstractor Optional abstractor for synthesizing higher-level insights - * @param abstractionThreshold Minimum propositions per entity group to trigger abstraction - * @param abstractionTargetCount How many abstractions to generate per group - * @param retireBelow Effective confidence threshold for retirement; null disables retirement - * @param retireDecayK Decay rate multiplier for retirement calculation + * The consolidation and abstraction logic delegates to [SessionConsolidationPass] and + * [AbstractionPass] respectively — they own the details of what changes. This orchestrator + * just drives the sequence and persists results in the legacy per-step call shape so + * [maintain]'s observable contract stays intact. + * + * A few intentional divergences from the dream-loop path worth knowing: + * - Every qualifying entity group is re-abstracted on every [maintain] call with no level + * ceiling. The dream-loop's skip-covered idempotency guard has nothing to match here because + * [maintain] only ever sees the level-0 ACTIVE snapshot. + * - The decay/retirement step **hard-deletes** propositions below [retireBelow], unlike the + * dream-loop which soft-transitions them to STALE by default. This preserves the original + * behavior for backward compatibility. + * + * @param repository Storage backend for propositions. + * @param consolidator Decides how session propositions are folded into long-term memory. + * @param abstractor Optional; synthesizes higher-level insights from entity groups. + * @param abstractionThreshold Minimum propositions per entity group before abstraction runs. + * @param abstractionTargetCount How many abstractions to generate per group. + * @param retireBelow Hard-delete propositions whose effective confidence falls below this; null disables retirement. + * @param retireDecayK Decay-rate multiplier used when computing effective confidence for retirement. + * @param collector Optional mark-and-sweep collector run as the final phase; null disables it. */ data class DefaultMemoryMaintenanceOrchestrator( private val repository: PropositionRepository, @@ -42,26 +59,32 @@ data class DefaultMemoryMaintenanceOrchestrator( private val abstractionTargetCount: Int = 3, private val retireBelow: Double? = null, private val retireDecayK: Double = 2.0, + private val collector: CollectorRunner? = null, ) : MemoryMaintenanceOrchestrator { override fun maintain( contextId: ContextId, sessionPropositions: List, ): MaintenanceResult { - // Phase 1: Consolidate session propositions + // Phase 1: Consolidate session propositions (single source of truth: SessionConsolidationPass) val consolidation = consolidate(contextId, sessionPropositions) - // Phase 2: Abstract entity groups + // Phase 2: Abstract entity groups (single source of truth: AbstractionPass) val (abstractions, superseded) = abstract(contextId) - // Phase 3: Retire expired propositions + // Phase 3: Retire expired propositions (legacy hard delete; see class KDoc) val retired = retire(contextId) + // Phase 4 (optional): run the mark-and-sweep collector when one is configured. + // Default off; when null the behavior is identical to the three-phase pipeline. + val collectorResult = collector?.run(contextId) + return MaintenanceResult( consolidation = consolidation, abstractions = abstractions, superseded = superseded, retired = retired, + collectorResult = collectorResult, ) } @@ -76,9 +99,17 @@ data class DefaultMemoryMaintenanceOrchestrator( .withStatus(PropositionStatus.ACTIVE) ) + // SessionConsolidationPass is the single source of truth for *how* a session is folded into + // long-term memory: it delegates verbatim to the injected consolidator (the identical + // delegate the dream loop uses), adding no promote/reinforce/merge logic of its own. Because + // that delegation is exactly one consolidate() call, maintain() makes that call once here to + // recover the typed ConsolidationResult it must both surface in MaintenanceResult and persist + // in the legacy per-list call shape, rather than invoking the (potentially LLM-backed) + // consolidator a second time through the pass. The shared delegate keeps the two paths in + // lockstep; see [SessionConsolidationPass]. val result = consolidator.consolidate(sessionPropositions, existing) - // Persist promoted, reinforced, and merged propositions + // Persist promoted, reinforced, and merged propositions in the legacy per-list call shape. repository.saveAll(result.promoted) repository.saveAll(result.reinforced) repository.saveAll(result.merged.map { it.result }) @@ -87,15 +118,27 @@ data class DefaultMemoryMaintenanceOrchestrator( } private fun abstract(contextId: ContextId): Pair, List> { - if (abstractor == null) return emptyList() to emptyList() - + val abstractor = this.abstractor ?: return emptyList() to emptyList() + + // The level-0-only snapshot is deliberate and load-bearing for the legacy contract: because + // no higher-level propositions are ever in scope here, the AbstractionPass idempotency guard + // (skip a group already covered by an existing higher-level proposition) has nothing to match + // and is therefore an explicit no-op on this path. maintain() re-abstracts every qualifying + // group on every call, as it did pre-refactor — it does NOT carry the dream-loop's + // skip-covered semantics. Do not widen this query to include higher levels without revisiting + // that contract. val active = repository.query( PropositionQuery.Companion.forContextId(contextId) .withStatus(PropositionStatus.ACTIVE) .withMaxLevel(0) ) - // Group by entity resolvedId + // Recover the per-entity grouping so abstraction runs per group, in the legacy granularity. + // Each group is abstracted in isolation through its OWN AbstractionPass invocation: a group's + // abstractions are exactly those returned by that group's abstract() call (no cross-group + // sourceId matching, so a source mentioning two entities is never saved or counted twice), + // persisted per group in the legacy call shape (one saveAll for the group's abstractions, one + // for its superseded sources). val entityGroups = active .flatMap { prop -> prop.mentions.mapNotNull { it.resolvedId }.map { entityId -> entityId to prop } } .groupBy({ it.first }, { it.second }) @@ -105,18 +148,35 @@ data class DefaultMemoryMaintenanceOrchestrator( val allAbstractions = mutableListOf() val allSuperseded = mutableListOf() - for ((entityId, propositions) in entityGroups) { - val group = PropositionGroup.Companion.of(entityId, propositions) - val abstractions = abstractor.abstract(group, abstractionTargetCount) - - // Persist abstractions - repository.saveAll(abstractions) - allAbstractions.addAll(abstractions) + // maxLevel = Int.MAX_VALUE preserves the pre-refactor "persist every abstraction the + // abstractor returns" contract — maintain() applies no level ceiling. Any cap belongs only on + // the dream-loop path, not here. + val pass = AbstractionPass( + abstractor = abstractor, + abstractionThreshold = abstractionThreshold, + abstractionTargetCount = abstractionTargetCount, + maxLevel = Int.MAX_VALUE, + ) - // Mark source propositions as SUPERSEDED - val superseded = propositions.map { it.withStatus(PropositionStatus.SUPERSEDED) } - repository.saveAll(superseded) - allSuperseded.addAll(superseded) + for ((_, propositions) in entityGroups) { + val outcome = pass.run(contextId, propositions) + if (outcome is ConsolidationPassResult.Failed) { + throw outcome.cause + } + if (outcome !is ConsolidationPassResult.Changed) { + continue + } + + val groupAbstractions = outcome.propositionsToSave.filter { it.level > 0 } + val groupSuperseded = outcome.propositionsToSave.filter { + it.status == PropositionStatus.SUPERSEDED + } + + repository.saveAll(groupAbstractions) + allAbstractions.addAll(groupAbstractions) + + repository.saveAll(groupSuperseded) + allSuperseded.addAll(groupSuperseded) } return allAbstractions to allSuperseded @@ -153,4 +213,7 @@ data class DefaultMemoryMaintenanceOrchestrator( fun withRetireDecayK(k: Double): DefaultMemoryMaintenanceOrchestrator = copy(retireDecayK = k) + + fun withCollector(collector: CollectorRunner): DefaultMemoryMaintenanceOrchestrator = + copy(collector = collector) } diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestrator.kt new file mode 100644 index 00000000..82ee848b --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestrator.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.operations.consolidation.ConsolidationPass + +/** + * Composes [ConsolidationPass]es into a repeatable consolidation cycle — the "dream loop". + * + * Each cycle fetches the ACTIVE snapshot for a context once, runs every registered pass over + * it in registration order, aggregates what all passes want to save into a single write, and + * returns a [DreamLoopReport]. Passes are pure; only the orchestrator writes to the repository. + * + * There are two ways to trigger a cycle: + * - [consolidate] is threshold-gated and returns `null` when not enough has changed. + * - [consolidateNow] always runs regardless of how much has changed. + * + * ```kotlin + * val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository) + * .withPass(sessionConsolidationPass) + * .withPass(abstractionPass) + * .withPass(contradictionResolutionPass) + * .withPass(decaySweepPass) + * + * // Scheduled / threshold-gated: may return null if little has changed. + * val maybeReport = orchestrator.consolidate(contextId) + * + * // Always runs. + * val report = orchestrator.consolidateNow(contextId) + * ``` + */ +interface DreamLoopOrchestrator : MemoryMaintenanceOrchestrator { + + /** + * Run a consolidation cycle only if enough has changed since the last cycle. + * + * Returns `null` when the change-volume threshold isn't met — use [consolidateNow] to + * bypass the threshold entirely. + * + * @param contextId The context to consolidate. + * @return The cycle report when the cycle ran, or `null` if the threshold wasn't met. + */ + fun consolidate(contextId: ContextId): DreamLoopReport? + + /** + * Run a consolidation cycle unconditionally, regardless of how much has changed. + * + * @param contextId The context to consolidate. + * @return The cycle report; never `null`. + */ + fun consolidateNow(contextId: ContextId): DreamLoopReport + + /** + * Add a pass and return a new orchestrator with it appended. Registration order is + * execution order. + * + * @param pass The pass to append. + * @return A new orchestrator with the existing passes plus [pass]. + */ + fun withPass(pass: ConsolidationPass): DreamLoopOrchestrator +} diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DreamLoopReport.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DreamLoopReport.kt new file mode 100644 index 00000000..ea24e6af --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DreamLoopReport.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.operations.consolidation.ConsolidationPassResult +import java.time.Instant + +/** + * Summary of a single dream-loop consolidation cycle. + * + * Holds the result from every pass that ran, plus roll-up totals for how many propositions + * were examined, changed, or newly created. + * + * @property contextId The context the cycle ran for. + * @property cycleStarted When the cycle began. + * @property cycleCompleted When the cycle finished; defaults to construction time. + * @property passResults Each pass's result, in registration (execution) order. + * @property totalExamined How many propositions were in the ACTIVE snapshot. + * @property totalTransitioned How many propositions had their status or content changed across all passes. + * @property totalNewPropositions How many brand-new propositions were produced across all passes. + * @property triggered Always true for a returned report; a threshold-gated skip returns `null` + * rather than a report with this set to false. Retained for forward compatibility. + */ +data class DreamLoopReport( + val contextId: ContextId, + val cycleStarted: Instant, + val cycleCompleted: Instant = Instant.now(), + val passResults: List, + val totalExamined: Int, + val totalTransitioned: Int, + val totalNewPropositions: Int, + val triggered: Boolean, +) diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestrator.kt index a925c0b1..630db95b 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestrator.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestrator.kt @@ -26,12 +26,14 @@ import com.embabel.dice.proposition.PropositionRepository * @property abstractions New abstract propositions generated from entity groups * @property superseded Source propositions that were marked SUPERSEDED after abstraction * @property retired Propositions that were deleted because their effective confidence fell below the threshold + * @property collectorResult Result of the optional collector phase, or null when no collector was configured */ data class MaintenanceResult( val consolidation: ConsolidationResult?, val abstractions: List, val superseded: List, val retired: List, + val collectorResult: CollectorRunResult? = null, ) { /** Total number of propositions persisted (promoted + reinforced + merged + abstractions + superseded status changes) */ val totalPersisted: Int @@ -81,7 +83,7 @@ interface MemoryMaintenanceOrchestrator { ): MaintenanceResult /** - * Builder intermediate: requires a consolidator before producing the orchestrator. + * Intermediate builder step — call [withConsolidator] to get a usable orchestrator. */ class NeedsConsolidator internal constructor( private val repository: PropositionRepository, @@ -95,7 +97,7 @@ interface MemoryMaintenanceOrchestrator { companion object { /** - * Start building a MemoryMaintenanceOrchestrator with the given repository. + * Start building a [MemoryMaintenanceOrchestrator] backed by the given repository. */ @JvmStatic fun withRepository(repository: PropositionRepository): NeedsConsolidator = diff --git a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/AbstractionPassTest.kt b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/AbstractionPassTest.kt new file mode 100644 index 00000000..a7092657 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/AbstractionPassTest.kt @@ -0,0 +1,163 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.operations.consolidation + +import com.embabel.agent.core.ContextId +import com.embabel.dice.operations.PropositionGroup +import com.embabel.dice.operations.abstraction.PropositionAbstractor +import com.embabel.dice.proposition.EntityMention +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import io.mockk.verify +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +class AbstractionPassTest { + + private val contextId = ContextId("test-context") + + private fun proposition( + id: String, + entityId: String, + status: PropositionStatus = PropositionStatus.ACTIVE, + level: Int = 0, + sourceIds: List = emptyList(), + ): Proposition = + Proposition( + id = id, + contextId = contextId, + text = "prop-$id", + mentions = listOf(EntityMention(span = "e", type = "Person", resolvedId = entityId)), + confidence = 0.8, + decay = 0.1, + status = status, + level = level, + sourceIds = sourceIds, + ) + + private fun group(entityId: String, count: Int): List = + (1..count).map { proposition("$entityId-$it", entityId) } + + @Test + fun `name is abstraction`() { + assertEquals("abstraction", AbstractionPass(mockk(relaxed = true)).name) + } + + @Test + fun `below threshold yields NoOp and never calls the abstractor`() { + val abstractor = mockk() + val result = AbstractionPass(abstractor, abstractionThreshold = 5) + .run(contextId, group("bob", 4)) + + assertInstanceOf(ConsolidationPassResult.NoOp::class.java, result) + verify(exactly = 0) { abstractor.abstract(any(), any()) } + } + + @Test + fun `qualifying group is abstracted and sources are SUPERSEDED via withStatus`() { + val members = group("bob", 5) + val abstraction = proposition("abs-1", "bob", level = 1, sourceIds = members.map { it.id }) + val abstractor = mockk() + val groupSlot = slot() + val targetSlot = slot() + every { + abstractor.abstract(capture(groupSlot), capture(targetSlot)) + } returns listOf(abstraction) + + val before = Instant.now() + val result = AbstractionPass(abstractor, abstractionThreshold = 5, abstractionTargetCount = 3) + .run(contextId, members) + + val changed = assertInstanceOf(ConsolidationPassResult.Changed::class.java, result) + assertEquals(3, targetSlot.captured) + assertEquals(5, groupSlot.captured.size) + // abstraction + 5 superseded sources + assertEquals(6, changed.propositionsToSave.size) + assertTrue(changed.propositionsToSave.contains(abstraction)) + val supersededSources = changed.propositionsToSave.filter { it.status == PropositionStatus.SUPERSEDED } + assertEquals(5, supersededSources.size) + // withStatus preserves the contentRevised decay anchor while bumping metadataRevised + supersededSources.forEach { sup -> + val original = members.first { it.id == sup.id } + assertEquals(original.contentRevised, sup.contentRevised) + assertTrue(!sup.metadataRevised.isBefore(before)) + } + } + + @Test + fun `level-inflation guard skips a group already covered by an existing higher-level proposition`() { + val members = group("bob", 5) + val existingAbstraction = proposition( + "abs-1", "bob", level = 1, sourceIds = members.map { it.id }, + ) + val abstractor = mockk() + + val result = AbstractionPass(abstractor, abstractionThreshold = 5) + .run(contextId, members + existingAbstraction) + + // fully covered -> NoOp, abstractor never invoked + assertInstanceOf(ConsolidationPassResult.NoOp::class.java, result) + verify(exactly = 0) { abstractor.abstract(any(), any()) } + } + + @Test + fun `re-running over the pass's own previous output yields NoOp (round-trip idempotency)`() { + // The SPI mandates: running the same pass twice with no intervening write must yield NoOp on + // the second run. The existing guard test hand-crafts a covering abstraction; this proves the + // ACTUAL abstractor output trips the guard, so a real cycle converges instead of inflating. + val members = group("bob", 5) + val produced = proposition("abs-1", "bob", level = 1, sourceIds = members.map { it.id }) + val abstractor = mockk() + every { abstractor.abstract(any(), any()) } returns listOf(produced) + + val pass = AbstractionPass(abstractor, abstractionThreshold = 5) + + // First run produces an abstraction and supersedes the 5 sources. + val first = assertInstanceOf(ConsolidationPassResult.Changed::class.java, pass.run(contextId, members)) + val abstraction = first.propositionsToSave.single { it.level > 0 } + val supersededSources = first.propositionsToSave.filter { it.status == PropositionStatus.SUPERSEDED } + + // Feed the pass's OWN output back in (sources are now SUPERSEDED, plus the new abstraction). + // A converged snapshot must re-run as NoOp and must NOT call the abstractor a second time. + val secondInput = supersededSources + abstraction + val second = pass.run(contextId, secondInput) + + assertInstanceOf(ConsolidationPassResult.NoOp::class.java, second) + verify(exactly = 1) { abstractor.abstract(any(), any()) } + } + + @Test + fun `abstractions exceeding maxLevel are filtered out`() { + val members = group("bob", 5) + val tooHigh = proposition("abs-high", "bob", level = 4, sourceIds = members.map { it.id }) + val abstractor = mockk() + every { abstractor.abstract(any(), any()) } returns listOf(tooHigh) + + val result = AbstractionPass(abstractor, abstractionThreshold = 5, maxLevel = 3) + .run(contextId, members) + + val changed = assertInstanceOf(ConsolidationPassResult.Changed::class.java, result) + // the over-cap abstraction is dropped; only the 5 superseded sources remain + assertTrue(changed.propositionsToSave.none { it.level > 3 }) + assertEquals(5, changed.propositionsToSave.count { it.status == PropositionStatus.SUPERSEDED }) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ConsolidationPassResultTest.kt b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ConsolidationPassResultTest.kt new file mode 100644 index 00000000..2b4eef48 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ConsolidationPassResultTest.kt @@ -0,0 +1,101 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.operations.consolidation + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class ConsolidationPassResultTest { + + private val contextId = ContextId("test-context") + + private fun proposition(text: String = "p"): Proposition = + Proposition( + contextId = contextId, + text = text, + mentions = emptyList(), + confidence = 0.8, + decay = 0.1, + ) + + @Test + fun `Changed carries passName and propositionsToSave with empty defaults`() { + val p = proposition() + val changed = ConsolidationPassResult.Changed("abstraction", propositionsToSave = listOf(p)) + + assertEquals("abstraction", changed.passName) + assertEquals(listOf(p), changed.propositionsToSave) + assertTrue(changed.propositionsToDelete.isEmpty()) + assertEquals(0, changed.skipped) + assertEquals("", changed.summary) + } + + @Test + fun `NoOp carries passName and reason`() { + val noOp = ConsolidationPassResult.NoOp("decay-sweep", "below threshold") + + assertEquals("decay-sweep", noOp.passName) + assertEquals("below threshold", noOp.reason) + } + + @Test + fun `NoOp reason defaults to empty`() { + assertEquals("", ConsolidationPassResult.NoOp("decay-sweep").reason) + } + + @Test + fun `Failed carries passName and the originating cause`() { + val ex = IllegalStateException("boom") + val failed = ConsolidationPassResult.Failed("contradiction-resolution", ex) + + assertEquals("contradiction-resolution", failed.passName) + assertSame(ex, failed.cause) + } + + @Test + fun `every result case exposes passName via the sealed parent`() { + val results: List = listOf( + ConsolidationPassResult.Changed("a"), + ConsolidationPassResult.NoOp("b"), + ConsolidationPassResult.Failed("c", RuntimeException()), + ) + + assertEquals(listOf("a", "b", "c"), results.map { it.passName }) + } + + @Test + fun `a pass runs over the given snapshot without any repository`() { + val snapshot = listOf(proposition("one"), proposition("two")) + val pass = object : ConsolidationPass { + override val name = "echo" + override fun run( + contextId: ContextId, + propositions: List, + ): ConsolidationPassResult = + ConsolidationPassResult.Changed(name, propositionsToSave = propositions) + } + + val result = pass.run(contextId, snapshot) + + assertEquals("echo", pass.name) + assertTrue(result is ConsolidationPassResult.Changed) + assertEquals(snapshot, (result as ConsolidationPassResult.Changed).propositionsToSave) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt new file mode 100644 index 00000000..0706dac6 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt @@ -0,0 +1,122 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.operations.consolidation + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.EntityMention +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.proposition.revision.ClassifiedProposition +import com.embabel.dice.proposition.revision.PropositionRelation +import com.embabel.dice.proposition.revision.PropositionReviser +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class ContradictionResolutionPassTest { + + private val contextId = ContextId("test-context") + + private fun proposition( + id: String, + entityId: String, + confidence: Double, + status: PropositionStatus = PropositionStatus.ACTIVE, + ): Proposition = + Proposition( + id = id, + contextId = contextId, + text = "prop-$id", + mentions = listOf(EntityMention(span = "e", type = "Person", resolvedId = entityId)), + confidence = confidence, + decay = 0.1, + status = status, + ) + + @Test + fun `name is contradiction-resolution`() { + assertEquals("contradiction-resolution", ContradictionResolutionPass(mockk(relaxed = true)).name) + } + + @Test + fun `classifies only against entity-overlapping candidates`() { + val a = proposition("a", "bob", 0.9) + val b = proposition("b", "bob", 0.4) + val unrelated = proposition("c", "alice", 0.7) + val reviser = mockk() + every { reviser.classify(any(), any()) } returns emptyList() + + ContradictionResolutionPass(reviser).run(contextId, listOf(a, b, unrelated)) + + // For 'a', candidate set must be only entity-overlapping 'b' (not 'alice') + verify { reviser.classify(a, listOf(b)) } + verify { reviser.classify(unrelated, emptyList()) } + } + + @Test + fun `transitions the weaker of a contradictory pair to CONTRADICTED`() { + val stronger = proposition("a", "bob", 0.9) + val weaker = proposition("b", "bob", 0.3) + val reviser = mockk() + // When classifying 'stronger', 'weaker' comes back as CONTRADICTORY + every { reviser.classify(stronger, listOf(weaker)) } returns listOf( + ClassifiedProposition(weaker, PropositionRelation.CONTRADICTORY, 0.5), + ) + every { reviser.classify(weaker, listOf(stronger)) } returns listOf( + ClassifiedProposition(stronger, PropositionRelation.CONTRADICTORY, 0.5), + ) + + val result = ContradictionResolutionPass(reviser).run(contextId, listOf(stronger, weaker)) + + val changed = assertInstanceOf(ConsolidationPassResult.Changed::class.java, result) + assertEquals(1, changed.propositionsToSave.size) + val transitioned = changed.propositionsToSave.single() + assertEquals("b", transitioned.id) + assertEquals(PropositionStatus.CONTRADICTED, transitioned.status) + } + + @Test + fun `no contradictions yields NoOp`() { + val a = proposition("a", "bob", 0.9) + val b = proposition("b", "bob", 0.4) + val reviser = mockk() + every { reviser.classify(any(), any()) } returns listOf( + ClassifiedProposition(b, PropositionRelation.SIMILAR, 0.5), + ) + + val result = ContradictionResolutionPass(reviser).run(contextId, listOf(a, b)) + assertInstanceOf(ConsolidationPassResult.NoOp::class.java, result) + } + + @Test + fun `already CONTRADICTED props are not compared so a resolved snapshot re-runs as NoOp`() { + val active = proposition("a", "bob", 0.9) + val resolved = proposition("b", "bob", 0.3, status = PropositionStatus.CONTRADICTED) + val reviser = mockk() + // active has no ACTIVE entity-overlapping peer left, so classify gets empty candidates + every { reviser.classify(active, emptyList()) } returns emptyList() + + val result = ContradictionResolutionPass(reviser).run(contextId, listOf(active, resolved)) + + assertInstanceOf(ConsolidationPassResult.NoOp::class.java, result) + // the already-resolved proposition is never classified + verify(exactly = 0) { reviser.classify(resolved, any()) } + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPassTest.kt b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPassTest.kt new file mode 100644 index 00000000..3f9dc8d6 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPassTest.kt @@ -0,0 +1,193 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.operations.consolidation + +import com.embabel.agent.core.ContextId +import com.embabel.dice.projection.memory.CollectorRunner +import com.embabel.dice.projection.memory.DecayCollectorStrategy +import com.embabel.dice.projection.memory.PropositionMark +import com.embabel.dice.projection.memory.SweepAction +import com.embabel.dice.projection.memory.SweepPolicy +import com.embabel.dice.proposition.EntityMention +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.proposition.PropositionStatus +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.time.Instant +import java.time.temporal.ChronoUnit + +class DecaySweepPassTest { + + private lateinit var repository: PropositionRepository + private val contextId = ContextId("test-context") + + private fun proposition( + text: String, + confidence: Double = 0.8, + decay: Double = 0.1, + entityId: String? = null, + status: PropositionStatus = PropositionStatus.ACTIVE, + pinned: Boolean = false, + revised: Instant = Instant.now(), + ): Proposition { + val mentions = if (entityId != null) { + listOf(EntityMention(span = entityId, type = "Entity", resolvedId = entityId)) + } else { + emptyList() + } + return Proposition( + contextId = contextId, + text = text, + mentions = mentions, + confidence = confidence, + decay = decay, + status = status, + pinned = pinned, + contentRevised = revised, + metadataRevised = revised, + ) + } + + @BeforeEach + fun setup() { + repository = mockk(relaxed = true) + every { repository.query(any()) } returns emptyList() + } + + private fun softRunner(): CollectorRunner = + CollectorRunner + .withRepository(repository) + .withStrategy(DecayCollectorStrategy(retireBelow = 0.3)) + .build() + + private fun hardDeleteRunner(): CollectorRunner = + CollectorRunner + .withRepository(repository) + .withStrategy(DecayCollectorStrategy(retireBelow = 0.3)) + .withPolicy(SweepPolicy { _, marks -> + if (marks.isEmpty()) SweepAction.Skip else SweepAction.HardDelete + }) + .build() + + @Test + fun `name is decay-sweep`() { + assertEquals("decay-sweep", DecaySweepPass(softRunner()).name) + } + + @Test + fun `no candidates yields NoOp mentioning the H threshold`() { + every { repository.query(any()) } returns emptyList() + + val result = DecaySweepPass(softRunner()).run(contextId, emptyList()) + + val noOp = assertInstanceOf(ConsolidationPassResult.NoOp::class.java, result) + assertEquals("decay-sweep", noOp.passName) + assertTrue(noOp.reason.contains("0.1"), "NoOp reason should surface the H threshold: ${noOp.reason}") + } + + @Test + fun `ACTIVE proposition decayed below H transitions to STALE and reports Changed with empty save and delete`() { + val old = Instant.now().minus(365, ChronoUnit.DAYS) + val decayed = proposition("old fact", confidence = 0.5, decay = 0.5, revised = old) + + every { repository.query(any()) } returns listOf(decayed) + val saved = mutableListOf() + every { repository.save(capture(saved)) } answers { saved.last() } + + val result = DecaySweepPass(softRunner()).run(contextId, listOf(decayed)) + + val changed = assertInstanceOf(ConsolidationPassResult.Changed::class.java, result) + assertEquals("decay-sweep", changed.passName) + // Option A: the collector persisted; the pass is report-only. + assertTrue(changed.propositionsToSave.isEmpty(), "pass must not return anything to save") + assertTrue(changed.propositionsToDelete.isEmpty(), "pass must not return anything to delete") + assertTrue(changed.summary.contains("STALE"), "summary should mention STALE transitions") + // The collector itself drove the STALE transition. + assertTrue(saved.any { it.id == decayed.id && it.status == PropositionStatus.STALE }) + } + + @Test + fun `pinned proposition is skipped by the inherited collector behavior`() { + val old = Instant.now().minus(365, ChronoUnit.DAYS) + val pinned = proposition("pinned fact", confidence = 0.5, decay = 0.5, pinned = true, revised = old) + + every { repository.query(any()) } returns listOf(pinned) + val saved = mutableListOf() + every { repository.save(capture(saved)) } answers { saved.last() } + + val result = DecaySweepPass(softRunner()).run(contextId, listOf(pinned)) + + // Pinned is marked-then-skipped by the policy, so nothing is applied -> NoOp. + assertInstanceOf(ConsolidationPassResult.NoOp::class.java, result) + assertFalse(saved.any { it.id == pinned.id && it.status == PropositionStatus.STALE }) + } + + @Test + fun `hard-delete policy reports hard deletes in summary but still returns empty propositionsToDelete`() { + val old = Instant.now().minus(365, ChronoUnit.DAYS) + val decayed = proposition("old fact", confidence = 0.5, decay = 0.5, revised = old) + + every { repository.query(any()) } returns listOf(decayed) + + val result = DecaySweepPass(hardDeleteRunner()).run(contextId, listOf(decayed)) + + val changed = assertInstanceOf(ConsolidationPassResult.Changed::class.java, result) + assertTrue(changed.propositionsToDelete.isEmpty(), "collector owns the delete; pass returns empty list") + assertTrue(changed.summary.contains("hard-deleted"), "summary should report hard-deleted count") + } + + @Test + fun `thresholds are exposed as constructor params and surfaced in NoOp text`() { + every { repository.query(any()) } returns emptyList() + + val pass = DecaySweepPass(softRunner(), staleThresholdH = 0.15, recoveryThresholdS = 0.4) + val result = pass.run(contextId, emptyList()) + + val noOp = assertInstanceOf(ConsolidationPassResult.NoOp::class.java, result) + assertTrue(noOp.reason.contains("0.15"), "custom H should surface in NoOp text: ${noOp.reason}") + } + + @Test + fun `default thresholds are H 0point1 and S 0point25`() { + // Documented defaults; the marks list is unused here but proves the runner is the only collaborator. + val unusedMarks: List = emptyList() + assertTrue(unusedMarks.isEmpty()) + + val pass = DecaySweepPass(softRunner()) + val result = pass.run(contextId, emptyList()) + val noOp = assertInstanceOf(ConsolidationPassResult.NoOp::class.java, result) + assertTrue(noOp.reason.contains("0.1")) + } + + @Test + fun `failure in the runner is reported as Failed`() { + val runner = mockk() + every { runner.run(any(), any()) } throws IllegalStateException("boom") + + val result = DecaySweepPass(runner).run(contextId, emptyList()) + + val failed = assertInstanceOf(ConsolidationPassResult.Failed::class.java, result) + assertEquals("decay-sweep", failed.passName) + assertInstanceOf(IllegalStateException::class.java, failed.cause) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/SessionConsolidationPassTest.kt b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/SessionConsolidationPassTest.kt new file mode 100644 index 00000000..0d75385c --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/SessionConsolidationPassTest.kt @@ -0,0 +1,122 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.operations.consolidation + +import com.embabel.agent.core.ContextId +import com.embabel.dice.projection.memory.ConsolidationResult +import com.embabel.dice.projection.memory.MemoryConsolidator +import com.embabel.dice.projection.memory.PropositionMerge +import com.embabel.dice.proposition.Proposition +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import io.mockk.verify +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class SessionConsolidationPassTest { + + private val contextId = ContextId("test-context") + + private fun proposition(text: String): Proposition = + Proposition( + contextId = contextId, + text = text, + mentions = emptyList(), + confidence = 0.8, + decay = 0.1, + ) + + @Test + fun `name is session-consolidation`() { + val pass = SessionConsolidationPass(mockk(relaxed = true), emptyList()) + assertEquals("session-consolidation", pass.name) + } + + @Test + fun `delegates session and snapshot to the consolidator`() { + val session = listOf(proposition("session-1")) + val snapshot = listOf(proposition("existing-1")) + val consolidator = mockk() + val sessionSlot = slot>() + val existingSlot = slot>() + every { + consolidator.consolidate(capture(sessionSlot), capture(existingSlot)) + } returns ConsolidationResult(emptyList(), emptyList(), emptyList(), emptyList()) + + SessionConsolidationPass(consolidator, session).run(contextId, snapshot) + + verify(exactly = 1) { consolidator.consolidate(any(), any()) } + assertSame(session, sessionSlot.captured) + assertSame(snapshot, existingSlot.captured) + } + + @Test + fun `aggregates promoted reinforced and merged into propositionsToSave`() { + val promoted = proposition("promoted") + val reinforced = proposition("reinforced") + val mergedResult = proposition("merged") + val consolidator = mockk() + every { consolidator.consolidate(any(), any()) } returns ConsolidationResult( + promoted = listOf(promoted), + reinforced = listOf(reinforced), + discarded = listOf(proposition("discarded")), + merged = listOf(PropositionMerge(listOf(proposition("a"), proposition("b")), mergedResult)), + ) + + val result = SessionConsolidationPass(consolidator, listOf(proposition("s"))) + .run(contextId, emptyList()) + + val changed = assertInstanceOf(ConsolidationPassResult.Changed::class.java, result) + assertEquals("session-consolidation", changed.passName) + assertEquals(listOf(promoted, reinforced, mergedResult), changed.propositionsToSave) + assertTrue(changed.summary.isNotBlank()) + } + + @Test + fun `returns NoOp when consolidation yields nothing to store`() { + val consolidator = mockk() + every { consolidator.consolidate(any(), any()) } returns ConsolidationResult( + promoted = emptyList(), + reinforced = emptyList(), + discarded = listOf(proposition("discarded")), + merged = emptyList(), + ) + + val result = SessionConsolidationPass(consolidator, listOf(proposition("s"))) + .run(contextId, emptyList()) + + val noOp = assertInstanceOf(ConsolidationPassResult.NoOp::class.java, result) + assertEquals("session-consolidation", noOp.passName) + } + + @Test + fun `wraps consolidator failure in a Failed result`() { + val boom = RuntimeException("boom") + val consolidator = mockk() + every { consolidator.consolidate(any(), any()) } throws boom + + val result = SessionConsolidationPass(consolidator, listOf(proposition("s"))) + .run(contextId, emptyList()) + + val failed = assertInstanceOf(ConsolidationPassResult.Failed::class.java, result) + assertSame(boom, failed.cause) + assertEquals("session-consolidation", failed.passName) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestratorTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestratorTest.kt new file mode 100644 index 00000000..73771f7f --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestratorTest.kt @@ -0,0 +1,269 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.operations.consolidation.ConsolidationPass +import com.embabel.dice.operations.consolidation.ConsolidationPassResult +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionRepository +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import io.mockk.verify +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNotSame +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class DreamLoopOrchestratorTest { + + private lateinit var repository: PropositionRepository + private val contextId = ContextId("test-context") + + private fun proposition(text: String): Proposition = + Proposition(contextId = contextId, text = text, mentions = emptyList(), confidence = 0.8) + + /** Tiny fake pass returning a fixed result, recording how many times it ran. */ + private class FakePass( + override val name: String, + private val result: (List) -> ConsolidationPassResult, + ) : ConsolidationPass { + var runCount: Int = 0 + override fun run(contextId: ContextId, propositions: List): ConsolidationPassResult { + runCount++ + return result(propositions) + } + } + + @BeforeEach + fun setup() { + repository = mockk(relaxed = true) + every { repository.query(any()) } returns emptyList() + } + + @Test + fun `withPass appends and returns a new instance preserving order`() { + val base = DefaultDreamLoopOrchestrator.withRepository(repository) + val a = FakePass("a") { ConsolidationPassResult.NoOp("a") } + val b = FakePass("b") { ConsolidationPassResult.NoOp("b") } + + val withA = base.withPass(a) + val withAB = withA.withPass(b) + + assertNotSame(base, withA) + assertNotSame(withA, withAB) + + val report = withAB.consolidateNow(contextId) + // registration order = execution order: results reported a then b + assertEquals(listOf("a", "b"), report.passResults.map { it.passName }) + } + + @Test + fun `consolidateNow runs each pass once over a single snapshot`() { + val snapshot = listOf(proposition("p1"), proposition("p2")) + every { repository.query(any()) } returns snapshot + + val a = FakePass("a") { ConsolidationPassResult.NoOp("a") } + val b = FakePass("b") { ConsolidationPassResult.NoOp("b") } + + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository) + .withPass(a) + .withPass(b) + + val report = orchestrator.consolidateNow(contextId) + + assertEquals(1, a.runCount) + assertEquals(1, b.runCount) + assertTrue(report.triggered) + assertEquals(2, report.totalExamined) + } + + @Test + fun `consolidateNow aggregates all saves into a single saveAll`() { + val saved = proposition("saved") + val a = FakePass("a") { + ConsolidationPassResult.Changed("a", propositionsToSave = listOf(saved)) + } + val b = FakePass("b") { + ConsolidationPassResult.Changed("b", propositionsToSave = listOf(proposition("saved-b"))) + } + + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository) + .withPass(a) + .withPass(b) + + val captured = slot>() + every { repository.saveAll(capture(captured)) } returns Unit + + orchestrator.consolidateNow(contextId) + + verify(exactly = 1) { repository.saveAll(any()) } + assertEquals(2, captured.captured.size) + } + + @Test + fun `allowHardDelete defaults off so deletes are ignored`() { + val pass = FakePass("d") { + ConsolidationPassResult.Changed("d", propositionsToDelete = listOf("doomed")) + } + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository) + .withPass(pass) + + orchestrator.consolidateNow(contextId) + + verify(exactly = 0) { repository.delete(any()) } + } + + @Test + fun `allowHardDelete on deletes each aggregated id`() { + val pass = FakePass("d") { + ConsolidationPassResult.Changed("d", propositionsToDelete = listOf("doomed-1", "doomed-2")) + } + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository) + .withPass(pass) + .withAllowHardDelete(true) + + orchestrator.consolidateNow(contextId) + + verify(exactly = 1) { repository.delete("doomed-1") } + verify(exactly = 1) { repository.delete("doomed-2") } + } + + @Test + fun `consolidate returns null below the change-volume threshold`() { + // active count delta from 0 is 3, below default threshold of 10 + every { repository.query(any()) } returns List(3) { proposition("p$it") } + + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository) + .withPass(FakePass("a") { ConsolidationPassResult.NoOp("a") }) + + assertNull(orchestrator.consolidate(contextId)) + } + + @Test + fun `consolidate returns a triggered report at or above the threshold`() { + every { repository.query(any()) } returns List(15) { proposition("p$it") } + + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository) + .withPass(FakePass("a") { ConsolidationPassResult.NoOp("a") }) + + val report = orchestrator.consolidate(contextId) + assertNotNull(report) + assertTrue(report!!.triggered) + } + + @Test + fun `consolidate honors a lowered threshold`() { + every { repository.query(any()) } returns List(3) { proposition("p$it") } + + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository) + .withPass(FakePass("a") { ConsolidationPassResult.NoOp("a") }) + .withChangeVolumeThreshold(2) + + assertNotNull(orchestrator.consolidate(contextId)) + } + + @Test + fun `consolidateNow always runs regardless of threshold`() { + every { repository.query(any()) } returns List(1) { proposition("p$it") } + + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository) + .withPass(FakePass("a") { ConsolidationPassResult.NoOp("a") }) + + val report = orchestrator.consolidateNow(contextId) + assertNotNull(report) + assertTrue(report.triggered) + } + + @Test + fun `consolidateNow with no passes returns an all-NoOp report and saves nothing of substance`() { + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository) + + val report = orchestrator.consolidateNow(contextId) + + assertTrue(report.passResults.isEmpty()) + assertTrue(report.triggered) + verify(exactly = 0) { repository.saveAll(any()) } + } + + @Test + fun `copy-built orchestrators do not share trigger state`() { + // A snapshot below the default threshold-10 delta. After a consolidate() call the + // orchestrator records this count as its per-context baseline. A copy()-derived sibling must + // start from its OWN empty baseline, not inherit the first orchestrator's recorded count. + every { repository.query(any()) } returns List(3) { proposition("p$it") } + + val base = DefaultDreamLoopOrchestrator.withRepository(repository) + .withPass(FakePass("a") { ConsolidationPassResult.NoOp("a") }) + .withChangeVolumeThreshold(3) + + // First call: baseline is 0, delta == 3 >= threshold(3) -> triggers, then records baseline=3. + assertNotNull(base.consolidate(contextId)) + // Second call on the SAME instance: delta == 3 - 3 == 0 < threshold -> skips (state recorded). + assertNull(base.consolidate(contextId)) + + // A sibling built via the copy-backed builder must NOT see base's recorded baseline. + // If trigger state were shared by reference, this would skip (delta 0); it must trigger. + val sibling = base.withChangeVolumeThreshold(3) + assertNotNull(sibling.consolidate(contextId)) + } + + @Test + fun `maintain runs a cycle and returns a legacy-shaped empty MaintenanceResult`() { + // This orchestrator also satisfies the shared MemoryMaintenanceOrchestrator contract: + // maintain() must run a consolidation cycle (so passes execute) yet report nothing in the + // legacy-shaped result, since dream-loop output is surfaced via consolidateNow/DreamLoopReport. + val snapshot = listOf(proposition("p1"), proposition("p2")) + every { repository.query(any()) } returns snapshot + + val pass = FakePass("a") { ConsolidationPassResult.NoOp("a") } + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository).withPass(pass) + + val result = orchestrator.maintain(contextId, sessionPropositions = emptyList()) + + // A cycle actually ran: the registered pass executed. + assertEquals(1, pass.runCount) + // Legacy-shaped result is empty (no consolidation/abstractions/superseded/retired). + assertNull(result.consolidation) + assertTrue(result.abstractions.isEmpty()) + assertTrue(result.superseded.isEmpty()) + assertTrue(result.retired.isEmpty()) + } + + @Test + fun `a failing pass is captured as Failed and the cycle continues`() { + val boom = FakePass("boom") { throw IllegalStateException("kaboom") } + val after = FakePass("after") { ConsolidationPassResult.NoOp("after") } + + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository) + .withPass(boom) + .withPass(after) + + val report = orchestrator.consolidateNow(contextId) + + val failed = report.passResults.filterIsInstance() + assertEquals(1, failed.size) + assertEquals("boom", failed.single().passName) + // cycle continued: the later pass still ran + assertEquals(1, after.runCount) + assertFalse(report.passResults.none { it.passName == "after" }) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestratorTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestratorTest.kt index ec457522..684ad5a3 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestratorTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestratorTest.kt @@ -56,6 +56,7 @@ class MemoryMaintenanceOrchestratorTest { status = status, created = created, contentRevised = revised, + metadataRevised = revised, ) } @@ -300,6 +301,49 @@ class MemoryMaintenanceOrchestratorTest { assertTrue(result.superseded.all { it.status == PropositionStatus.SUPERSEDED }) } + @Test + fun `superseding source propositions preserves their decay clock`() { + // contentRevised is the decay anchor. An administrative status change to + // SUPERSEDED must NOT reset it, otherwise superseded facts spuriously + // regain effective confidence. + val anchored = Instant.now().minus(30, ChronoUnit.DAYS) + val props = (1..5).map { + proposition("fact $it", entityId = "alice", revised = anchored) + } + + every { repository.query(any()) } returns props + every { abstractor.abstract(any(), any()) } returns listOf( + Proposition( + contextId = contextId, + text = "abstraction", + mentions = emptyList(), + confidence = 0.9, + level = 1, + sourceIds = props.map { it.id }, + ) + ) + + val orchestrator = MemoryMaintenanceOrchestrator + .withRepository(repository) + .withConsolidator(consolidator) + .withAbstractor(abstractor) + .withAbstractionThreshold(5) + + val result = orchestrator.maintain(contextId) + + assertEquals(5, result.superseded.size) + assertTrue(result.superseded.all { it.status == PropositionStatus.SUPERSEDED }) + // Decay anchor preserved across the supersede; only metadata advances. + assertTrue( + result.superseded.all { it.contentRevised == anchored }, + "supersede must preserve contentRevised (decay clock)", + ) + assertTrue( + result.superseded.all { it.metadataRevised.isAfter(anchored) }, + "supersede is an administrative touch that advances metadataRevised", + ) + } + @Test fun `persists abstractions and superseded propositions`() { val props = (1..5).map { proposition("fact $it", entityId = "alice") } @@ -379,6 +423,133 @@ class MemoryMaintenanceOrchestratorTest { assertEquals(10, result.superseded.size) } + @Test + fun `persists high-level abstractions without a level cap`() { + // Pre-refactor maintain() applied NO level ceiling: any abstraction the abstractor + // returns is persisted, including level >= 2. Guards against re-introducing a maxLevel + // cap into the legacy path (which would silently drop higher-level abstractions). + val props = (1..5).map { proposition("fact $it about alice", entityId = "alice") } + val level2 = Proposition( + contextId = contextId, + text = "level-2 alice abstraction", + mentions = emptyList(), + confidence = 0.9, + level = 2, + sourceIds = props.map { it.id }, + ) + + every { repository.query(any()) } returns props + every { abstractor.abstract(any(), any()) } returns listOf(level2) + + val orchestrator = MemoryMaintenanceOrchestrator + .withRepository(repository) + .withConsolidator(consolidator) + .withAbstractor(abstractor) + .withAbstractionThreshold(5) + + val result = orchestrator.maintain(contextId) + + assertEquals(1, result.abstractions.size) + assertEquals(2, result.abstractions[0].level) + verify { repository.saveAll(listOf(level2)) } + } + + @Test + fun `persists level 4 abstractions without dropping them`() { + // A level-4 abstraction sits above any plausible default ceiling. maintain() must still + // persist it, proving no cap is silently applied on the legacy path. + val props = (1..5).map { proposition("fact $it about alice", entityId = "alice") } + val level4 = Proposition( + contextId = contextId, + text = "level-4 alice abstraction", + mentions = emptyList(), + confidence = 0.95, + level = 4, + sourceIds = props.map { it.id }, + ) + + every { repository.query(any()) } returns props + every { abstractor.abstract(any(), any()) } returns listOf(level4) + + val orchestrator = MemoryMaintenanceOrchestrator + .withRepository(repository) + .withConsolidator(consolidator) + .withAbstractor(abstractor) + .withAbstractionThreshold(5) + + val result = orchestrator.maintain(contextId) + + assertEquals(1, result.abstractions.size) + assertEquals(4, result.abstractions[0].level) + verify { repository.saveAll(listOf(level4)) } + } + + @Test + fun `abstraction spanning multiple entity groups is saved and counted exactly once`() { + // A single source proposition mentioning two entities belongs to BOTH entity groups. + // Its abstraction must be saved and counted exactly once — not double-saved/double-counted + // by a cross-group sourceId re-split, and not conflated with a structurally-equal sibling + // from the other group. Each group is abstracted in isolation, so each group's abstract() + // call returns that group's own (distinct-instance) abstraction. + val shared = Proposition( + contextId = contextId, + text = "shared fact mentioning alice and bob", + mentions = listOf( + EntityMention(span = "alice", type = "Entity", resolvedId = "alice"), + EntityMention(span = "bob", type = "Entity", resolvedId = "bob"), + ), + confidence = 0.8, + ) + val aliceOnly = (1..4).map { proposition("alice fact $it", entityId = "alice") } + val bobOnly = (1..4).map { proposition("bob fact $it", entityId = "bob") } + val snapshot = listOf(shared) + aliceOnly + bobOnly + + every { repository.query(any()) } returns snapshot + // Each group's abstract() call returns its own abstraction referencing the shared source. + val aliceAbstraction = Proposition( + contextId = contextId, + text = "alice group abstraction", + mentions = emptyList(), + confidence = 0.9, + level = 1, + sourceIds = listOf(shared.id), + ) + val bobAbstraction = Proposition( + contextId = contextId, + text = "bob group abstraction", + mentions = emptyList(), + confidence = 0.9, + level = 1, + sourceIds = listOf(shared.id), + ) + every { + abstractor.abstract(match { it.label == "alice" }, any()) + } returns listOf(aliceAbstraction) + every { + abstractor.abstract(match { it.label == "bob" }, any()) + } returns listOf(bobAbstraction) + + val orchestrator = MemoryMaintenanceOrchestrator + .withRepository(repository) + .withConsolidator(consolidator) + .withAbstractor(abstractor) + .withAbstractionThreshold(5) + + val result = orchestrator.maintain(contextId) + + // Exactly two abstractions — alice's and bob's — each present once. The pre-refactor + // cross-group sourceId re-split would have matched BOTH abstractions to BOTH groups + // (their sourceIds reference the shared source, which is in each group's member set) and + // saved/counted each one twice, inflating this to 4. Per-group isolation keeps it at 2. + assertEquals(2, result.abstractions.size) + assertTrue(result.abstractions.any { it.text == "alice group abstraction" }) + assertTrue(result.abstractions.any { it.text == "bob group abstraction" }) + // Each abstraction appears exactly once (no duplicate-save of the same instance). + assertEquals(result.abstractions.size, result.abstractions.distinct().size) + verify(exactly = 1) { repository.saveAll(listOf(aliceAbstraction)) } + verify(exactly = 1) { repository.saveAll(listOf(bobAbstraction)) } + } + @Test fun `only groups level 0 propositions for abstraction`() { // The query uses withMaxLevel(0), so higher-level propositions should be excluded @@ -540,6 +711,50 @@ class MemoryMaintenanceOrchestratorTest { } } + @Nested + inner class CollectorPhaseTests { + + @Test + fun `no collector configured yields null collectorResult`() { + val orchestrator = MemoryMaintenanceOrchestrator + .withRepository(repository) + .withConsolidator(consolidator) + + val result = orchestrator.maintain(contextId) + + assertNull(result.collectorResult) + } + + @Test + fun `withCollector runs the collector and transitions decayed active propositions to STALE`() { + val old = Instant.now().minus(365, ChronoUnit.DAYS) + val decayed = proposition("old fact", confidence = 0.5, decay = 0.5, revised = old) + + // Only the collector phase consumes candidates here: consolidation is skipped (no session), + // abstraction is skipped (no abstractor), and retirement is skipped (no retireBelow). + every { repository.query(any()) } returns listOf(decayed) + val saved = mutableListOf() + every { repository.save(capture(saved)) } answers { saved.last() } + + val collector = CollectorRunner + .withRepository(repository) + .withStrategy(DecayCollectorStrategy(retireBelow = 0.3)) + .build() + + val orchestrator = MemoryMaintenanceOrchestrator + .withRepository(repository) + .withConsolidator(consolidator) + .withCollector(collector) + + val result = orchestrator.maintain(contextId) + + assertNotNull(result.collectorResult) + assertEquals(1, result.collectorResult!!.applied.size) + assertEquals(decayed.id, result.collectorResult!!.applied.first().propositionId) + assertTrue(saved.any { it.id == decayed.id && it.status == PropositionStatus.STALE }) + } + } + @Nested inner class FullPipelineTests { From 7906549c6b19404adeb0430c9eb437392dc99f22 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:01:42 -0400 Subject: [PATCH 04/27] feat(gate): post-extraction gates and evidence-floor demotion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a pipeline that inspects each freshly extracted proposition and decides keep / reject / route-to-review / demote before it is stored. Opt-in — the default keeps current behavior. - ExtractionGate and ExtractionGatePipeline; StandardGates are the built-in checks - ObservableGate wraps a gate and emits its decision as an event, so a rejection or demotion is never silent - EvidenceFloor demotes a relation that lacks enough supporting evidence to a weaker one (PropositionDemoted) rather than rejecting it outright Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../com/embabel/dice/common/DiceEvent.kt | 16 + .../com/embabel/dice/common/EvidenceFloor.kt | 85 +++++ .../com/embabel/dice/common/Relation.kt | 41 +-- .../com/embabel/dice/common/Relations.kt | 126 ++++---- .../dice/proposition/gate/ExtractionGate.kt | 174 +++++++++++ .../gate/ExtractionGatePipeline.kt | 129 ++++++++ .../dice/proposition/gate/ObservableGate.kt | 85 +++++ .../dice/proposition/gate/StandardGates.kt | 290 ++++++++++++++++++ .../proposition/gate/EvidenceFloorGateTest.kt | 123 ++++++++ .../gate/ExtractionGatePipelineTest.kt | 170 ++++++++++ .../proposition/gate/ExtractionGateTest.kt | 119 +++++++ .../gate/GatePipelineConsumerFlowTest.kt | 199 ++++++++++++ .../proposition/gate/ObservableGateTest.kt | 156 ++++++++++ .../proposition/gate/StandardGatesTest.kt | 270 ++++++++++++++++ 14 files changed, 1900 insertions(+), 83 deletions(-) create mode 100644 dice/src/main/kotlin/com/embabel/dice/common/EvidenceFloor.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGate.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGatePipeline.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/proposition/gate/ObservableGate.kt create mode 100644 dice/src/main/kotlin/com/embabel/dice/proposition/gate/StandardGates.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/proposition/gate/EvidenceFloorGateTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/proposition/gate/ExtractionGatePipelineTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/proposition/gate/ExtractionGateTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/proposition/gate/GatePipelineConsumerFlowTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/proposition/gate/ObservableGateTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/proposition/gate/StandardGatesTest.kt diff --git a/dice/src/main/kotlin/com/embabel/dice/common/DiceEvent.kt b/dice/src/main/kotlin/com/embabel/dice/common/DiceEvent.kt index 795d1d2f..0f3ac31b 100644 --- a/dice/src/main/kotlin/com/embabel/dice/common/DiceEvent.kt +++ b/dice/src/main/kotlin/com/embabel/dice/common/DiceEvent.kt @@ -240,6 +240,22 @@ data class PropositionProjectionSkipped @JvmOverloads constructor( override val timestamp: Instant = Instant.now(), ) : DiceEvent +/** + * A proposition's relation was demoted to a weaker one because it didn't have enough + * supporting evidence — kept and downgraded rather than thrown away. + * + * @property proposition The proposition whose relation was demoted. + * @property toRelation The weaker relation it was demoted to. + * @property reason Why it was demoted. + * @property timestamp When the event was created. + */ +data class PropositionDemoted @JvmOverloads constructor( + val proposition: Proposition, + val toRelation: String, + val reason: String, + override val timestamp: Instant = Instant.now(), +) : DiceEvent + /** * Implement this to react to [DiceEvent]s as they happen. Keep in mind handlers run inline * on the emitting thread, so do anything slow off to the side. diff --git a/dice/src/main/kotlin/com/embabel/dice/common/EvidenceFloor.kt b/dice/src/main/kotlin/com/embabel/dice/common/EvidenceFloor.kt new file mode 100644 index 00000000..377fb329 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/common/EvidenceFloor.kt @@ -0,0 +1,85 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.common + +/** + * The minimum evidence a proposition must carry before a relation may be asserted at full strength. + * + * This is the knob that stops a cheap structural signal — two people sharing an email domain, a + * single calendar co-attendance — from being promoted to a strong semantic claim like "works for" + * or "collaborates with". A relation can declare a floor on two independent dimensions, and an + * assertion has to clear both: + * + * - [minConfidence]: the proposition's decay-adjusted confidence must reach this value. + * - [minAuthority]: the proposition's source must be at least this authoritative (see [AuthorityTier], + * where a *lower* ordinal means *more* authoritative). + * + * Either dimension may be left null, in which case it is simply not checked; a floor with both null + * never blocks anything. When an assertion falls short, [demoteTo] names a weaker predicate to fall + * back to (for example "works for" → "affiliated with"). If [demoteTo] is null the assertion is held + * for review instead. + * + * DICE only carries and checks the floor — it never decides what the threshold values should be or + * what "weaker" means for a given predicate. Those are the consumer's to declare on each [Relation]. + * + * @property minConfidence Lowest decay-adjusted confidence that clears the floor, in `0.0..1.0`; + * null to skip the confidence check. + * @property minAuthority Weakest source authority that clears the floor; null to skip the authority + * check. + * @property demoteTo Predicate to fall back to when the floor is not met; null to hold for review + * instead of demoting. + * @throws IllegalArgumentException if [minConfidence] is outside `0.0..1.0`, or if [demoteTo] is blank. + */ +data class EvidenceFloor @JvmOverloads constructor( + val minConfidence: Double? = null, + val minAuthority: AuthorityTier? = null, + val demoteTo: String? = null, +) { + + init { + require(minConfidence == null || minConfidence in 0.0..1.0) { + "minConfidence must be in 0.0..1.0, was $minConfidence" + } + require(demoteTo == null || demoteTo.isNotBlank()) { "demoteTo must not be blank" } + } + + /** + * Does an assertion with this [confidence] and source [authority] clear the floor? + * + * A null bound on either dimension passes that dimension. Authority clears when it is at least + * as strong as [minAuthority], i.e. its ordinal is less than or equal to the floor's. + */ + fun isSatisfiedBy(confidence: Double, authority: AuthorityTier): Boolean { + val confidenceOk = minConfidence == null || confidence >= minConfidence + val authorityOk = minAuthority == null || authority.ordinal <= minAuthority.ordinal + return confidenceOk && authorityOk + } + + companion object { + + /** A floor on confidence alone. */ + @JvmStatic + @JvmOverloads + fun ofConfidence(minConfidence: Double, demoteTo: String? = null): EvidenceFloor = + EvidenceFloor(minConfidence = minConfidence, demoteTo = demoteTo) + + /** A floor on source authority alone. */ + @JvmStatic + @JvmOverloads + fun ofAuthority(minAuthority: AuthorityTier, demoteTo: String? = null): EvidenceFloor = + EvidenceFloor(minAuthority = minAuthority, demoteTo = demoteTo) + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/common/Relation.kt b/dice/src/main/kotlin/com/embabel/dice/common/Relation.kt index e0ab29f4..661232b1 100644 --- a/dice/src/main/kotlin/com/embabel/dice/common/Relation.kt +++ b/dice/src/main/kotlin/com/embabel/dice/common/Relation.kt @@ -22,8 +22,16 @@ package com.embabel.dice.common * @param predicate The verb phrase expressing the relationship, e.g., "likes", "works at" * @param meaning A description of what the relationship means, e.g., "expresses positive preference for" * @param knowledgeType The epistemological nature of this relationship - * @param subjectType Optional constraint on the subject entity type, e.g., "Person" - * @param objectType Optional constraint on the object entity type, e.g., "Company" + * @param subjectType Optional constraint on the subject entity type, e.g., "Person". + * When set from a Kotlin class via [withSubject] it is the class `simpleName`; a + * mention's type must match this (tolerant of case) for an edge to project. + * @param objectType Optional constraint on the object entity type, e.g., "Company". + * When set from a Kotlin class via [withObject] it is the class `simpleName`; a + * mention's type must match this (tolerant of case) for an edge to project. + * @param evidenceFloor Optional minimum evidence required before this relation may be asserted at + * full strength. Left null, the relation has no floor and behaves as before. When set, a gate such + * as `EvidenceFloorGate` reads it to demote or hold under-supported assertions — so a structural + * signal can't masquerade as a strong claim. */ data class Relation @JvmOverloads constructor( val predicate: String, @@ -31,6 +39,7 @@ data class Relation @JvmOverloads constructor( val knowledgeType: KnowledgeType, val subjectType: String? = null, val objectType: String? = null, + val evidenceFloor: EvidenceFloor? = null, ) { fun withSubject(type: Class<*>): Relation = @@ -39,49 +48,41 @@ data class Relation @JvmOverloads constructor( fun withObject(type: Class<*>): Relation = copy(objectType = type.simpleName) + /** Declare the minimum evidence required before this relation may be asserted at full strength. */ + fun withEvidenceFloor(floor: EvidenceFloor): Relation = + copy(evidenceFloor = floor) + companion object { - /** - * Create a semantic (factual) relation with no type constraints. - */ + /** A factual relation with no subject or object type constraint. */ @JvmStatic @JvmOverloads fun semantic(predicate: String, meaning: String = predicate): Relation = Relation(predicate, meaning, KnowledgeType.SEMANTIC) - /** - * Create a procedural (preference/behavioral) relation with no type constraints. - */ + /** A preference/behavioral relation with no subject or object type constraint. */ @JvmStatic @JvmOverloads fun procedural(predicate: String, meaning: String = predicate): Relation = Relation(predicate, meaning, KnowledgeType.PROCEDURAL) - /** - * Create an episodic (event-based) relation with no type constraints. - */ + /** An event-based relation with no subject or object type constraint. */ @JvmStatic @JvmOverloads fun episodic(predicate: String, meaning: String = predicate): Relation = Relation(predicate, meaning, KnowledgeType.EPISODIC) - /** - * Create a semantic relation with subject type constraint. - */ + /** A factual relation that only applies when the subject matches the given type. */ @JvmStatic fun semanticForSubject(predicate: String, meaning: String, subjectType: String): Relation = Relation(predicate, meaning, KnowledgeType.SEMANTIC, subjectType = subjectType) - /** - * Create a procedural relation with subject type constraint. - */ + /** A preference/behavioral relation that only applies when the subject matches the given type. */ @JvmStatic fun proceduralForSubject(predicate: String, meaning: String, subjectType: String): Relation = Relation(predicate, meaning, KnowledgeType.PROCEDURAL, subjectType = subjectType) - /** - * Create a semantic relation with both type constraints. - */ + /** A factual relation constrained to a specific subject and object type pair. */ @JvmStatic fun semanticBetween(predicate: String, meaning: String, subjectType: String, objectType: String): Relation = Relation(predicate, meaning, KnowledgeType.SEMANTIC, subjectType = subjectType, objectType = objectType) diff --git a/dice/src/main/kotlin/com/embabel/dice/common/Relations.kt b/dice/src/main/kotlin/com/embabel/dice/common/Relations.kt index 9b28d884..42ee9872 100644 --- a/dice/src/main/kotlin/com/embabel/dice/common/Relations.kt +++ b/dice/src/main/kotlin/com/embabel/dice/common/Relations.kt @@ -16,10 +16,12 @@ package com.embabel.dice.common /** - * A reusable collection of relation types with builder-style methods. - * Can be shared across multiple SourceAnalysisContext instances. + * A reusable, immutable collection of [Relation] types with a fluent builder API. + * + * Build one up with `withSemantic`, `withProcedural`, etc., then pass it to a + * `SourceAnalysisContext` via `withRelations`. The same instance can be shared + * across multiple contexts. * - * Usage: * ```kotlin * val relations = Relations.empty() * .withSemantic("works at", "is employed by") @@ -32,106 +34,111 @@ package com.embabel.dice.common * .withRelations(relations) * ``` * - * @property relations The list of relation types + * @property relations The relation types in this collection. */ data class Relations( val relations: List = emptyList(), ) : Iterable { companion object { - /** - * Create an empty Relations collection. - */ + /** Start with an empty collection and build up with `with*` calls. */ @JvmStatic fun empty(): Relations = Relations() - /** - * Create a Relations collection from existing relations. - */ + /** Wrap an existing vararg of relations. */ @JvmStatic fun of(vararg relations: Relation): Relations = Relations(relations.toList()) - /** - * Create a Relations collection from a list of relations. - */ + /** Wrap an existing list of relations. */ @JvmStatic fun of(relations: List): Relations = Relations(relations) } override fun iterator(): Iterator = relations.iterator() - /** - * Returns the number of relations in this collection. - */ + /** Number of relations in this collection. */ fun size(): Int = relations.size - /** - * Returns true if this collection is empty. - */ + /** True if this collection has no relations. */ fun isEmpty(): Boolean = relations.isEmpty() - /** - * Combines this collection with another. - */ + /** Returns a new collection containing the relations from both. */ operator fun plus(other: Relations): Relations = Relations(relations + other.relations) - /** - * Adds a single relation. - */ + /** Returns a new collection with the given relation appended. */ operator fun plus(relation: Relation): Relations = Relations(relations + relation) - /** - * Add a semantic (factual) relation. - */ + /** Add a semantic (factual) relation. */ @JvmOverloads fun withSemantic(predicate: String, meaning: String = predicate): Relations = this + Relation.semantic(predicate, meaning) - /** - * Add a procedural (behavioral/preference) relation. - */ + /** Add a procedural (behavioral/preference) relation. */ @JvmOverloads fun withProcedural(predicate: String, meaning: String = predicate): Relations = this + Relation.procedural(predicate, meaning) - /** - * Add an episodic (event-based) relation. - */ + /** Add an episodic (event-based) relation. */ @JvmOverloads fun withEpisodic(predicate: String, meaning: String = predicate): Relations = this + Relation.episodic(predicate, meaning) - /** - * Add a semantic relation with subject type constraint. - */ + /** Add a semantic relation that only applies when the subject is of the given type. */ fun withSemanticForSubject(subjectType: String, predicate: String, meaning: String): Relations = this + Relation.semanticForSubject(predicate, meaning, subjectType) - /** - * Add a procedural relation with subject type constraint. - */ + /** Add a procedural relation that only applies when the subject is of the given type. */ fun withProceduralForSubject(subjectType: String, predicate: String, meaning: String): Relations = this + Relation.proceduralForSubject(predicate, meaning, subjectType) + /** Add a semantic relation constrained to a specific subject and object type pair. */ + fun withSemanticBetween( + subjectType: String, + objectType: String, + predicate: String, + meaning: String, + ): Relations = this + Relation.semanticBetween(predicate, meaning, subjectType, objectType) + /** - * Add a semantic relation with both subject and object type constraints. + * Add a semantic relation constrained to a specific subject and object type pair, with an + * evidence floor declared inline. This lets you stay in the fluent builder chain rather than + * stepping outside it to call [Relation.withEvidenceFloor] separately. + * + * ```kotlin + * Relations.empty() + * .withSemanticBetween( + * subjectType = "Person", objectType = "Organization", + * predicate = "works for", meaning = "is employed by", + * floor = EvidenceFloor.ofConfidence(0.7, demoteTo = "affiliated with"), + * ) + * ``` + * + * Java callers: use [Relation.semanticBetween] + [Relation.withEvidenceFloor] to attach a + * floor, since this overload is not `@JvmOverloads`-compatible with the four-parameter form. + * + * @param floor the minimum evidence the relation requires before it may be asserted at full + * strength; pass null to add the relation without a floor (equivalent to the overload above) */ fun withSemanticBetween( subjectType: String, objectType: String, predicate: String, meaning: String, - ): Relations = this + Relation.semanticBetween(predicate, meaning, subjectType, objectType) + floor: EvidenceFloor?, + ): Relations { + val relation = Relation.semanticBetween(predicate, meaning, subjectType, objectType) + return this + if (floor != null) relation.withEvidenceFloor(floor) else relation + } /** - * Add multiple relations for the same subject type and knowledge type. - * Uses the predicate as the meaning for brevity. + * Add several relations at once, all sharing the same subject type and knowledge type. + * The predicate string is used as the meaning for each one. * - * @param subjectType The entity type that can be the subject - * @param knowledgeType The epistemological nature of these relations - * @param predicates The predicate strings to add + * @param subjectType Entity type that can be the subject (string form, e.g. `"Person"`) + * @param knowledgeType Knowledge type shared by all added relations + * @param predicates Predicate strings to add */ fun withPredicatesForSubject( subjectType: String, @@ -150,12 +157,13 @@ data class Relations( } /** - * Add multiple relations for the same subject type and knowledge type. - * Uses the predicate as the meaning for brevity. + * Same as [withPredicatesForSubject] but takes a class instead of a string — the subject + * type is recorded as the class `simpleName`. A mention's type must match it (case-insensitive) + * for an edge to project. * - * @param subjectType The entity class that can be the subject - * @param knowledgeType The epistemological nature of these relations - * @param predicates The predicate strings to add + * @param subjectType Entity class whose simple name is used as the subject type constraint + * @param knowledgeType Knowledge type shared by all added relations + * @param predicates Predicate strings to add */ fun withPredicatesForSubject( subjectType: Class<*>, @@ -163,22 +171,14 @@ data class Relations( vararg predicates: String, ): Relations = withPredicatesForSubject(subjectType.simpleName, knowledgeType, *predicates) - /** - * Add multiple procedural relations for a subject type. - * Convenience method for common preference/behavior predicates. - */ + /** Add several procedural relations at once for a subject type. */ fun withProceduralPredicatesForSubject(subjectType: String, vararg predicates: String): Relations = withPredicatesForSubject(subjectType, KnowledgeType.PROCEDURAL, *predicates) - /** - * Add multiple semantic relations for a subject type. - * Convenience method for common factual predicates. - */ + /** Add several semantic relations at once for a subject type. */ fun withSemanticPredicatesForSubject(subjectType: String, vararg predicates: String): Relations = withPredicatesForSubject(subjectType, KnowledgeType.SEMANTIC, *predicates) - /** - * Returns the relations as a List for compatibility. - */ + /** The relations as a plain list. */ fun toList(): List = relations } diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGate.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGate.kt new file mode 100644 index 00000000..0aa67a23 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGate.kt @@ -0,0 +1,174 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition.gate + +import com.embabel.dice.common.SourceAnalysisContext +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.revision.RevisionResult + +/** + * A policy check applied to a proposition after extraction but before it is persisted or + * projected. A gate inspects a single proposition together with its [GateContext] and returns + * a [GateEvaluation] expressing how the proposition should be routed. + * + * Gates are a standalone, consumer-invoked stage that operates on pipeline output BEFORE the + * consumer calls `save()`. They are not embedded in the extraction pipeline itself — the + * proposition remains the canonical source of truth and gates only route or annotate. + * + * Example usage: + * ```kotlin + * // Run the extraction pipeline as usual. + * val results = pipeline.process(chunks, context) + * + * // A simple confidence gate. + * val confidenceGate = ExtractionGate { proposition, _ -> + * val decision = if (proposition.confidence < 0.5) { + * GateDecision.Reject("confidence below threshold") + * } else { + * GateDecision.Persist + * } + * GateEvaluation("ConfidenceGate", proposition, decision) + * } + * + * // Evaluate each proposition before persisting, reading the trust score from metadata. + * results.propositions.forEach { proposition -> + * val gateContext = GateContext( + * // Coerce numerically: a wrong-typed value (Float, Int, BigDecimal) would otherwise + // read as null via `as? Double` and the gate would silently fail open. + trustScore = (proposition.metadata["dice.trust.score"] as? Number)?.toDouble(), + * ) + * val evaluation = confidenceGate.evaluate(proposition, gateContext) + * when (evaluation.decision) { + * is GateDecision.Persist -> repository.save(proposition) + * is GateDecision.RouteToReview -> reviewQueue.add(proposition) + * is GateDecision.Reject -> { /* drop */ } + * is GateDecision.SkipProjection -> repository.save(proposition) // persist but do not project + * } + * } + * ``` + */ +fun interface ExtractionGate { + + /** + * Evaluate a single proposition against this gate's policy. + * + * @param proposition the proposition to evaluate + * @param context the surrounding gate context (trust score, revision result, source context) + * @return an evaluation pairing this gate's name with its decision for the proposition + */ + fun evaluate( + proposition: Proposition, + context: GateContext, + ): GateEvaluation +} + +/** + * Context passed to a gate alongside the proposition under evaluation. + * + * @property revisionResult the revision outcome for this proposition, if it was revised against + * existing knowledge; null when no revision was performed + * @property trustScore a consumer-injected trust value READ from the proposition's metadata. + * The gate layer never computes this score — the consumer supplies the cached value so gates + * can route on it without coupling to any scoring component. Callers should normalize the raw + * metadata value to `Double` (e.g. `(value as? Number)?.toDouble()`) because a wrong-typed + * value reads as missing and the trust gate then fails open. + * @property sourceContext the analysis context the proposition was extracted under, if available + * @property metadata free-form additional context for custom gates + */ +data class GateContext @JvmOverloads constructor( + val revisionResult: RevisionResult? = null, + val trustScore: Double? = null, + val sourceContext: SourceAnalysisContext? = null, + val metadata: Map = emptyMap(), +) + +/** + * The routing decision a gate reaches for a proposition. Exactly five outcomes are possible. + */ +sealed interface GateDecision { + + /** Persist the proposition normally. */ + data object Persist : GateDecision + + /** Hold the proposition for human or downstream review rather than persisting it directly. */ + data class RouteToReview(val reason: String) : GateDecision + + /** Reject the proposition outright; it should not be persisted. */ + data class Reject(val reason: String) : GateDecision + + /** Persist the proposition but exclude it from projection to downstream representations. */ + data class SkipProjection(val reason: String) : GateDecision + + /** + * Persist the proposition, but signal that the relation it asserts should be projected as a + * weaker predicate ([toRelation]) because its evidence did not clear the relation's floor. The + * proposition itself stays canonical and untouched — the consumer applies the relabel when it + * projects, so a cheap structural signal lands as the modest claim it actually supports. + * + * @throws IllegalArgumentException if [toRelation] or [reason] is blank. + */ + data class Demote(val toRelation: String, val reason: String) : GateDecision { + init { + require(toRelation.isNotBlank()) { "toRelation must not be blank" } + require(reason.isNotBlank()) { "reason must not be blank" } + } + } +} + +/** + * The outcome of evaluating a single proposition against a single gate. + * + * @property gateName the name of the gate that produced this evaluation + * @property proposition the proposition that was evaluated + * @property decision the routing decision the gate reached + */ +data class GateEvaluation( + val gateName: String, + val proposition: Proposition, + val decision: GateDecision, +) + +/** + * The aggregated outcome of running one or more gates against a proposition, together with the + * final routing decision. + * + * @property proposition the proposition that was gated + * @property evaluations the per-gate evaluations that contributed to the final decision + * @property finalDecision the decision the consumer should act on + */ +data class GatedPropositionResult( + val proposition: Proposition, + val evaluations: List, + val finalDecision: GateDecision, +) { + /** True when the proposition should be persisted normally. */ + val shouldPersist: Boolean get() = finalDecision is GateDecision.Persist + + /** True when the proposition should be routed to review. */ + val shouldReview: Boolean get() = finalDecision is GateDecision.RouteToReview + + /** True when the proposition was rejected. */ + val isRejected: Boolean get() = finalDecision is GateDecision.Reject + + /** True when the proposition should be persisted but excluded from projection. */ + val skipProjection: Boolean get() = finalDecision is GateDecision.SkipProjection + + /** True when the asserted relation should be demoted to a weaker predicate at projection time. */ + val isDemoted: Boolean get() = finalDecision is GateDecision.Demote + + /** The weaker predicate to project, when the final decision is a demotion; null otherwise. */ + val demoteTo: String? get() = (finalDecision as? GateDecision.Demote)?.toRelation +} diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGatePipeline.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGatePipeline.kt new file mode 100644 index 00000000..048cd4c3 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGatePipeline.kt @@ -0,0 +1,129 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition.gate + +import com.embabel.dice.proposition.Proposition + +/** + * A standalone runner that applies an ordered list of [ExtractionGate]s to propositions and + * aggregates each gate's [GateEvaluation] into a [GatedPropositionResult]. + * + * This is a consumer-invoked, pre-persistence stage. The consumer runs the extraction pipeline, + * then passes the resulting propositions through this runner BEFORE calling `save()` — it is not + * embedded inside the extraction pipeline, so the proposition remains the canonical source of + * truth and gates only route or annotate. Insertion point: + * + * ```kotlin + * val results = pipeline.process(chunks, context) + * val gated = gatePipeline.evaluateAll(results.allPropositions) { proposition -> + * // Coerce numerically so a wrong-typed value does not silently read as missing. + * GateContext(trustScore = (proposition.metadata["dice.trust.score"] as? Number)?.toDouble()) + * } + * gated.forEach { result -> + * when (result.finalDecision) { + * is GateDecision.Persist -> repository.save(result.proposition) + * is GateDecision.RouteToReview -> reviewQueue.add(result.proposition) + * is GateDecision.Reject -> { /* drop */ } + * is GateDecision.SkipProjection -> repository.save(result.proposition) // persist, no projection + * is GateDecision.Demote -> { + * // Persist as-is; consumer applies the weaker predicate at projection time. + * repository.save(result.proposition) + * projector.project(result.proposition, demoteLabel = result.demoteTo) + * } + * } + * } + * ``` + * + * **Convention ordering.** Gates run in the order they are supplied; the runner does not reorder + * them. The recommended convention is to order from cheapest/most-decisive to most-permissive — + * confidence, then dedup/merge, then conflict, then trust, then projection-eligibility — so an + * early hard rejection short-circuits the rest. + * + * **First-non-Persist-wins.** The final decision is the first non-[GateDecision.Persist] decision + * encountered; once set, it is never overwritten. This guarantees a later [GateDecision.SkipProjection] + * (or any other decision) cannot mask an earlier [GateDecision.Reject]. + * + * @property gates the gates to run, in convention order + * @property shortCircuitOnReject when true (the default), gate execution stops as soon as a gate + * returns [GateDecision.Reject] (after recording that evaluation); when false, every gate runs and + * every evaluation is recorded regardless of decision. Note that short-circuiting can truncate + * the recorded evaluation trail even when the [GateDecision.Reject] is not the winning decision — + * for example if an earlier gate already set the final decision to [GateDecision.RouteToReview], a + * later [GateDecision.Reject] still stops execution, so any gates after it never run and are absent + * from [GatedPropositionResult.evaluations]. Set this to false if the consumer needs the complete + * trail for audit or observability. + */ +class ExtractionGatePipeline @JvmOverloads constructor( + gates: List, + val shortCircuitOnReject: Boolean = true, +) { + + /** + * The gates to run, in convention order. Snapshotted at construction so that mutating an + * aliased backing list a caller passed in cannot change the pipeline's iteration afterwards. + */ + val gates: List = gates.toList() + + /** + * Run every gate against a single proposition, in declared order, applying first-non-Persist-wins + * precedence and the configured short-circuit behaviour. + * + * @param proposition the proposition to evaluate + * @param context the gate context for this proposition (trust score, revision result, source context) + * @return the aggregated result: every recorded evaluation plus the final routing decision + */ + fun evaluate( + proposition: Proposition, + context: GateContext, + ): GatedPropositionResult { + val evaluations = mutableListOf() + var finalDecision: GateDecision = GateDecision.Persist + + for (gate in gates) { + val evaluation = gate.evaluate(proposition, context) + evaluations.add(evaluation) + + // First non-Persist decision wins; never overwrite it with a later decision. + if (finalDecision is GateDecision.Persist && evaluation.decision !is GateDecision.Persist) { + finalDecision = evaluation.decision + } + + // Short-circuit on a hard reject AFTER recording the evaluation that produced it. + if (shortCircuitOnReject && evaluation.decision is GateDecision.Reject) { + break + } + } + + return GatedPropositionResult(proposition, evaluations, finalDecision) + } + + /** + * Evaluate a batch of propositions, producing one [GatedPropositionResult] per proposition. + * + * The [contextFor] factory supplies the [GateContext] for each proposition, so the consumer + * can derive per-proposition context — for example, reading a cached trust score from the + * proposition's metadata. + * + * @param propositions the propositions to gate + * @param contextFor produces the gate context for each proposition + * @return one result per input proposition, in input order + */ + fun evaluateAll( + propositions: List, + contextFor: (Proposition) -> GateContext, + ): List = + propositions.map { evaluate(it, contextFor(it)) } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ObservableGate.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ObservableGate.kt new file mode 100644 index 00000000..d7f1e4b2 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ObservableGate.kt @@ -0,0 +1,85 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition.gate + +import com.embabel.dice.common.DiceEventListener +import com.embabel.dice.common.PropositionDemoted +import com.embabel.dice.common.PropositionProjectionSkipped +import com.embabel.dice.common.PropositionRejected +import com.embabel.dice.common.PropositionRoutedToReview +import com.embabel.dice.common.SafeDiceEventListener +import com.embabel.dice.proposition.Proposition + +/** + * Decorator that adds event emission to any [ExtractionGate]. + * + * Delegates the evaluation to the wrapped gate and then, depending on the [GateDecision], + * emits the matching routing event through the supplied [DiceEventListener]. The delegate's + * [GateEvaluation] is always returned unchanged — the decorator fires events, it never alters + * the decision. + * + * Event mapping: + * - [GateDecision.Reject] → [PropositionRejected] + * - [GateDecision.RouteToReview] → [PropositionRoutedToReview] + * - [GateDecision.SkipProjection] → [PropositionProjectionSkipped] + * - [GateDecision.Demote] → [PropositionDemoted] + * - [GateDecision.Persist] → nothing emitted; the persist signal fires at the save boundary + * and emitting one here would double-emit it. + * + * Each emitted event carries the full [Proposition]. Wrap the listener in [SafeDiceEventListener] + * so a throwing listener cannot abort the gate run; this decorator does not handle exceptions itself. + * + * Example usage: + * ```kotlin + * val gate = ObservableGate(confidenceGate, SafeDiceEventListener(myListener)) + * val evaluation = gate.evaluate(proposition, gateContext) + * ``` + * + * @property delegate the underlying gate to wrap + * @property listener the listener that receives routing events; defaults to + * [DiceEventListener.DEV_NULL] (no observation) + */ +class ObservableGate( + private val delegate: ExtractionGate, + private val listener: DiceEventListener = DiceEventListener.DEV_NULL, +) : ExtractionGate { + + override fun evaluate( + proposition: Proposition, + context: GateContext, + ): GateEvaluation { + val evaluation = delegate.evaluate(proposition, context) + + when (val decision = evaluation.decision) { + is GateDecision.Reject -> + listener.onEvent(PropositionRejected(proposition, decision.reason)) + + is GateDecision.RouteToReview -> + listener.onEvent(PropositionRoutedToReview(proposition, decision.reason)) + + is GateDecision.SkipProjection -> + listener.onEvent(PropositionProjectionSkipped(proposition, decision.reason)) + + is GateDecision.Demote -> + listener.onEvent(PropositionDemoted(proposition, decision.toRelation, decision.reason)) + + // The durable persist signal fires at the save boundary; emitting here would double-emit it. + is GateDecision.Persist -> Unit + } + + return evaluation + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/StandardGates.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/StandardGates.kt new file mode 100644 index 00000000..0c16b1a6 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/StandardGates.kt @@ -0,0 +1,290 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition.gate + +import com.embabel.dice.common.AuthorityResolver +import com.embabel.dice.common.Relations +import com.embabel.dice.common.StructuralAuthorityResolver +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.proposition.revision.RevisionResult + +/** + * Routes a proposition based purely on its own confidence: anything below [minConfidence] is + * rejected, everything else is persisted. + * + * This gate reads the decay-adjusted [Proposition.effectiveConfidence] — it has no dependency on + * the surrounding context and therefore makes the same decision regardless of revision or trust + * signals. Using the effective (rather than raw extraction-time) confidence keeps the threshold + * consistent with the query layer (`PropositionQuery.withMinEffectiveConfidence`) and the decay + * collector, so an aged proposition is gated on the same value every other read path scores it on. + * + * @property minConfidence the inclusive lower bound a proposition must meet to be persisted; + * must be in `0.0..1.0` + * @throws IllegalArgumentException if [minConfidence] is outside `0.0..1.0` + */ +class ConfidenceGate(private val minConfidence: Double) : ExtractionGate { + + init { + require(minConfidence in 0.0..1.0) { "minConfidence must be in 0.0..1.0, was $minConfidence" } + } + + override fun evaluate( + proposition: Proposition, + context: GateContext, + ): GateEvaluation { + val effectiveConfidence = proposition.effectiveConfidence() + val decision = if (effectiveConfidence >= minConfidence) { + GateDecision.Persist + } else { + GateDecision.Reject("effective confidence $effectiveConfidence < $minConfidence") + } + return GateEvaluation(GATE_NAME, proposition, decision) + } + + private companion object { + const val GATE_NAME = "ConfidenceGate" + } +} + +/** + * Routes propositions that were merged into, or reinforced an existing proposition to review so a + * consumer can confirm the consolidation. The gate carries no policy: it reads only + * [GateContext.revisionResult]. + * + * Fail-open: when no revision result is present (the proposition was not revised against existing + * knowledge), the gate persists. The reviser owns the classification — this gate only interprets it. + */ +class MergeCandidateGate : ExtractionGate { + + override fun evaluate( + proposition: Proposition, + context: GateContext, + ): GateEvaluation { + val decision = when (context.revisionResult) { + is RevisionResult.Merged -> + GateDecision.RouteToReview("merged into an existing proposition; confirm consolidation") + is RevisionResult.Reinforced -> + GateDecision.RouteToReview("reinforced an existing proposition; confirm consolidation") + // null (not revised) and all other outcomes persist — fail open. + else -> GateDecision.Persist + } + return GateEvaluation(GATE_NAME, proposition, decision) + } + + private companion object { + const val GATE_NAME = "MergeCandidateGate" + } +} + +/** + * Routes propositions that contradicted existing knowledge to review. The gate carries no policy: + * it reads only [GateContext.revisionResult]. + * + * Conservative default: any contradiction routes to review regardless of its conflict type. A + * consumer that wants finer behaviour (for example treating world progression as benign) can + * supply its own gate. + * + * Fail-open: when no revision result is present, the gate persists. + */ +class ConflictClassificationGate : ExtractionGate { + + override fun evaluate( + proposition: Proposition, + context: GateContext, + ): GateEvaluation { + val decision = when (context.revisionResult) { + is RevisionResult.Contradicted -> + GateDecision.RouteToReview("contradicts an existing proposition; review the conflict") + // null (not revised) and all other outcomes persist — fail open. + else -> GateDecision.Persist + } + return GateEvaluation(GATE_NAME, proposition, decision) + } + + private companion object { + const val GATE_NAME = "ConflictClassificationGate" + } +} + +/** + * Routes propositions whose trust score falls below [minTrustScore]. The score is supplied by the + * consumer through [GateContext.trustScore]; this gate never computes trust and depends on no + * scoring component — it only reads the injected value. + * + * Fail-open: when the trust score is absent the gate applies [onMissingScore], which defaults to + * persisting. A consumer that wants a missing score to be treated as untrusted can supply, for + * example, `GateDecision.Reject(...)`. + * + * @property minTrustScore the inclusive lower bound a trust score must meet to be persisted; + * must be in `0.0..1.0` + * @property onMissingScore the decision to apply when [GateContext.trustScore] is null + * @throws IllegalArgumentException if [minTrustScore] is outside `0.0..1.0` + */ +class TrustGate @JvmOverloads constructor( + private val minTrustScore: Double, + private val onMissingScore: GateDecision = GateDecision.Persist, +) : ExtractionGate { + + init { + require(minTrustScore in 0.0..1.0) { "minTrustScore must be in 0.0..1.0, was $minTrustScore" } + } + + override fun evaluate( + proposition: Proposition, + context: GateContext, + ): GateEvaluation { + val trustScore = context.trustScore + val decision = when { + trustScore == null -> onMissingScore + trustScore >= minTrustScore -> GateDecision.Persist + else -> GateDecision.RouteToReview("trust score $trustScore < $minTrustScore") + } + return GateEvaluation(GATE_NAME, proposition, decision) + } + + private companion object { + const val GATE_NAME = "TrustGate" + } +} + +/** + * Decides whether a proposition should be projected to downstream representations. + * + * A proposition is excluded from projection (but may still be persisted) when its + * decay-adjusted effective confidence is below [minConfidence] or its status is + * [PropositionStatus.CONTRADICTED]. Using effective rather than raw confidence keeps + * the threshold consistent with what the query and decay-collector layers see. + * + * @property minConfidence the inclusive lower bound below which a proposition is excluded from + * projection; must be in `0.0..1.0`. Defaults to [DEFAULT_MIN_PROJECTION_CONFIDENCE]. + * @throws IllegalArgumentException if [minConfidence] is outside `0.0..1.0` + */ +class ProjectionEligibilityGate @JvmOverloads constructor( + private val minConfidence: Double = DEFAULT_MIN_PROJECTION_CONFIDENCE, +) : ExtractionGate { + + init { + require(minConfidence in 0.0..1.0) { "minConfidence must be in 0.0..1.0, was $minConfidence" } + } + + override fun evaluate( + proposition: Proposition, + context: GateContext, + ): GateEvaluation { + val effectiveConfidence = proposition.effectiveConfidence() + val decision = when { + effectiveConfidence < minConfidence -> + GateDecision.SkipProjection("effective confidence $effectiveConfidence < $minConfidence") + proposition.status == PropositionStatus.CONTRADICTED -> + GateDecision.SkipProjection("status is ${PropositionStatus.CONTRADICTED}") + else -> GateDecision.Persist + } + return GateEvaluation(GATE_NAME, proposition, decision) + } + + companion object { + /** + * Permissive default: only very low-confidence propositions are blocked from projection, + * so moderately confident ones still reach downstream representations. + */ + const val DEFAULT_MIN_PROJECTION_CONFIDENCE: Double = 0.3 + + private const val GATE_NAME = "ProjectionEligibilityGate" + } +} + +/** + * Stops a cheap structural signal from being asserted as a strong claim. + * + * Matches a proposition to a declared [com.embabel.dice.common.Relation] by checking whether + * the relation's predicate appears in the proposition text. If that relation declares an + * [com.embabel.dice.common.EvidenceFloor] and the proposition's decay-adjusted confidence and + * source authority clear it, the proposition persists. If it falls short, the gate either demotes + * the assertion to the floor's weaker predicate ([GateDecision.Demote]) or, when the floor names + * none, holds it for review ([GateDecision.RouteToReview]). + * + * The mechanism is generic; the policy is not. DICE only checks the floor a relation declares — + * what "works for" demotes to, and how high its bar sits, are the consumer's to configure on each + * relation. The gate never mutates the proposition: it routes, leaving the proposition canonical. + * + * Fail-open: a proposition that matches no declared relation, or matches one with no floor, persists. + * + * @property relations the declared relations whose floors this gate enforces + * @property authorityResolver resolves a proposition's source authority; defaults to the + * grounding-driven [StructuralAuthorityResolver] + * @property caseSensitive whether predicate matching is case-sensitive (default false) + */ +class EvidenceFloorGate @JvmOverloads constructor( + private val relations: Relations, + private val authorityResolver: AuthorityResolver = StructuralAuthorityResolver(), + private val caseSensitive: Boolean = false, +) : ExtractionGate { + + override fun evaluate( + proposition: Proposition, + context: GateContext, + ): GateEvaluation { + val relation = matchingRelation(proposition) + val floor = relation?.evidenceFloor + val decision = if (relation == null || floor == null) { + // No declared relation, or no floor on it — nothing to enforce. + GateDecision.Persist + } else { + val confidence = proposition.effectiveConfidence() + val authority = authorityResolver.resolve(proposition) + if (floor.isSatisfiedBy(confidence, authority)) { + GateDecision.Persist + } else { + val reason = + "evidence floor for '${relation.predicate}' not met " + + "(confidence=$confidence, authority=$authority)" + val demoteTo = floor.demoteTo + if (demoteTo != null) { + GateDecision.Demote(demoteTo, reason) + } else { + GateDecision.RouteToReview(reason) + } + } + } + return GateEvaluation(GATE_NAME, proposition, decision) + } + + /** + * Returns the first relation whose predicate appears as a substring in the proposition text, + * or null if none match. + * + * **Matching is substring-based and order-sensitive.** A short predicate such as `"works"` + * will match a proposition about "networks with" (because "networks" contains "works"). When + * multiple relations could match — for example both `"works"` and `"works for"` are declared — + * whichever appears first in [relations] wins. This means a broader predicate declared before a + * narrower one will always shadow the narrower one, so the narrower relation's floor is never + * applied. **Declare more-specific predicates first** to ensure the tightest floor always wins. + * + * If word-boundary matching is needed (to avoid the "networks"/"works" false positive), replace + * `text.contains(predicate)` with a `\b${Regex.escape(predicate)}\b` regex match. + */ + private fun matchingRelation(proposition: Proposition) = + relations.firstOrNull { relation -> + val text = if (caseSensitive) proposition.text else proposition.text.lowercase() + val predicate = if (caseSensitive) relation.predicate else relation.predicate.lowercase() + text.contains(predicate) + } + + private companion object { + const val GATE_NAME = "EvidenceFloorGate" + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/gate/EvidenceFloorGateTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/EvidenceFloorGateTest.kt new file mode 100644 index 00000000..7c6801ac --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/EvidenceFloorGateTest.kt @@ -0,0 +1,123 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition.gate + +import com.embabel.agent.core.ContextId +import com.embabel.dice.common.AuthorityTier +import com.embabel.dice.common.EvidenceFloor +import com.embabel.dice.common.FixedAuthorityResolver +import com.embabel.dice.common.Relation +import com.embabel.dice.common.Relations +import com.embabel.dice.proposition.Proposition +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Test + +class EvidenceFloorGateTest { + + private val contextId = ContextId("test-context") + + private fun proposition( + text: String = "Alice works for Acme", + confidence: Double = 0.9, + ): Proposition = Proposition( + contextId = contextId, + text = text, + mentions = emptyList(), + confidence = confidence, + ) + + /** A "works for" relation that demands strong confidence and a named source, demoting to "affiliated with". */ + private fun worksForWithFloor(demoteTo: String? = "affiliated with"): Relations = + Relations.of( + Relation.semanticBetween("works for", "is employed by", "Person", "Organization") + .withEvidenceFloor( + EvidenceFloor(minConfidence = 0.7, minAuthority = AuthorityTier.SECONDARY, demoteTo = demoteTo), + ), + ) + + private fun gate(relations: Relations, authority: AuthorityTier) = + EvidenceFloorGate(relations, FixedAuthorityResolver(authority)) + + @Test + fun `persists when confidence and authority clear the floor`() { + val decision = gate(worksForWithFloor(), AuthorityTier.PRIMARY) + .evaluate(proposition(confidence = 0.9), GateContext()).decision + assertEquals(GateDecision.Persist, decision) + } + + @Test + fun `demotes to the weaker predicate when confidence falls short`() { + val decision = gate(worksForWithFloor(), AuthorityTier.PRIMARY) + .evaluate(proposition(confidence = 0.5), GateContext()).decision + val demote = assertInstanceOf(GateDecision.Demote::class.java, decision) + assertEquals("affiliated with", demote.toRelation) + } + + @Test + fun `demotes when source authority is too weak even with high confidence`() { + // A bare email domain resolves to DERIVED authority — below the SECONDARY floor. + val decision = gate(worksForWithFloor(), AuthorityTier.DERIVED) + .evaluate(proposition(confidence = 0.95), GateContext()).decision + assertInstanceOf(GateDecision.Demote::class.java, decision) + } + + @Test + fun `holds for review when the floor is unmet and no demote target is declared`() { + val decision = gate(worksForWithFloor(demoteTo = null), AuthorityTier.DERIVED) + .evaluate(proposition(confidence = 0.5), GateContext()).decision + assertInstanceOf(GateDecision.RouteToReview::class.java, decision) + } + + @Test + fun `fails open when the proposition matches no declared relation`() { + val decision = gate(worksForWithFloor(), AuthorityTier.DERIVED) + .evaluate(proposition(text = "Bob likes pizza", confidence = 0.1), GateContext()).decision + assertEquals(GateDecision.Persist, decision) + } + + @Test + fun `fails open when the matched relation declares no floor`() { + val noFloor = Relations.of(Relation.semanticBetween("works for", "is employed by", "Person", "Organization")) + val decision = gate(noFloor, AuthorityTier.DERIVED) + .evaluate(proposition(confidence = 0.1), GateContext()).decision + assertEquals(GateDecision.Persist, decision) + } + + @Test + fun `persists when confidence is exactly at the floor boundary`() { + // confidence == minConfidence (0.7) must pass — isSatisfiedBy uses >=. + val decision = gate(worksForWithFloor(), AuthorityTier.SECONDARY) + .evaluate(proposition(confidence = 0.7), GateContext()).decision + assertEquals(GateDecision.Persist, decision) + } + + @Test + fun `demotes when confidence is just below the floor boundary`() { + // confidence just below minConfidence (0.7) must demote. + val decision = gate(worksForWithFloor(), AuthorityTier.SECONDARY) + .evaluate(proposition(confidence = 0.6999), GateContext()).decision + assertInstanceOf(GateDecision.Demote::class.java, decision) + } + + @Test + fun `persists when authority is exactly at the floor boundary`() { + // authority == minAuthority (SECONDARY) must pass — isSatisfiedBy uses <= on ordinal. + val decision = gate(worksForWithFloor(), AuthorityTier.SECONDARY) + .evaluate(proposition(confidence = 0.9), GateContext()).decision + assertEquals(GateDecision.Persist, decision) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/gate/ExtractionGatePipelineTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/ExtractionGatePipelineTest.kt new file mode 100644 index 00000000..21f0df78 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/ExtractionGatePipelineTest.kt @@ -0,0 +1,170 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition.gate + +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +class ExtractionGatePipelineTest { + + private fun proposition(id: String = "p1"): Proposition = Proposition.create( + id = id, + contextIdValue = "ctx", + text = "The sky is blue", + mentions = emptyList(), + confidence = 0.8, + decay = 0.0, + reasoning = null, + grounding = emptyList(), + created = Instant.now(), + revised = Instant.now(), + status = PropositionStatus.ACTIVE, + ) + + private fun gate(name: String, decision: GateDecision): ExtractionGate = + ExtractionGate { proposition, _ -> GateEvaluation(name, proposition, decision) } + + private fun alwaysPersist(name: String = "Persist"): ExtractionGate = + gate(name, GateDecision.Persist) + + @Test + fun `two persist gates produce a Persist final decision and record both evaluations`() { + val pipeline = ExtractionGatePipeline(listOf(alwaysPersist("a"), alwaysPersist("b"))) + val result = pipeline.evaluate(proposition(), GateContext()) + assertTrue(result.finalDecision is GateDecision.Persist) + assertTrue(result.shouldPersist) + assertEquals(2, result.evaluations.size) + assertEquals(listOf("a", "b"), result.evaluations.map { it.gateName }) + } + + @Test + fun `first non-Persist wins so a later Reject cannot override an earlier RouteToReview`() { + val pipeline = ExtractionGatePipeline( + listOf( + alwaysPersist("persist"), + gate("review", GateDecision.RouteToReview("dup")), + gate("reject", GateDecision.Reject("low")), + ), + shortCircuitOnReject = true, + ) + val result = pipeline.evaluate(proposition(), GateContext()) + + // First non-Persist decision wins: RouteToReview, NOT the later Reject. + assertTrue(result.finalDecision is GateDecision.RouteToReview) + assertEquals("dup", (result.finalDecision as GateDecision.RouteToReview).reason) + + // The Reject gate is still reached and recorded; short-circuit fires after it. + assertEquals(listOf("persist", "review", "reject"), result.evaluations.map { it.gateName }) + } + + @Test + fun `short-circuit stops execution after the first Reject`() { + val pipeline = ExtractionGatePipeline( + listOf( + gate("reject", GateDecision.Reject("a")), + gate("skip", GateDecision.SkipProjection("b")), + ), + shortCircuitOnReject = true, + ) + val result = pipeline.evaluate(proposition(), GateContext()) + assertTrue(result.finalDecision is GateDecision.Reject) + assertEquals("a", (result.finalDecision as GateDecision.Reject).reason) + // The skipProjection gate never runs. + assertEquals(listOf("reject"), result.evaluations.map { it.gateName }) + } + + @Test + fun `first non-Persist wins still yields the earlier Reject when short-circuit is off`() { + val pipeline = ExtractionGatePipeline( + listOf( + gate("reject", GateDecision.Reject("a")), + gate("skip", GateDecision.SkipProjection("b")), + ), + shortCircuitOnReject = false, + ) + val result = pipeline.evaluate(proposition(), GateContext()) + assertTrue(result.finalDecision is GateDecision.Reject) + assertEquals("a", (result.finalDecision as GateDecision.Reject).reason) + } + + @Test + fun `with short-circuit off every gate runs and every evaluation is recorded`() { + val pipeline = ExtractionGatePipeline( + listOf( + gate("review", GateDecision.RouteToReview("dup")), + gate("reject", GateDecision.Reject("low")), + gate("skip", GateDecision.SkipProjection("noisy")), + ), + shortCircuitOnReject = false, + ) + val result = pipeline.evaluate(proposition(), GateContext()) + // First non-Persist encountered is RouteToReview. + assertTrue(result.finalDecision is GateDecision.RouteToReview) + assertEquals(3, result.evaluations.size) + assertEquals(listOf("review", "reject", "skip"), result.evaluations.map { it.gateName }) + } + + @Test + fun `evaluateAll maps a batch with a per-proposition context factory`() { + val seen = mutableListOf() + val recordingGate = ExtractionGate { proposition, context -> + seen.add(context.trustScore) + GateEvaluation("recorder", proposition, GateDecision.Persist) + } + val pipeline = ExtractionGatePipeline(listOf(recordingGate)) + val p1 = proposition("p1") + val p2 = proposition("p2") + + val results = pipeline.evaluateAll(listOf(p1, p2)) { prop -> + GateContext(trustScore = if (prop.id == "p1") 0.1 else 0.9) + } + + assertEquals(2, results.size) + assertEquals(p1, results[0].proposition) + assertEquals(p2, results[1].proposition) + // Each proposition was evaluated with the context produced by the factory. + assertEquals(listOf(0.1, 0.9), seen) + } + + @Test + fun `an empty gate list yields a Persist final decision and no evaluations`() { + val pipeline = ExtractionGatePipeline(emptyList()) + val result = pipeline.evaluate(proposition(), GateContext()) + assertTrue(result.finalDecision is GateDecision.Persist) + assertTrue(result.evaluations.isEmpty()) + } + + @Test + fun `mutating the caller's backing list after construction does not change pipeline behaviour`() { + val backing = mutableListOf(gate("reject", GateDecision.Reject("low"))) + val pipeline = ExtractionGatePipeline(backing) + + // Mutate the list the caller passed in after the pipeline was constructed. + backing.clear() + backing.add(alwaysPersist("persist")) + + val result = pipeline.evaluate(proposition(), GateContext()) + + // The pipeline still runs its snapshot (the original Reject gate), unaffected by mutation. + assertTrue(result.isRejected) + assertEquals(listOf("reject"), result.evaluations.map { it.gateName }) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/gate/ExtractionGateTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/ExtractionGateTest.kt new file mode 100644 index 00000000..b406d48c --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/ExtractionGateTest.kt @@ -0,0 +1,119 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition.gate + +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +class ExtractionGateTest { + + private fun proposition(): Proposition = Proposition.create( + id = "p1", + contextIdValue = "ctx", + text = "The sky is blue", + mentions = emptyList(), + confidence = 0.8, + decay = 0.0, + reasoning = null, + grounding = emptyList(), + created = Instant.now(), + revised = Instant.now(), + status = PropositionStatus.ACTIVE, + ) + + @Test + fun `GateContext default-constructs with all-null and empty fields`() { + val context = GateContext() + assertEquals(null, context.revisionResult) + assertEquals(null, context.trustScore) + assertEquals(null, context.sourceContext) + assertTrue(context.metadata.isEmpty()) + } + + @Test + fun `GateContext reads an injected trust score`() { + val context = GateContext(trustScore = 0.4) + assertEquals(0.4, context.trustScore) + } + + @Test + fun `GateDecision has exactly four cases and Persist is a singleton`() { + assertTrue(GateDecision.Persist is GateDecision) + assertTrue(GateDecision.RouteToReview("dup") is GateDecision) + assertTrue(GateDecision.Reject("low") is GateDecision) + assertTrue(GateDecision.SkipProjection("noisy") is GateDecision) + assertEquals("low", (GateDecision.Reject("low")).reason) + } + + @Test + fun `GateEvaluation pairs gate name, proposition, and decision`() { + val prop = proposition() + val eval = GateEvaluation("ConfidenceGate", prop, GateDecision.Reject("low")) + assertEquals("ConfidenceGate", eval.gateName) + assertEquals(prop, eval.proposition) + assertTrue(eval.decision is GateDecision.Reject) + } + + @Test + fun `GatedPropositionResult exposes shouldPersist for a Persist decision`() { + val prop = proposition() + val result = GatedPropositionResult(prop, emptyList(), GateDecision.Persist) + assertTrue(result.shouldPersist) + assertFalse(result.isRejected) + assertFalse(result.shouldReview) + assertFalse(result.skipProjection) + } + + @Test + fun `GatedPropositionResult exposes shouldReview for a RouteToReview decision`() { + val prop = proposition() + val result = GatedPropositionResult(prop, emptyList(), GateDecision.RouteToReview("dup")) + assertTrue(result.shouldReview) + assertFalse(result.shouldPersist) + } + + @Test + fun `GatedPropositionResult exposes isRejected for a Reject decision`() { + val prop = proposition() + val result = GatedPropositionResult(prop, emptyList(), GateDecision.Reject("low")) + assertTrue(result.isRejected) + assertFalse(result.shouldPersist) + } + + @Test + fun `GatedPropositionResult exposes skipProjection for a SkipProjection decision`() { + val prop = proposition() + val result = GatedPropositionResult(prop, emptyList(), GateDecision.SkipProjection("noisy")) + assertTrue(result.skipProjection) + assertFalse(result.shouldPersist) + } + + @Test + fun `ExtractionGate is a SAM that returns a GateEvaluation`() { + val prop = proposition() + val gate = ExtractionGate { proposition, _ -> + GateEvaluation("AlwaysPersist", proposition, GateDecision.Persist) + } + val eval = gate.evaluate(prop, GateContext()) + assertEquals("AlwaysPersist", eval.gateName) + assertTrue(eval.decision is GateDecision.Persist) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/gate/GatePipelineConsumerFlowTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/GatePipelineConsumerFlowTest.kt new file mode 100644 index 00000000..c3f87137 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/GatePipelineConsumerFlowTest.kt @@ -0,0 +1,199 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition.gate + +import com.embabel.agent.core.ContextId +import com.embabel.dice.common.AuthorityTier +import com.embabel.dice.common.EvidenceFloor +import com.embabel.dice.common.FixedAuthorityResolver +import com.embabel.dice.common.PropositionDemoted +import com.embabel.dice.common.PropositionProjectionSkipped +import com.embabel.dice.common.PropositionRejected +import com.embabel.dice.common.PropositionRoutedToReview +import com.embabel.dice.common.RecordingDiceEventListener +import com.embabel.dice.common.Relations +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.proposition.revision.RevisionResult +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +/** + * End-to-end behavioural test for the gate layer as a standalone, consumer-invoked + * pre-persistence stage: real standard gates composed in a pipeline, wrapped for + * observation, with each final decision routed by the consumer the way the public KDoc + * documents. The per-component tests cover each piece in isolation; this exercises the + * documented composition that a consumer actually wires up. + */ +class GatePipelineConsumerFlowTest { + + private val contextId = ContextId("ctx") + + private fun proposition( + id: String, + confidence: Double = 0.9, + status: PropositionStatus = PropositionStatus.ACTIVE, + ): Proposition = Proposition( + contextId = contextId, + text = "proposition $id", + mentions = emptyList(), + confidence = confidence, + status = status, + ) + + @Test + fun `consumer composes standard gates in a pipeline and routes each proposition by its final decision`() { + val recorder = RecordingDiceEventListener() + // Convention ordering: confidence, merge, conflict, trust, projection-eligibility, + // each wrapped so routing events are observed. + val pipeline = ExtractionGatePipeline( + listOf( + ObservableGate(ConfidenceGate(0.5), recorder), + ObservableGate(MergeCandidateGate(), recorder), + ObservableGate(ConflictClassificationGate(), recorder), + ObservableGate(TrustGate(0.4), recorder), + ObservableGate(ProjectionEligibilityGate(0.3), recorder), + ), + shortCircuitOnReject = true, + ) + + val rejected = proposition("rejected", confidence = 0.2) + val merged = proposition("merged") + val lowTrust = proposition("low-trust") + val skipped = proposition("skipped", status = PropositionStatus.CONTRADICTED) + val clean = proposition("clean") + + // Per-proposition context, as a consumer would derive it from pipeline output. + val contextFor: (Proposition) -> GateContext = { prop -> + when (prop.id) { + merged.id -> GateContext(revisionResult = RevisionResult.Merged(prop, prop)) + lowTrust.id -> GateContext(trustScore = 0.1) + else -> GateContext() + } + } + + val results = pipeline.evaluateAll( + listOf(rejected, merged, lowTrust, skipped, clean), + contextFor, + ) + + // The consumer routes each proposition by the final decision; we capture where each lands. + val persisted = mutableListOf() + val reviewQueue = mutableListOf() + val projected = mutableListOf() + results.forEach { result -> + when (result.finalDecision) { + is GateDecision.Persist -> { + persisted += result.proposition + projected += result.proposition + } + is GateDecision.RouteToReview -> reviewQueue += result.proposition + is GateDecision.Reject -> { /* dropped: not persisted */ } + is GateDecision.SkipProjection -> persisted += result.proposition // saved, not projected + is GateDecision.Demote -> { + // Saved as-is and projected, but under the weaker predicate the consumer applies. + persisted += result.proposition + projected += result.proposition + } + } + } + + // rejected: dropped entirely. + assertTrue(persisted.none { it.id == rejected.id }) + assertTrue(reviewQueue.none { it.id == rejected.id }) + // merged: routed to review (first non-Persist), not persisted directly. + assertTrue(reviewQueue.any { it.id == merged.id }) + assertTrue(persisted.none { it.id == merged.id }) + // low-trust: trust gate routes to review. + assertTrue(reviewQueue.any { it.id == lowTrust.id }) + // contradicted: persisted but excluded from projection. + assertTrue(persisted.any { it.id == skipped.id }) + assertTrue(projected.none { it.id == skipped.id }) + // clean: persisted and projected. + assertTrue(persisted.any { it.id == clean.id }) + assertTrue(projected.any { it.id == clean.id }) + + // Observation: every non-Persist final decision produced exactly the matching event. + assertEquals(1, recorder.eventsOfType().size) + assertEquals(2, recorder.eventsOfType().size) + assertEquals(1, recorder.eventsOfType().size) + } + + @Test + fun `EvidenceFloorGate in a composed pipeline produces a Demote decision and emits PropositionDemoted`() { + val recorder = RecordingDiceEventListener() + + // A "works for" relation requiring confidence >= 0.7 from at least SECONDARY authority, + // demoting to "affiliated with" when the floor is not met. + val relations = Relations.empty() + .withSemanticBetween( + subjectType = "Person", + objectType = "Organization", + predicate = "works for", + meaning = "is employed by", + floor = EvidenceFloor( + minConfidence = 0.7, + minAuthority = AuthorityTier.SECONDARY, + demoteTo = "affiliated with", + ), + ) + + // Authority resolver always returns DERIVED, which is below the SECONDARY floor. + val pipeline = ExtractionGatePipeline( + listOf( + ObservableGate( + EvidenceFloorGate(relations, FixedAuthorityResolver(AuthorityTier.DERIVED)), + recorder, + ), + ), + ) + + // Proposition text matches "works for"; confidence is fine but authority is too weak. + val prop = Proposition( + contextId = contextId, + text = "Alice works for Acme", + mentions = emptyList(), + confidence = 0.9, + ) + + val results = pipeline.evaluateAll(listOf(prop)) { GateContext() } + + // Final decision must be Demote, not Persist. + val result = results.single() + assertInstanceOf(GateDecision.Demote::class.java, result.finalDecision) + assertEquals("affiliated with", result.demoteTo) + + // ObservableGate must have emitted exactly one PropositionDemoted event. + val event = recorder.eventsOfType().single() + assertEquals(prop, event.proposition) + assertEquals("affiliated with", event.toRelation) + } + + @Test + fun `the gate stage does not run inside the extraction pipeline so an empty stage persists everything unchanged`() { + // A consumer that wires up no gates must get the canonical proposition back untouched: + // the gate stage is additive and standalone, never mutating pipeline output. + val pipeline = ExtractionGatePipeline(emptyList()) + val input = listOf(proposition("a"), proposition("b")) + + val results = pipeline.evaluateAll(input) { GateContext() } + + assertEquals(input, results.map { it.proposition }) + assertTrue(results.all { it.shouldPersist }) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/gate/ObservableGateTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/ObservableGateTest.kt new file mode 100644 index 00000000..578e69dd --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/ObservableGateTest.kt @@ -0,0 +1,156 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition.gate + +import com.embabel.dice.common.DiceEventListener +import com.embabel.dice.common.PropositionDemoted +import com.embabel.dice.common.PropositionProjectionSkipped +import com.embabel.dice.common.PropositionRejected +import com.embabel.dice.common.PropositionRoutedToReview +import com.embabel.dice.common.RecordingDiceEventListener +import com.embabel.dice.common.SafeDiceEventListener +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +class ObservableGateTest { + + private fun proposition(): Proposition = Proposition.create( + id = "p1", + contextIdValue = "ctx", + text = "The sky is blue", + mentions = emptyList(), + confidence = 0.8, + decay = 0.0, + reasoning = null, + grounding = emptyList(), + created = Instant.now(), + revised = Instant.now(), + status = PropositionStatus.ACTIVE, + ) + + private fun gateDeciding(decision: GateDecision): ExtractionGate = + ExtractionGate { proposition, _ -> GateEvaluation("TestGate", proposition, decision) } + + @Test + fun `a Reject decision emits PropositionRejected carrying the proposition and reason`() { + val prop = proposition() + val recorder = RecordingDiceEventListener() + val gate = ObservableGate(gateDeciding(GateDecision.Reject("low")), recorder) + + gate.evaluate(prop, GateContext()) + + assertEquals(1, recorder.count()) + val event = recorder.eventsOfType().single() + assertEquals(prop, event.proposition) + assertEquals("low", event.reason) + } + + @Test + fun `a RouteToReview decision emits PropositionRoutedToReview carrying the proposition and reason`() { + val prop = proposition() + val recorder = RecordingDiceEventListener() + val gate = ObservableGate(gateDeciding(GateDecision.RouteToReview("dup")), recorder) + + gate.evaluate(prop, GateContext()) + + assertEquals(1, recorder.count()) + val event = recorder.eventsOfType().single() + assertEquals(prop, event.proposition) + assertEquals("dup", event.reason) + } + + @Test + fun `a SkipProjection decision emits PropositionProjectionSkipped carrying the proposition and reason`() { + val prop = proposition() + val recorder = RecordingDiceEventListener() + val gate = ObservableGate(gateDeciding(GateDecision.SkipProjection("low conf")), recorder) + + gate.evaluate(prop, GateContext()) + + assertEquals(1, recorder.count()) + val event = recorder.eventsOfType().single() + assertEquals(prop, event.proposition) + assertEquals("low conf", event.reason) + } + + @Test + fun `a Demote decision emits PropositionDemoted carrying the proposition, toRelation, and reason`() { + val prop = proposition() + val recorder = RecordingDiceEventListener() + val gate = ObservableGate(gateDeciding(GateDecision.Demote("affiliated with", "floor not met")), recorder) + + gate.evaluate(prop, GateContext()) + + assertEquals(1, recorder.count()) + val event = recorder.eventsOfType().single() + assertEquals(prop, event.proposition) + assertEquals("affiliated with", event.toRelation) + assertEquals("floor not met", event.reason) + } + + @Test + fun `a Persist decision emits nothing`() { + val prop = proposition() + val recorder = RecordingDiceEventListener() + val gate = ObservableGate(gateDeciding(GateDecision.Persist), recorder) + + gate.evaluate(prop, GateContext()) + + assertEquals(0, recorder.count()) + } + + @Test + fun `evaluate returns the delegate evaluation unchanged`() { + val prop = proposition() + val delegateEvaluation = GateEvaluation("TestGate", prop, GateDecision.Reject("low")) + val delegate = ExtractionGate { _, _ -> delegateEvaluation } + val gate = ObservableGate(delegate, RecordingDiceEventListener()) + + val result = gate.evaluate(prop, GateContext()) + + assertSame(delegateEvaluation, result) + assertEquals("TestGate", result.gateName) + assertTrue(result.decision is GateDecision.Reject) + } + + @Test + fun `the listener defaults to DEV_NULL when none is supplied`() { + val prop = proposition() + val gate = ObservableGate(gateDeciding(GateDecision.Reject("low"))) + + // No listener supplied: must not throw and must still return the delegate evaluation. + val result = gate.evaluate(prop, GateContext()) + + assertTrue(result.decision is GateDecision.Reject) + } + + @Test + fun `a throwing listener wrapped in SafeDiceEventListener does not break the gate run`() { + val prop = proposition() + val throwing = DiceEventListener { throw RuntimeException("boom") } + val gate = ObservableGate(gateDeciding(GateDecision.Reject("low")), SafeDiceEventListener(throwing)) + + // The throw is isolated; evaluate completes and returns the delegate evaluation. + val result = gate.evaluate(prop, GateContext()) + + assertTrue(result.decision is GateDecision.Reject) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/gate/StandardGatesTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/StandardGatesTest.kt new file mode 100644 index 00000000..ce8c4a9a --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/StandardGatesTest.kt @@ -0,0 +1,270 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition.gate + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.proposition.revision.RevisionResult +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Instant +import java.time.temporal.ChronoUnit + +class StandardGatesTest { + + private val contextId = ContextId("test-context") + + private fun proposition( + confidence: Double = 0.8, + status: PropositionStatus = PropositionStatus.ACTIVE, + ): Proposition = Proposition( + contextId = contextId, + text = "Alice knows Kubernetes", + mentions = emptyList(), + confidence = confidence, + status = status, + ) + + /** + * A proposition whose raw confidence is high but which has aged enough, with a nonzero decay + * rate, that its decay-adjusted effective confidence drops well below the raw value. + */ + private fun agedProposition( + confidence: Double = 0.9, + decay: Double = 0.5, + ageInDays: Long = 60, + status: PropositionStatus = PropositionStatus.ACTIVE, + ): Proposition = Proposition( + contextId = contextId, + text = "Alice knows Kubernetes", + mentions = emptyList(), + confidence = confidence, + decay = decay, + status = status, + contentRevised = Instant.now().minus(ageInDays, ChronoUnit.DAYS), + ) + + @Nested + inner class ConfidenceGateTests { + + @Test + fun `rejects below threshold with reason naming value and threshold`() { + val evaluation = ConfidenceGate(0.5).evaluate(proposition(confidence = 0.4), GateContext()) + val decision = evaluation.decision + assertTrue(decision is GateDecision.Reject) + val reason = (decision as GateDecision.Reject).reason + assertTrue(reason.contains("0.4")) + assertTrue(reason.contains("0.5")) + } + + @Test + fun `persists at threshold`() { + val evaluation = ConfidenceGate(0.5).evaluate(proposition(confidence = 0.5), GateContext()) + assertEquals(GateDecision.Persist, evaluation.decision) + } + + @Test + fun `persists above threshold`() { + val evaluation = ConfidenceGate(0.5).evaluate(proposition(confidence = 0.9), GateContext()) + assertEquals(GateDecision.Persist, evaluation.decision) + } + + @Test + fun `gate name is simple class name`() { + val evaluation = ConfidenceGate(0.5).evaluate(proposition(), GateContext()) + assertEquals("ConfidenceGate", evaluation.gateName) + } + + @Test + fun `rejects out-of-range policy at construction`() { + assertThrows { ConfidenceGate(1.5) } + } + + @Test + fun `thresholds an aged proposition on decay-adjusted effective confidence not raw`() { + val aged = agedProposition(confidence = 0.9) + // Sanity: raw confidence is above threshold, effective (decayed) confidence is below it. + assertTrue(aged.confidence >= 0.6) + assertTrue(aged.effectiveConfidence() < 0.6) + + val evaluation = ConfidenceGate(0.6).evaluate(aged, GateContext()) + val decision = evaluation.decision + assertTrue(decision is GateDecision.Reject) + assertTrue((decision as GateDecision.Reject).reason.contains("effective confidence")) + } + } + + @Nested + inner class MergeCandidateGateTests { + + private val gate = MergeCandidateGate() + private val prop = proposition() + + @Test + fun `routes merged to review`() { + val context = GateContext(revisionResult = RevisionResult.Merged(prop, prop)) + assertTrue(gate.evaluate(prop, context).decision is GateDecision.RouteToReview) + } + + @Test + fun `routes reinforced to review`() { + val context = GateContext(revisionResult = RevisionResult.Reinforced(prop, prop)) + assertTrue(gate.evaluate(prop, context).decision is GateDecision.RouteToReview) + } + + @Test + fun `persists new`() { + val context = GateContext(revisionResult = RevisionResult.New(prop)) + assertEquals(GateDecision.Persist, gate.evaluate(prop, context).decision) + } + + @Test + fun `fails open to persist when revision result is null`() { + assertEquals(GateDecision.Persist, gate.evaluate(prop, GateContext()).decision) + } + + @Test + fun `gate name is simple class name`() { + assertEquals("MergeCandidateGate", gate.evaluate(prop, GateContext()).gateName) + } + } + + @Nested + inner class ConflictClassificationGateTests { + + private val gate = ConflictClassificationGate() + private val prop = proposition() + + @Test + fun `routes contradicted to review`() { + val context = GateContext(revisionResult = RevisionResult.Contradicted(prop, prop)) + assertTrue(gate.evaluate(prop, context).decision is GateDecision.RouteToReview) + } + + @Test + fun `persists new`() { + val context = GateContext(revisionResult = RevisionResult.New(prop)) + assertEquals(GateDecision.Persist, gate.evaluate(prop, context).decision) + } + + @Test + fun `persists merged`() { + val context = GateContext(revisionResult = RevisionResult.Merged(prop, prop)) + assertEquals(GateDecision.Persist, gate.evaluate(prop, context).decision) + } + + @Test + fun `fails open to persist when revision result is null`() { + assertEquals(GateDecision.Persist, gate.evaluate(prop, GateContext()).decision) + } + + @Test + fun `gate name is simple class name`() { + assertEquals("ConflictClassificationGate", gate.evaluate(prop, GateContext()).gateName) + } + } + + @Nested + inner class TrustGateTests { + + @Test + fun `routes below threshold to review`() { + val evaluation = TrustGate(0.6).evaluate(proposition(), GateContext(trustScore = 0.4)) + assertTrue(evaluation.decision is GateDecision.RouteToReview) + } + + @Test + fun `persists at or above threshold`() { + val evaluation = TrustGate(0.6).evaluate(proposition(), GateContext(trustScore = 0.8)) + assertEquals(GateDecision.Persist, evaluation.decision) + } + + @Test + fun `applies default persist on missing score`() { + val evaluation = TrustGate(0.6).evaluate(proposition(), GateContext(trustScore = null)) + assertEquals(GateDecision.Persist, evaluation.decision) + } + + @Test + fun `applies configured on-missing decision when score is null`() { + val gate = TrustGate(0.6, onMissingScore = GateDecision.Reject("no trust")) + val evaluation = gate.evaluate(proposition(), GateContext(trustScore = null)) + assertEquals(GateDecision.Reject("no trust"), evaluation.decision) + } + + @Test + fun `gate name is simple class name`() { + assertEquals("TrustGate", TrustGate(0.6).evaluate(proposition(), GateContext()).gateName) + } + + @Test + fun `rejects out-of-range policy at construction`() { + assertThrows { TrustGate(-0.1) } + } + } + + @Nested + inner class ProjectionEligibilityGateTests { + + @Test + fun `skips projection for low confidence`() { + val evaluation = ProjectionEligibilityGate(0.3).evaluate(proposition(confidence = 0.2), GateContext()) + assertTrue(evaluation.decision is GateDecision.SkipProjection) + } + + @Test + fun `skips projection for contradicted status regardless of confidence`() { + val prop = proposition(confidence = 0.9, status = PropositionStatus.CONTRADICTED) + val evaluation = ProjectionEligibilityGate(0.3).evaluate(prop, GateContext()) + assertTrue(evaluation.decision is GateDecision.SkipProjection) + } + + @Test + fun `persists eligible active proposition`() { + val prop = proposition(confidence = 0.5, status = PropositionStatus.ACTIVE) + assertEquals(GateDecision.Persist, ProjectionEligibilityGate(0.3).evaluate(prop, GateContext()).decision) + } + + @Test + fun `default threshold is usable`() { + assertEquals(GateDecision.Persist, ProjectionEligibilityGate().evaluate(proposition(confidence = 0.5), GateContext()).decision) + } + + @Test + fun `gate name is simple class name`() { + assertEquals("ProjectionEligibilityGate", ProjectionEligibilityGate().evaluate(proposition(), GateContext()).gateName) + } + + @Test + fun `rejects out-of-range policy at construction`() { + assertThrows { ProjectionEligibilityGate(1.5) } + } + + @Test + fun `skips projection for an aged proposition on decay-adjusted effective confidence not raw`() { + val aged = agedProposition(confidence = 0.9) + assertTrue(aged.confidence >= 0.5) + assertTrue(aged.effectiveConfidence() < 0.5) + + val evaluation = ProjectionEligibilityGate(0.5).evaluate(aged, GateContext()) + assertTrue(evaluation.decision is GateDecision.SkipProjection) + } + } +} From ff7e2cd8f77c8afbd68a7bd5db2f05c27c67463a Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Fri, 19 Jun 2026 08:57:02 -0400 Subject: [PATCH 05/27] docs: knowledge-hygiene design note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add docs/design/knowledge-hygiene.md — the design decisions behind keeping the store healthy: admitting deliberately at post-extraction gates, reclaiming via mark-and-sweep with an audit trail, and consolidating between sessions with the dream loop. Frames the three as separate, pluggable, conservative-by-default interventions tied to the proposition lifecycle. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- docs/design/knowledge-hygiene.md | 146 +++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 docs/design/knowledge-hygiene.md diff --git a/docs/design/knowledge-hygiene.md b/docs/design/knowledge-hygiene.md new file mode 100644 index 00000000..b3d421cb --- /dev/null +++ b/docs/design/knowledge-hygiene.md @@ -0,0 +1,146 @@ +# Knowledge hygiene: admission, reclamation, and consolidation + +A store of facts that an LLM extracts from raw text will rot if you let it. Junk gets admitted, +duplicates pile up, contradictions sit unresolved, and stale entries never leave. DICE treats +hygiene as three deliberate interventions at three different moments — what we let *in*, what we +*reclaim*, and what we *consolidate* between sessions. This note is about why those three exist as +separate decisions, not about the classes that implement them. + +All three ship conservative defaults and are pluggable. The theme throughout: prefer reversible, +auditable moves over destructive ones, because a knowledge base you can't trust to keep what it was +told is worse than one that grows a little. + +```mermaid +flowchart LR + EX[Extraction] --> GATE{Admission gates} + GATE -->|persist| STORE[(Proposition store)] + GATE -->|route to review| REVIEW[Review queue] + GATE -->|reject| DROP[Dropped] + GATE -->|skip projection| STORE + GATE -->|demote| STORE + STORE --> COLLECT["Reclamation
mark & sweep"] + STORE --> DREAM["Consolidation
dream loop"] + COLLECT --> STORE + DREAM --> STORE +``` + +## Admission gates + +Freshly extracted propositions don't flow straight into the store. They pass through a gate +sequence that gives each one an explicit, recorded decision: **persist** it, **route it to +review**, **reject** it, **skip projecting** it into the graph, or **demote** a relation that +falls below an evidence floor. + +The reason to gate at admission rather than clean up afterwards is cost and clarity. It's cheaper +to never admit junk than to find and remove it later, and a low-confidence fact is easier to judge +the moment it arrives, with its extraction context still close at hand. The **evidence floor** is +the sharpest example of the design intent: a relation that isn't backed by enough evidence +shouldn't shape the graph, but the underlying proposition isn't worthless — so we demote the +relation rather than discard the fact. + +Every gate decision is recorded in the pipeline's result, so you can see after the fact why a +proposition was persisted, held for review, rejected, or demoted — which matters because admission +policy is exactly the kind of thing teams tune over time and need to be able to reason about. Wrap +the gates in the observable decorator and each non-persist decision also emits an event — rejected, +routed to review, projection-skipped, or demoted — so a consumer can watch admissions live. A +persisted proposition stays silent here, because the save boundary already emits that +(see [events](events.md)). + +## Reclamation: mark and sweep + +Reclamation borrows the shape of a tracing garbage collector on purpose: one stage decides *what +looks like garbage*, a separate stage decides *what to do about it*, and every action leaves a +record. + +Splitting "mark" from "sweep" keeps two independent judgments independent. A marker flags a +proposition with a reason — it's gone **stale**, it's a **duplicate** of a survivor, or some +domain-specific reason — without committing to a fate. The sweep policy then chooses the fate: +move it to a new lifecycle status, hard-delete it, or skip it. Keeping these apart means you can +change what counts as garbage without touching what happens to it, and vice versa. + +The default fate is deliberately non-destructive: a marked proposition is transitioned to a colder +status, not deleted. Hard deletion is available but opt-in. This is the same instinct as the +lifecycle's "decay, don't delete" stance (see [proposition-lifecycle](proposition-lifecycle.md)) — +losing data quietly erodes trust in the whole store. + +Reclamation is built to be auditable: when an audit trail is configured, a run records what was +marked, why, and what happened to it — and a **dry run** records that trail while changing nothing, +so you can preview a policy before it touches real data. Duplicate handling resolves overlapping +clusters down to a single survivor, so merging is deterministic rather than order-dependent. + +Each status transition a sweep applies also emits a `PropositionStatusChanged` event (on real runs, +not dry runs) — the same event the store emits when a status changes, so a transition made by the +collector looks identical to one made anywhere else (see [events](events.md)). + +A run marks candidates, lets the policy decide each one's fate, and records the outcome: + +```mermaid +sequenceDiagram + autonumber + participant Run as Collection run + participant Marker + participant Policy as Sweep policy + participant Store + participant Audit as Audit record + Run->>Marker: scan candidates + Marker-->>Run: marks — stale / duplicate / custom, each with a reason + loop for each mark + Run->>Policy: what should happen here? + Policy-->>Run: transition status / hard delete / skip + alt not a dry run + Run->>Store: apply the action + end + end + Run->>Audit: record the run — every mark, its reason, and its outcome +``` + +## Consolidation: the dream loop + +Admission and reclamation keep the store from filling with bad data, but they don't make good data +*better*. That's the job of the dream loop: a set of consolidation passes that run as repeatable +cycles, ideally during idle time, to tidy what's already there. + +The passes are composable and each does one thing: fold a session's raw facts together, abstract a +cluster of related facts into a higher-level proposition, resolve lingering contradictions, and +sweep decayed entries. Composability is the design decision — consolidation isn't one monolithic +LLM step you either run or don't; it's a pipeline of small, individually understandable, repeatable +operations you can reorder, extend, or run on their own. + +The loop is threshold-gated: it only does expensive work when there's enough accumulated material +to be worth it, so running it often is cheap when nothing has changed. And it ties directly back to +the lifecycle — abstraction is what drives **supersession**, contradiction resolution is what +drives **contradiction**, and the decay sweep is what moves cold facts toward **stale**. The dream +loop is, in effect, where most lifecycle transitions other than first ingestion actually get +triggered. + +A cycle runs its passes over a context's propositions and reports what changed: + +```mermaid +sequenceDiagram + autonumber + participant DreamLoop as Dream loop + participant Store + Note over DreamLoop: skip the cycle unless enough has accumulated + DreamLoop->>Store: read the context's propositions + DreamLoop->>DreamLoop: session consolidation + DreamLoop->>DreamLoop: abstraction — folds clusters, supersedes the sources + DreamLoop->>DreamLoop: contradiction resolution + DreamLoop->>DreamLoop: decay sweep — marks cold facts stale + DreamLoop->>Store: persist the resulting changes + DreamLoop-->>DreamLoop: emit a report of the cycle +``` + +## How the stages combine + +The three stages act at different points in a proposition's life. The **gates** run at admission and +decide what enters the store. The **mark-and-sweep collector** runs continuously and reclaims +propositions that are no longer worth keeping, preferring a status transition over deletion. The +**dream loop** runs periodically and consolidates what's already stored so it stays concise and +consistent rather than only growing. The proposition lifecycle ties all three together. + +## Configurable behavior + +The gate sequence, the marker strategies and sweep policy, and the consolidation passes are all +pluggable. What ships is intentionally cautious — admit unless clearly bad, reclaim into a colder +status rather than delete, consolidate only past a threshold — so the safe behaviour is the default +and a deployment opts into anything more aggressive. From 105fc60b076b300aa3b92995983536c3472eef10 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 09:36:30 -0400 Subject: [PATCH 06/27] feat(observability): add debug/trace logging across extraction, maintenance, and store paths Wire SLF4J loggers through the store, trust, conflict, event, gate, and memory-maintenance seams so a consuming application can see the decision and persistence paths: - gates: log each gate's decision outcome, pipeline short-circuits, and every observable gate event - maintenance: log collector-run and dream-loop summaries, decay/duplicate strategy mark counts, and retirement counts - trust/events/store: resolved authority tier and trust score, conflict verdict, emitted lifecycle and projection events, query match/return counts, JSON-file load count and flush-failure rollbacks, and no-embedder vector skips Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../memory/DecayCollectorStrategy.kt | 10 +++++++-- .../memory/DefaultCollectorRunner.kt | 12 +++++++++- .../memory/DefaultDreamLoopOrchestrator.kt | 7 +++++- .../DefaultMemoryMaintenanceOrchestrator.kt | 17 ++++++++++++++ .../memory/DuplicateCollectorStrategy.kt | 10 ++++++++- .../gate/ExtractionGatePipeline.kt | 8 +++++++ .../dice/proposition/gate/ObservableGate.kt | 19 ++++++++++++---- .../dice/proposition/gate/StandardGates.kt | 22 +++++++++++++++++++ 8 files changed, 96 insertions(+), 9 deletions(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt index 7be71549..ff9907cc 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt @@ -18,6 +18,7 @@ package com.embabel.dice.projection.memory import com.embabel.agent.core.ContextId import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionRepository +import org.slf4j.LoggerFactory /** * A [CollectorStrategy] that marks propositions whose decayed confidence has fallen below a @@ -38,14 +39,19 @@ class DecayCollectorStrategy @JvmOverloads constructor( private val retireDecayK: Double = 2.0, ) : CollectorStrategy { + private val logger = LoggerFactory.getLogger(DecayCollectorStrategy::class.java) + override fun mark( candidates: List, repository: PropositionRepository, contextId: ContextId, - ): List = - candidates + ): List { + val marks = candidates .filter { it.effectiveConfidence(retireDecayK) < retireBelow } .map { PropositionMark(propositionId = it.id, reason = MarkReason.Stale, strategyName = STRATEGY_NAME) } + logger.debug("DecayCollectorStrategy: marked {} of {} candidates as stale (threshold={})", marks.size, candidates.size, retireBelow) + return marks + } companion object { private const val STRATEGY_NAME = "decay" diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt index c0b41666..a635958d 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt @@ -63,6 +63,7 @@ class DefaultCollectorRunner( override fun collect(contextId: ContextId): CollectorRunResult { val startedAt = Instant.now() val (_, marks) = markPhase(contextId) + logger.debug("collect (read-only): {} mark(s) produced for context {}", marks.size, contextId) // Pure-read: no repository write, no run record. Nothing is persisted, so there is no run // to cross-reference — the runId is blank to signal it is not queryable in any store. return CollectorRunResult( @@ -80,6 +81,10 @@ class DefaultCollectorRunner( val startedAt = Instant.now() val runId = newRunId() val (candidatesById, marks) = markPhase(contextId) + logger.info( + "Collector run {} started for context {} (dryRun={}, candidates={}, marks={})", + runId, contextId, dryRun, candidatesById.size, marks.size, + ) val marksByProposition = marks.groupBy { it.propositionId } val applied = mutableListOf() @@ -124,7 +129,7 @@ class DefaultCollectorRunner( val finishedAt = Instant.now() persistRun(runId, startedAt, finishedAt, dryRun, records) - return CollectorRunResult( + val result = CollectorRunResult( runId = runId, dryRun = dryRun, marks = marks, @@ -134,6 +139,11 @@ class DefaultCollectorRunner( startedAt = startedAt, finishedAt = finishedAt, ) + logger.info( + "Collector run {} complete: applied={} skipped={} hardDeleted={} (dryRun={})", + runId, result.applied.size, result.skipped.size, result.hardDeleted.size, dryRun, + ) + return result } /** diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt index 50591951..fc385f8e 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt @@ -116,7 +116,7 @@ data class DefaultDreamLoopOrchestrator( // (6) Build the report. val transitioned = changed.sumOf { it.propositionsToSave.size + it.propositionsToDelete.size } val newPropositions = toSave.size - return DreamLoopReport( + val report = DreamLoopReport( contextId = contextId, cycleStarted = cycleStarted, passResults = passResults, @@ -125,6 +125,11 @@ data class DefaultDreamLoopOrchestrator( totalNewPropositions = newPropositions, triggered = true, ) + logger.info( + "Dream-loop cycle complete for {}: examined={} transitioned={} newPropositions={}", + contextId, report.totalExamined, report.totalTransitioned, report.totalNewPropositions, + ) + return report } /** diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt index b259f769..f3a5c9df 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt @@ -24,6 +24,9 @@ import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionQuery import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.proposition.PropositionStatus +import org.slf4j.LoggerFactory + +private val logger = LoggerFactory.getLogger("com.embabel.dice.projection.memory.DefaultMemoryMaintenanceOrchestrator") /** * Default [MemoryMaintenanceOrchestrator]: runs consolidation, abstraction, retirement, and @@ -79,6 +82,17 @@ data class DefaultMemoryMaintenanceOrchestrator( // Default off; when null the behavior is identical to the three-phase pipeline. val collectorResult = collector?.run(contextId) + logger.info( + "Maintenance complete for {}: promoted={} reinforced={} merged={} abstractions={} superseded={} retired={} collectorApplied={}", + contextId, + consolidation?.promoted?.size ?: 0, + consolidation?.reinforced?.size ?: 0, + consolidation?.merged?.size ?: 0, + abstractions.size, + superseded.size, + retired.size, + collectorResult?.applied?.size ?: 0, + ) return MaintenanceResult( consolidation = consolidation, abstractions = abstractions, @@ -196,6 +210,9 @@ data class DefaultMemoryMaintenanceOrchestrator( repository.delete(prop.id) } + if (toRetire.isNotEmpty()) { + logger.debug("Retired (hard-deleted) {} proposition(s) below confidence {} for {}", toRetire.size, threshold, contextId) + } return toRetire } diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategy.kt index 40ee2dfa..2ccde537 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategy.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategy.kt @@ -20,6 +20,7 @@ import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionQuery import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.proposition.PropositionStatus +import org.slf4j.LoggerFactory /** * A [CollectorStrategy] that finds near-duplicate propositions and marks all but the strongest @@ -46,6 +47,8 @@ class DuplicateCollectorStrategy @JvmOverloads constructor( private val topK: Int = 10, ) : CollectorStrategy { + private val logger = LoggerFactory.getLogger(DuplicateCollectorStrategy::class.java) + override fun mark( candidates: List, repository: PropositionRepository, @@ -76,7 +79,7 @@ class DuplicateCollectorStrategy @JvmOverloads constructor( .mapNotNull { id -> byId[id] } .groupBy { unionFind.find(it.id) } - return componentMembers.values + val marks = componentMembers.values .filter { it.size >= 2 } .flatMap { members -> // Global survivor per component: max effectiveConfidence, then reinforceCount, @@ -98,6 +101,11 @@ class DuplicateCollectorStrategy @JvmOverloads constructor( // proposition is never marked more than once across overlapping clusters. .distinctBy { it.propositionId } .sortedBy { it.propositionId } + logger.debug( + "DuplicateCollectorStrategy: {} cluster(s) -> {} duplicate mark(s) from {} candidate(s)", + clusters.size, marks.size, candidates.size, + ) + return marks } /** diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGatePipeline.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGatePipeline.kt index 048cd4c3..21fc42ca 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGatePipeline.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGatePipeline.kt @@ -16,6 +16,7 @@ package com.embabel.dice.proposition.gate import com.embabel.dice.proposition.Proposition +import org.slf4j.LoggerFactory /** * A standalone runner that applies an ordered list of [ExtractionGate]s to propositions and @@ -71,6 +72,8 @@ class ExtractionGatePipeline @JvmOverloads constructor( val shortCircuitOnReject: Boolean = true, ) { + private val logger = LoggerFactory.getLogger(ExtractionGatePipeline::class.java) + /** * The gates to run, in convention order. Snapshotted at construction so that mutating an * aliased backing list a caller passed in cannot change the pipeline's iteration afterwards. @@ -99,10 +102,15 @@ class ExtractionGatePipeline @JvmOverloads constructor( // First non-Persist decision wins; never overwrite it with a later decision. if (finalDecision is GateDecision.Persist && evaluation.decision !is GateDecision.Persist) { finalDecision = evaluation.decision + logger.debug( + "Gate '{}' set final decision to {} for proposition {}", + evaluation.gateName, finalDecision::class.simpleName, proposition.id.take(8), + ) } // Short-circuit on a hard reject AFTER recording the evaluation that produced it. if (shortCircuitOnReject && evaluation.decision is GateDecision.Reject) { + logger.debug("Short-circuiting gate pipeline on Reject from '{}' for proposition {}", evaluation.gateName, proposition.id.take(8)) break } } diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ObservableGate.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ObservableGate.kt index d7f1e4b2..5bd7dc71 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ObservableGate.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ObservableGate.kt @@ -22,6 +22,7 @@ import com.embabel.dice.common.PropositionRejected import com.embabel.dice.common.PropositionRoutedToReview import com.embabel.dice.common.SafeDiceEventListener import com.embabel.dice.proposition.Proposition +import org.slf4j.LoggerFactory /** * Decorator that adds event emission to any [ExtractionGate]. @@ -57,6 +58,8 @@ class ObservableGate( private val listener: DiceEventListener = DiceEventListener.DEV_NULL, ) : ExtractionGate { + private val logger = LoggerFactory.getLogger(ObservableGate::class.java) + override fun evaluate( proposition: Proposition, context: GateContext, @@ -64,17 +67,25 @@ class ObservableGate( val evaluation = delegate.evaluate(proposition, context) when (val decision = evaluation.decision) { - is GateDecision.Reject -> + is GateDecision.Reject -> { + logger.debug("Emitting PropositionRejected for {}: {}", proposition.id.take(8), decision.reason) listener.onEvent(PropositionRejected(proposition, decision.reason)) + } - is GateDecision.RouteToReview -> + is GateDecision.RouteToReview -> { + logger.debug("Emitting PropositionRoutedToReview for {}: {}", proposition.id.take(8), decision.reason) listener.onEvent(PropositionRoutedToReview(proposition, decision.reason)) + } - is GateDecision.SkipProjection -> + is GateDecision.SkipProjection -> { + logger.debug("Emitting PropositionProjectionSkipped for {}: {}", proposition.id.take(8), decision.reason) listener.onEvent(PropositionProjectionSkipped(proposition, decision.reason)) + } - is GateDecision.Demote -> + is GateDecision.Demote -> { + logger.debug("Emitting PropositionDemoted for {}: demote to '{}', reason: {}", proposition.id.take(8), decision.toRelation, decision.reason) listener.onEvent(PropositionDemoted(proposition, decision.toRelation, decision.reason)) + } // The durable persist signal fires at the save boundary; emitting here would double-emit it. is GateDecision.Persist -> Unit diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/StandardGates.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/StandardGates.kt index 0c16b1a6..b2e789b8 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/StandardGates.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/StandardGates.kt @@ -21,6 +21,7 @@ import com.embabel.dice.common.StructuralAuthorityResolver import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionStatus import com.embabel.dice.proposition.revision.RevisionResult +import org.slf4j.LoggerFactory /** * Routes a proposition based purely on its own confidence: anything below [minConfidence] is @@ -38,6 +39,8 @@ import com.embabel.dice.proposition.revision.RevisionResult */ class ConfidenceGate(private val minConfidence: Double) : ExtractionGate { + private val logger = LoggerFactory.getLogger(ConfidenceGate::class.java) + init { require(minConfidence in 0.0..1.0) { "minConfidence must be in 0.0..1.0, was $minConfidence" } } @@ -52,6 +55,7 @@ class ConfidenceGate(private val minConfidence: Double) : ExtractionGate { } else { GateDecision.Reject("effective confidence $effectiveConfidence < $minConfidence") } + logger.debug("ConfidenceGate: {} for proposition {} (confidence={})", decision::class.simpleName, proposition.id.take(8), effectiveConfidence) return GateEvaluation(GATE_NAME, proposition, decision) } @@ -70,6 +74,8 @@ class ConfidenceGate(private val minConfidence: Double) : ExtractionGate { */ class MergeCandidateGate : ExtractionGate { + private val logger = LoggerFactory.getLogger(MergeCandidateGate::class.java) + override fun evaluate( proposition: Proposition, context: GateContext, @@ -82,6 +88,7 @@ class MergeCandidateGate : ExtractionGate { // null (not revised) and all other outcomes persist — fail open. else -> GateDecision.Persist } + logger.debug("MergeCandidateGate: {} for proposition {} (revisionResult={})", decision::class.simpleName, proposition.id.take(8), context.revisionResult?.let { it::class.simpleName }) return GateEvaluation(GATE_NAME, proposition, decision) } @@ -102,6 +109,8 @@ class MergeCandidateGate : ExtractionGate { */ class ConflictClassificationGate : ExtractionGate { + private val logger = LoggerFactory.getLogger(ConflictClassificationGate::class.java) + override fun evaluate( proposition: Proposition, context: GateContext, @@ -112,6 +121,7 @@ class ConflictClassificationGate : ExtractionGate { // null (not revised) and all other outcomes persist — fail open. else -> GateDecision.Persist } + logger.debug("ConflictClassificationGate: {} for proposition {}", decision::class.simpleName, proposition.id.take(8)) return GateEvaluation(GATE_NAME, proposition, decision) } @@ -139,6 +149,8 @@ class TrustGate @JvmOverloads constructor( private val onMissingScore: GateDecision = GateDecision.Persist, ) : ExtractionGate { + private val logger = LoggerFactory.getLogger(TrustGate::class.java) + init { require(minTrustScore in 0.0..1.0) { "minTrustScore must be in 0.0..1.0, was $minTrustScore" } } @@ -153,6 +165,7 @@ class TrustGate @JvmOverloads constructor( trustScore >= minTrustScore -> GateDecision.Persist else -> GateDecision.RouteToReview("trust score $trustScore < $minTrustScore") } + logger.debug("TrustGate: {} for proposition {} (trustScore={})", decision::class.simpleName, proposition.id.take(8), trustScore) return GateEvaluation(GATE_NAME, proposition, decision) } @@ -177,6 +190,8 @@ class ProjectionEligibilityGate @JvmOverloads constructor( private val minConfidence: Double = DEFAULT_MIN_PROJECTION_CONFIDENCE, ) : ExtractionGate { + private val logger = LoggerFactory.getLogger(ProjectionEligibilityGate::class.java) + init { require(minConfidence in 0.0..1.0) { "minConfidence must be in 0.0..1.0, was $minConfidence" } } @@ -193,6 +208,7 @@ class ProjectionEligibilityGate @JvmOverloads constructor( GateDecision.SkipProjection("status is ${PropositionStatus.CONTRADICTED}") else -> GateDecision.Persist } + logger.debug("ProjectionEligibilityGate: {} for proposition {} (confidence={}, status={})", decision::class.simpleName, proposition.id.take(8), effectiveConfidence, proposition.status) return GateEvaluation(GATE_NAME, proposition, decision) } @@ -234,6 +250,8 @@ class EvidenceFloorGate @JvmOverloads constructor( private val caseSensitive: Boolean = false, ) : ExtractionGate { + private val logger = LoggerFactory.getLogger(EvidenceFloorGate::class.java) + override fun evaluate( proposition: Proposition, context: GateContext, @@ -260,6 +278,10 @@ class EvidenceFloorGate @JvmOverloads constructor( } } } + logger.debug( + "EvidenceFloorGate: {} for proposition {} (relation={})", + decision::class.simpleName, proposition.id.take(8), relation?.predicate, + ) return GateEvaluation(GATE_NAME, proposition, decision) } From 8c5421b0a7b928f9f1b6b7d0d81c1e90597c7293 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 12:26:22 -0400 Subject: [PATCH 07/27] refactor(spi): extract policy SPIs into com.embabel.dice.spi and fix common layering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the policy extension points — TrustScorer, AuthorityResolver/AuthorityTier, AuthorityWeightedTrustScorer, ConflictDetector, and ConflictType (plus their shipped defaults) — out of common and proposition.revision into a dedicated com.embabel.dice.spi package, giving the library's extension points one home. This also fixes a layering wrinkle: common no longer imports proposition.revision.ConflictType. ConflictType now lives in spi alongside the policies that share it, and proposition.revision depends downward on spi. Tests mirror the move into the spi package; no behaviour change. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- dice/src/main/kotlin/com/embabel/dice/common/EvidenceFloor.kt | 2 ++ .../kotlin/com/embabel/dice/proposition/gate/StandardGates.kt | 4 ++-- .../embabel/dice/proposition/gate/EvidenceFloorGateTest.kt | 4 ++-- .../dice/proposition/gate/GatePipelineConsumerFlowTest.kt | 4 ++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/common/EvidenceFloor.kt b/dice/src/main/kotlin/com/embabel/dice/common/EvidenceFloor.kt index 377fb329..becc9afb 100644 --- a/dice/src/main/kotlin/com/embabel/dice/common/EvidenceFloor.kt +++ b/dice/src/main/kotlin/com/embabel/dice/common/EvidenceFloor.kt @@ -15,6 +15,8 @@ */ package com.embabel.dice.common +import com.embabel.dice.spi.AuthorityTier + /** * The minimum evidence a proposition must carry before a relation may be asserted at full strength. * diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/StandardGates.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/StandardGates.kt index b2e789b8..449453e4 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/StandardGates.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/StandardGates.kt @@ -15,9 +15,9 @@ */ package com.embabel.dice.proposition.gate -import com.embabel.dice.common.AuthorityResolver +import com.embabel.dice.spi.AuthorityResolver import com.embabel.dice.common.Relations -import com.embabel.dice.common.StructuralAuthorityResolver +import com.embabel.dice.spi.StructuralAuthorityResolver import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionStatus import com.embabel.dice.proposition.revision.RevisionResult diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/gate/EvidenceFloorGateTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/EvidenceFloorGateTest.kt index 7c6801ac..78eb708c 100644 --- a/dice/src/test/kotlin/com/embabel/dice/proposition/gate/EvidenceFloorGateTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/EvidenceFloorGateTest.kt @@ -16,9 +16,9 @@ package com.embabel.dice.proposition.gate import com.embabel.agent.core.ContextId -import com.embabel.dice.common.AuthorityTier +import com.embabel.dice.spi.AuthorityTier import com.embabel.dice.common.EvidenceFloor -import com.embabel.dice.common.FixedAuthorityResolver +import com.embabel.dice.spi.FixedAuthorityResolver import com.embabel.dice.common.Relation import com.embabel.dice.common.Relations import com.embabel.dice.proposition.Proposition diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/gate/GatePipelineConsumerFlowTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/GatePipelineConsumerFlowTest.kt index c3f87137..77d41e8d 100644 --- a/dice/src/test/kotlin/com/embabel/dice/proposition/gate/GatePipelineConsumerFlowTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/gate/GatePipelineConsumerFlowTest.kt @@ -16,9 +16,9 @@ package com.embabel.dice.proposition.gate import com.embabel.agent.core.ContextId -import com.embabel.dice.common.AuthorityTier +import com.embabel.dice.spi.AuthorityTier import com.embabel.dice.common.EvidenceFloor -import com.embabel.dice.common.FixedAuthorityResolver +import com.embabel.dice.spi.FixedAuthorityResolver import com.embabel.dice.common.PropositionDemoted import com.embabel.dice.common.PropositionProjectionSkipped import com.embabel.dice.common.PropositionRejected From 1e2e6274b4893fa493080af9a5ba9d06d919e325 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 22:58:21 -0400 Subject: [PATCH 08/27] fix(rest): persist revised propositions and harden extract endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The extract endpoints saved only the freshly extracted propositions, so when revision contradicted an existing proposition the retired original (reduced confidence, CONTRADICTED status) was never written — it stayed ACTIVE at full confidence. Persist propositionsToPersist() instead, which includes revised originals. Also: parse and apply the knownEntities part on file uploads (it was bound then dropped), surface failed chunk ids in the file response instead of an empty list, and reject blank extract text with 400 rather than calling the LLM. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../web/rest/PropositionPipelineController.kt | 30 ++++++++-- .../rest/PropositionPipelineControllerTest.kt | 55 +++++++++++++++++++ 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt b/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt index bbc08423..8bb59494 100644 --- a/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt +++ b/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt @@ -33,6 +33,9 @@ import com.embabel.dice.pipeline.ChunkPropositionResult import com.embabel.dice.pipeline.PropositionPipeline import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.proposition.revision.RevisionResult +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnBean import org.springframework.http.MediaType @@ -65,6 +68,7 @@ class PropositionPipelineController( config = ContentChunker.Config(), chunkTransformer = ChunkTransformer.NO_OP, ), + private val objectMapper: ObjectMapper = jacksonObjectMapper(), ) { private val logger = LoggerFactory.getLogger(PropositionPipelineController::class.java) @@ -82,6 +86,11 @@ class PropositionPipelineController( ): ResponseEntity { logger.info("Extracting propositions for context: {}", contextId) + if (request.text.isBlank()) { + logger.warn("Rejecting extract request for context {}: blank text", contextId) + return ResponseEntity.badRequest().build() + } + val chunk = Chunk.create( text = request.text, parentId = request.sourceId ?: "api-request", @@ -90,8 +99,9 @@ class PropositionPipelineController( val context = buildContext(contextId, request.knownEntities, request.schemaName) val result = propositionPipeline.processChunk(chunk, context) - // Save propositions - result.propositions.forEach { proposition -> + // Persist what revision says to keep — both the freshly extracted propositions and any + // revised originals (e.g. an existing proposition retired to CONTRADICTED), not just the new ones. + result.propositionsToPersist().forEach { proposition -> propositionRepository.save(proposition) } @@ -141,12 +151,12 @@ class PropositionPipelineController( } // Process each chunk through the pipeline - val context = buildContext(contextId, emptyList(), schemaName) + val context = buildContext(contextId, parseKnownEntities(knownEntitiesJson), schemaName) val chunkResults = chunks.map { chunk -> val result = propositionPipeline.processChunk(chunk, context) - // Save propositions - result.propositions.forEach { proposition -> + // Persist revised propositions too (e.g. a CONTRADICTED original), not just the new ones. + result.propositionsToPersist().forEach { proposition -> propositionRepository.save(proposition) } @@ -205,7 +215,7 @@ class PropositionPipelineController( entities = EntitySummary( created = createdIds, resolved = resolvedIds, - failed = emptyList(), + failed = chunkResults.filterIsInstance().map { it.chunkId }, ), revision = revisionSummary, ) @@ -218,6 +228,14 @@ class PropositionPipelineController( return ResponseEntity.ok(response) } + /** + * Parse the optional `knownEntities` multipart part — a JSON array of [KnownEntityDto] — into a + * list. A blank or absent part yields an empty list. This mirrors the JSON `/extract` endpoint, + * which binds `knownEntities` directly from the request body. + */ + private fun parseKnownEntities(json: String?): List = + if (json.isNullOrBlank()) emptyList() else objectMapper.readValue(json) + private fun buildContext( contextId: String, knownEntityDtos: List, diff --git a/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt b/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt index 5d274c29..979f8409 100644 --- a/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt @@ -26,6 +26,7 @@ import com.embabel.dice.common.support.InMemorySchemaRegistry import com.embabel.dice.pipeline.ChunkPropositionResult import com.embabel.dice.pipeline.PropositionPipeline import com.embabel.dice.proposition.* +import com.embabel.dice.proposition.revision.RevisionResult import com.fasterxml.jackson.annotation.JsonClassDescription import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule @@ -200,4 +201,58 @@ class PropositionPipelineControllerTest { val saved = propositionRepository.findAll().first() assert(saved.text == "Wagner composed Tristan") } + + @Test + fun `POST extract persists the contradicted original, not just the new proposition`() { + val contextId = "test-context" + val requestBody = """{"text": "Brahms was born in 1833", "sourceId": "c1"}""" + + // A revision that contradicts an existing proposition: the original is returned with + // reduced confidence and CONTRADICTED status, separate from the newly extracted one. + val original = Proposition( + id = "orig-1", + contextId = ContextId(contextId), + text = "Brahms was born in 1830", + mentions = listOf(EntityMention("Brahms", "Composer", "composer-brahms", MentionRole.SUBJECT)), + confidence = 0.3, + status = PropositionStatus.CONTRADICTED, + ) + val newProp = Proposition( + contextId = ContextId(contextId), + text = "Brahms was born in 1833", + mentions = listOf(EntityMention("Brahms", "Composer", "composer-brahms", MentionRole.SUBJECT)), + confidence = 0.95, + ) + val mockResult = ChunkPropositionResult.Success( + chunkId = "chunk-1", + suggestedPropositions = SuggestedPropositions(chunkId = "chunk-1", propositions = emptyList()), + entityResolutions = Resolutions(chunkIds = setOf("chunk-1"), resolutions = emptyList()), + // `propositions` carries only the new one; the original lives in the revision result. + propositions = listOf(newProp), + revisionResults = listOf(RevisionResult.Contradicted(original = original, new = newProp)), + ) + + every { propositionPipeline.processChunk(any(), any()) } returns mockResult + + mockMvc.perform( + post("/api/v1/contexts/$contextId/extract") + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody) + ).andExpect(status().isOk) + + // Both the new proposition AND the retired original must be persisted. + assert(propositionRepository.count() == 2) { "expected new + contradicted original to be saved" } + val saved = propositionRepository.findAll().associateBy { it.id } + assert(saved.containsKey("orig-1")) { "the contradicted original must be persisted" } + assert(saved["orig-1"]!!.status == PropositionStatus.CONTRADICTED) + } + + @Test + fun `POST extract rejects blank text with 400`() { + mockMvc.perform( + post("/api/v1/contexts/test-context/extract") + .contentType(MediaType.APPLICATION_JSON) + .content("""{"text": " "}""") + ).andExpect(status().isBadRequest) + } } From 1951c00e2e4a82f84cc3a34911dbb94263b348ac Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 22:58:28 -0400 Subject: [PATCH 09/27] fix(consolidation): resolve each contradiction pair once with a correct tie-break A symmetric reviser reports the same conflict from both sides, so the pass retired both members and left no survivor. Resolve each unordered pair only once. Also change the tie comparison from <= to <, so a tie genuinely favors the proposition being classified (as the KDoc states) rather than retiring it. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../ContradictionResolutionPass.kt | 11 ++++++- .../ContradictionResolutionPassTest.kt | 29 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt index 6fde0638..e829efbc 100644 --- a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt @@ -53,14 +53,23 @@ class ContradictionResolutionPass @JvmOverloads constructor( return try { val active = propositions.filter { it.status == PropositionStatus.ACTIVE } val toSave = mutableListOf() + // Each unordered pair is resolved once: a symmetric reviser reports the same conflict + // from both sides, and resolving both would retire both members and leave no survivor. + val resolvedPairs = mutableSetOf>() for (p in active) { val candidates = active.filter { it.id != p.id && sharesEntity(it, p) } val classified = reviser.classify(p, candidates) classified .filter { it.relation == PropositionRelation.CONTRADICTORY } .forEach { c -> + val pairKey = + if (p.id <= c.proposition.id) p.id to c.proposition.id + else c.proposition.id to p.id + if (!resolvedPairs.add(pairKey)) return@forEach + // The lower effective confidence loses; a tie favors the proposition being + // classified (p), so only the candidate is retired in that case. val weaker = - if (p.effectiveConfidence() <= c.proposition.effectiveConfidence()) p + if (p.effectiveConfidence() < c.proposition.effectiveConfidence()) p else c.proposition if (weaker.status == PropositionStatus.ACTIVE) { // withStatus preserves the contentRevised decay anchor. diff --git a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt index 0706dac6..39f940f7 100644 --- a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt @@ -29,6 +29,7 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertInstanceOf import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import java.time.Instant class ContradictionResolutionPassTest { @@ -39,6 +40,7 @@ class ContradictionResolutionPassTest { entityId: String, confidence: Double, status: PropositionStatus = PropositionStatus.ACTIVE, + contentRevised: Instant = Instant.now(), ): Proposition = Proposition( id = id, @@ -48,6 +50,7 @@ class ContradictionResolutionPassTest { confidence = confidence, decay = 0.1, status = status, + contentRevised = contentRevised, ) @Test @@ -92,6 +95,32 @@ class ContradictionResolutionPassTest { assertEquals(PropositionStatus.CONTRADICTED, transitioned.status) } + @Test + fun `equal-confidence symmetric contradiction retires exactly one, never both`() { + // Same effective confidence (same confidence, decay, and content anchor) and a symmetric + // reviser that reports the conflict from both sides. The pair must resolve to exactly one + // survivor — the bug was retiring both, leaving no survivor. A tie favors the proposition + // being classified, so 'a' (the outer-loop subject seen first) survives and 'b' is retired. + val anchor = Instant.now() + val a = proposition("a", "bob", 0.5, contentRevised = anchor) + val b = proposition("b", "bob", 0.5, contentRevised = anchor) + val reviser = mockk() + every { reviser.classify(a, listOf(b)) } returns listOf( + ClassifiedProposition(b, PropositionRelation.CONTRADICTORY, 0.5), + ) + every { reviser.classify(b, listOf(a)) } returns listOf( + ClassifiedProposition(a, PropositionRelation.CONTRADICTORY, 0.5), + ) + + val result = ContradictionResolutionPass(reviser).run(contextId, listOf(a, b)) + + val changed = assertInstanceOf(ConsolidationPassResult.Changed::class.java, result) + assertEquals(1, changed.propositionsToSave.size) + val transitioned = changed.propositionsToSave.single() + assertEquals("b", transitioned.id) + assertEquals(PropositionStatus.CONTRADICTED, transitioned.status) + } + @Test fun `no contradictions yields NoOp`() { val a = proposition("a", "bob", 0.9) From aaa81f94ea872877d7b8c0a0bc7e25a6cd5bf7ac Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 22:58:45 -0400 Subject: [PATCH 10/27] fix(pipeline): let concurrent extraction strategies own and close their pool A default-constructed ParallelExtractionStrategy/BatchedExtractionStrategy created a non-daemon thread pool in a private field with no way to shut it down, so the threads leaked. Both strategies are now AutoCloseable: close() shuts down a pool they created themselves and leaves a caller-supplied executor untouched. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../pipeline/ExtractionExecutionStrategy.kt | 63 ++++++++++++++----- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategy.kt index a349aa45..9ceba359 100644 --- a/dice/src/main/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategy.kt +++ b/dice/src/main/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategy.kt @@ -35,8 +35,10 @@ import java.util.concurrent.Executors * ## Executor ownership * Each implementation owns its execution mechanism. The [execute] signature intentionally * does not mention [ExecutorService] — any executor is an injected implementation detail. - * The executor lifecycle is always the caller's responsibility; no strategy ever calls - * `executor.shutdown()`. + * A caller-supplied executor is never shut down by the strategy — its lifecycle stays the + * caller's. When a strategy creates its own pool (the no-executor constructors), that pool is + * the strategy's to clean up: both concurrent strategies are [AutoCloseable], and [close] + * shuts down only an internally-created pool. * * ## Thread-safety gate for production * The default is [SerialExtractionStrategy] (no concurrency). Before switching to @@ -71,17 +73,30 @@ object SerialExtractionStrategy : ExtractionExecutionStrategy { } /** - * Fans all chunks out concurrently on the injected [executor] via [CompletableFuture], then - * joins them in input order. A failed extraction yields a `null` slot. + * Fans all chunks out concurrently on the [executor] via [CompletableFuture], then joins them in + * input order. A failed extraction yields a `null` slot. * - * The [executor] is caller-owned — this class never calls [ExecutorService.shutdown]. + * A caller-supplied executor is never shut down here; a pool created by the no-arg constructor is + * shut down by [close]. * * Production use requires verifying extractor thread-safety first (see [ExtractionExecutionStrategy]). */ -class ParallelExtractionStrategy @JvmOverloads constructor( - private val executor: ExecutorService = +class ParallelExtractionStrategy private constructor( + private val executor: ExecutorService, + private val ownsExecutor: Boolean, +) : ExtractionExecutionStrategy, AutoCloseable { + + /** Use a caller-supplied [executor]; [close] will not shut it down. */ + constructor(executor: ExecutorService) : this(executor, ownsExecutor = false) + + /** + * Create an internal fixed pool sized to the available processors. The pool is owned by this + * strategy, so call [close] (or use it as a resource) when you are done with it. + */ + constructor() : this( Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()), -) : ExtractionExecutionStrategy { + ownsExecutor = true, + ) override fun execute(chunks: List, extract: (Chunk) -> T): List = chunks.map { chunk -> @@ -89,6 +104,11 @@ class ParallelExtractionStrategy @JvmOverloads constructor( runCatching { extract(chunk) }.getOrElse { null } }, executor) }.map { it.join() } + + /** Shuts down an internally-created pool; a no-op when the executor was caller-supplied. */ + override fun close() { + if (ownsExecutor) executor.shutdown() + } } /** @@ -97,20 +117,30 @@ class ParallelExtractionStrategy @JvmOverloads constructor( * it caps in-flight `extract()` calls to [batchSize] at a time. Results are returned in input order. * * `BatchedExtractionStrategy(batchSize = 1)` is the safe degrade-to-serial path until extractor - * thread-safety is verified (see [ExtractionExecutionStrategy]). The [executor] is caller-owned — - * this class never calls [ExecutorService.shutdown]. + * thread-safety is verified (see [ExtractionExecutionStrategy]). A caller-supplied executor is + * never shut down here; a pool created by the no-executor constructor is shut down by [close]. * * @param batchSize maximum concurrent extractions per window; must be positive */ -class BatchedExtractionStrategy @JvmOverloads constructor( - private val batchSize: Int = 5, - private val executor: ExecutorService = Executors.newFixedThreadPool(batchSize), -) : ExtractionExecutionStrategy { +class BatchedExtractionStrategy private constructor( + private val batchSize: Int, + private val executor: ExecutorService, + private val ownsExecutor: Boolean, +) : ExtractionExecutionStrategy, AutoCloseable { init { require(batchSize > 0) { "batchSize must be positive" } } + /** Use a caller-supplied [executor]; [close] will not shut it down. */ + constructor(batchSize: Int = 5, executor: ExecutorService) : this(batchSize, executor, ownsExecutor = false) + + /** + * Create an internal fixed pool sized to [batchSize]. The pool is owned by this strategy, so + * call [close] (or use it as a resource) when you are done with it. + */ + constructor(batchSize: Int = 5) : this(batchSize, Executors.newFixedThreadPool(batchSize), ownsExecutor = true) + override fun execute(chunks: List, extract: (Chunk) -> T): List = chunks.chunked(batchSize).flatMap { window -> window.map { chunk -> @@ -119,4 +149,9 @@ class BatchedExtractionStrategy @JvmOverloads constructor( }, executor) }.map { it.join() } } + + /** Shuts down an internally-created pool; a no-op when the executor was caller-supplied. */ + override fun close() { + if (ownsExecutor) executor.shutdown() + } } From 8979c903dd18bf564ffeb52b227f07f946f1cc6d Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 22:58:45 -0400 Subject: [PATCH 11/27] fix(gate): add shouldSave and correct the confidence-gate example shouldPersist is true only for a plain Persist decision, so a consumer guarding save() with it silently dropped Demote and SkipProjection results, which both mean keep the proposition. Add shouldSave covering every non-reject, non-review decision. Also fix the KDoc example to read effectiveConfidence() rather than the raw confidence, matching the real ConfidenceGate. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../dice/proposition/gate/ExtractionGate.kt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGate.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGate.kt index 0aa67a23..3b76c551 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGate.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/gate/ExtractionGate.kt @@ -35,7 +35,9 @@ import com.embabel.dice.proposition.revision.RevisionResult * * // A simple confidence gate. * val confidenceGate = ExtractionGate { proposition, _ -> - * val decision = if (proposition.confidence < 0.5) { + * // Use effectiveConfidence() (decay-adjusted), not the raw extraction-time confidence, + * // so the gate threshold stays consistent with the query layer. + * val decision = if (proposition.effectiveConfidence() < 0.5) { * GateDecision.Reject("confidence below threshold") * } else { * GateDecision.Persist @@ -154,9 +156,18 @@ data class GatedPropositionResult( val evaluations: List, val finalDecision: GateDecision, ) { - /** True when the proposition should be persisted normally. */ + /** True when the proposition should be persisted normally (a plain [GateDecision.Persist]). */ val shouldPersist: Boolean get() = finalDecision is GateDecision.Persist + /** + * True when the proposition should be written to storage at all — that is, every decision + * except [GateDecision.Reject] and [GateDecision.RouteToReview]. Use this (not [shouldPersist]) + * to guard a `save()`, since [GateDecision.Demote] and [GateDecision.SkipProjection] both keep + * the proposition; they only change how it is projected. + */ + val shouldSave: Boolean + get() = finalDecision !is GateDecision.Reject && finalDecision !is GateDecision.RouteToReview + /** True when the proposition should be routed to review. */ val shouldReview: Boolean get() = finalDecision is GateDecision.RouteToReview From acd59e33d98543b50cb9315fec688ca8ca30579b Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 22:58:45 -0400 Subject: [PATCH 12/27] fix(collector): record dry-run sweeps as MARKED, not as applied outcomes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dry run recorded its preview rows with TRANSITIONED/HARD_DELETED outcomes — indistinguishable from a real sweep when querying records by outcome. Record them as MARKED (the purpose-built preview outcome). Also flag the pure-read collect() result as dryRun, since it applies nothing. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../memory/DefaultCollectorRunner.kt | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt index a635958d..9eba434f 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt @@ -65,10 +65,11 @@ class DefaultCollectorRunner( val (_, marks) = markPhase(contextId) logger.debug("collect (read-only): {} mark(s) produced for context {}", marks.size, contextId) // Pure-read: no repository write, no run record. Nothing is persisted, so there is no run - // to cross-reference — the runId is blank to signal it is not queryable in any store. + // to cross-reference — the runId is blank to signal it is not queryable in any store. It is + // flagged dryRun because, like a dry run, it applied no transition. return CollectorRunResult( runId = EPHEMERAL_RUN_ID, - dryRun = false, + dryRun = true, marks = marks, applied = emptyList(), skipped = emptyList(), @@ -97,9 +98,10 @@ class DefaultCollectorRunner( when (val action = policy.decide(proposition, propMarks)) { is SweepAction.TransitionStatus -> { if (dryRun) { - // Would-be transition: record it, but mutate nothing, emit nothing, and - // leave `applied` empty — `marks` already reflects what WOULD happen. - records += records(propMarks, runId, CollectorOutcome.TRANSITIONED, proposition.status, action.newStatus) + // Preview only: record the would-be transition as MARKED (nothing was swept), + // mutate nothing, emit nothing, and leave `applied` empty. The record's + // newStatus still carries what WOULD happen. + records += records(propMarks, runId, CollectorOutcome.MARKED, proposition.status, action.newStatus) } else { applyTransition(proposition, action.newStatus, propMarks) applied.addAll(propMarks) @@ -108,13 +110,15 @@ class DefaultCollectorRunner( } SweepAction.HardDelete -> { - if (!dryRun) { + if (dryRun) { + // Preview only: record what WOULD be removed as MARKED; delete nothing and + // leave `hardDeleted` empty. + records += records(propMarks, runId, CollectorOutcome.MARKED, proposition.status, null) + } else { repository.delete(proposition.id) - // Only an actually-removed proposition is reported as hard-deleted; on a - // dry run `hardDeleted` stays empty (the record still captures the preview). hardDeleted += proposition.id + records += records(propMarks, runId, CollectorOutcome.HARD_DELETED, proposition.status, null) } - records += records(propMarks, runId, CollectorOutcome.HARD_DELETED, proposition.status, null) } SweepAction.Skip -> { From e1e4b0a75e8765d1ab0c2765a885dee34f0aa090 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 22:58:45 -0400 Subject: [PATCH 13/27] fix(consolidation): correct dream-loop cycle totals totalNewPropositions counted re-saved snapshot members (e.g. SUPERSEDED sources) as new, and totalTransitioned missed decay transitions entirely because the decay pass writes through its own runner and returns empty save/delete lists. Count a proposition as new only when its id was absent from the examined snapshot, and let a pass report transitions it applied itself via a new Changed.externallyApplied count that the orchestrator folds into the transitioned total. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../operations/consolidation/ConsolidationPass.kt | 5 +++++ .../dice/operations/consolidation/DecaySweepPass.kt | 3 +++ .../projection/memory/DefaultDreamLoopOrchestrator.kt | 11 +++++++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ConsolidationPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ConsolidationPass.kt index bf1b62c4..b2adcb8b 100644 --- a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ConsolidationPass.kt +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ConsolidationPass.kt @@ -74,6 +74,10 @@ sealed class ConsolidationPassResult { * with `allowHardDelete`; otherwise ignored in favor of the soft-retire default, so a pass * can always express a delete intent without risking data loss. * @property skipped Count of snapshot propositions the pass deliberately left untouched. + * @property externallyApplied Count of transitions the pass already wrote itself rather than + * handing back via [propositionsToSave]/[propositionsToDelete]. Used by report-only passes + * (e.g. the decay sweep, which writes through its own runner) so cycle totals still reflect + * the work; zero for passes that return everything to the orchestrator. * @property summary Short description of what changed, for reports and logging. */ data class Changed @JvmOverloads constructor( @@ -81,6 +85,7 @@ sealed class ConsolidationPassResult { val propositionsToSave: List = emptyList(), val propositionsToDelete: List = emptyList(), val skipped: Int = 0, + val externallyApplied: Int = 0, val summary: String = "", ) : ConsolidationPassResult() diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPass.kt index 58cf9449..5bbb2bef 100644 --- a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPass.kt +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPass.kt @@ -87,6 +87,9 @@ class DecaySweepPass @JvmOverloads constructor( propositionsToSave = emptyList(), propositionsToDelete = emptyList(), skipped = result.skipped.size, + // The runner already wrote these transitions, so report them here rather than + // in the save/delete lists — otherwise cycle totals would miss the decay sweep. + externallyApplied = result.applied.size + result.hardDeleted.size, summary = "decay sweep: ${result.applied.size} -> STALE, ${result.hardDeleted.size} hard-deleted", ) } diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt index fc385f8e..f8ea6a3d 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt @@ -99,6 +99,7 @@ data class DefaultDreamLoopOrchestrator( val passResults = passes.map { pass -> runPass(pass, contextId, snapshot) } // (3) Aggregate saves into a single write. + val snapshotIds = snapshot.mapTo(HashSet()) { it.id } val changed = passResults.filterIsInstance() val toSave: List = changed.flatMap { it.propositionsToSave } if (toSave.isNotEmpty()) { @@ -114,8 +115,14 @@ data class DefaultDreamLoopOrchestrator( lastActiveCount[contextId] = activeSnapshot(contextId).size // (6) Build the report. - val transitioned = changed.sumOf { it.propositionsToSave.size + it.propositionsToDelete.size } - val newPropositions = toSave.size + // A "new" proposition is one whose id was not in the examined snapshot (e.g. a fresh + // abstraction); a re-saved snapshot member (e.g. a source marked SUPERSEDED, or a + // CONTRADICTED loser) is a transition, not a new proposition. Decay transitions are written + // by their own pass, so they are counted via externallyApplied rather than the save list. + val newPropositions = toSave.count { it.id !in snapshotIds } + val savedTransitions = toSave.count { it.id in snapshotIds } + val transitioned = savedTransitions + + changed.sumOf { it.propositionsToDelete.size + it.externallyApplied } val report = DreamLoopReport( contextId = contextId, cycleStarted = cycleStarted, From e0b1485bbec7c3b61aa88e220ee8e79a0d483b1d Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 23:02:50 -0400 Subject: [PATCH 14/27] docs(design): add the extraction pipeline design note Documents the part of the substrate that had no design note yet: the extraction pipeline. Covers why extraction is split into two phases, why only the resolver-free first phase may run concurrently while resolution stays serial, the execution-strategy SPI and its order-preserving / null-slot-on-failure contract, per-chunk failure isolation, and why the pipeline returns unsaved results for the caller to persist. Includes architecture, two-phase sequence, strategy, failure, and persistence diagrams, cross-linked to the existing lifecycle, hygiene, and events notes. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- docs/design/extraction-pipeline.md | 201 +++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 docs/design/extraction-pipeline.md diff --git a/docs/design/extraction-pipeline.md b/docs/design/extraction-pipeline.md new file mode 100644 index 00000000..a22a71bc --- /dev/null +++ b/docs/design/extraction-pipeline.md @@ -0,0 +1,201 @@ +# Extraction pipeline: two phases, optional concurrency, and unsaved results + +DICE turns raw text into grounded propositions through one pipeline. It chunks nothing and stores +nothing itself — it takes chunks in, extracts statements, resolves the entities those statements +mention, optionally reconciles them against what's already known, and hands back a result the +caller persists. This note is about three design decisions behind that flow: why extraction is +split into two phases, why only the first phase is allowed to run concurrently, and why the +pipeline deliberately writes nothing. + +The pipeline produces the propositions that everything downstream then governs — the admission +gates decide what to keep, the lifecycle decides how each one ages, and the dream loop tidies them +later (see [knowledge-hygiene](knowledge-hygiene.md) and +[proposition-lifecycle](proposition-lifecycle.md)). This is where they come from. + +```mermaid +flowchart LR + CHUNKS[Chunks] --> P1 + subgraph P1 [Phase 1 · extraction · concurrency-safe] + direction TB + EX["extract propositions
+ suggest entities"] + end + P1 --> P2 + subgraph P2 [Phase 2 · resolution · always serial] + direction TB + RES["resolve entities
through the shared resolver"] --> REV["optional revision
against the store"] + end + P2 --> RESULT["Unsaved result
(propositions + resolutions)"] + RESULT -.caller persists.-> STORE[(Proposition store)] +``` + +## Why two phases + +Extraction is two jobs wearing one coat. The first reads a chunk and produces *candidate* +statements and the entities they mention — pure work that depends only on the chunk in front of +it. The second takes those candidates and decides *which real entities* they refer to, which means +consulting a resolver that accumulates identity across the whole run. + +Splitting them is what makes safe concurrency possible at all. Phase 1 (`extractPhase`) touches no +resolver and shares no state between chunks, so two chunks can be extracted at the same time +without stepping on each other. Phase 2 (`resolvePhase`) runs every chunk through one shared +cross-chunk resolver so that "Brahms" mentioned in chunk 3 lands on the same entity as "Brahms" in +chunk 17 — and that shared, mutating identity map is exactly the thing you cannot touch from +several threads at once. So the line between the phases is also the line between "parallelizable" +and "must stay serial." + +```mermaid +sequenceDiagram + autonumber + participant Caller + participant Pipeline + participant Strategy as Execution strategy + participant Resolver as Shared cross-chunk resolver + Caller->>Pipeline: process(chunks, context) + Note over Pipeline,Strategy: Phase 1 — fan out, order preserved, a failure becomes a null slot + Pipeline->>Strategy: extract each chunk (resolver-free) + Strategy-->>Pipeline: per-chunk extractions (nulls where extraction threw) + Note over Pipeline,Resolver: Phase 2 — strictly serial, one shared entity identity + loop each extraction, in input order + alt extraction present + Pipeline->>Resolver: resolve entities, then optionally revise + Resolver-->>Pipeline: success result for the chunk + else null slot + Pipeline->>Pipeline: emit a typed Failed result for the chunk + end + end + Pipeline-->>Caller: aggregated, UNSAVED result +``` + +## Execution strategies + +How Phase 1 is dispatched is a pluggable choice, separated from the pipeline behind a small SPI so +that turning on concurrency is a one-line swap rather than a rewrite. Three strategies ship: + +- **Serial** (the default) runs each chunk one at a time on the calling thread. It is the exact + pre-concurrency behaviour, so the default pipeline changes nothing. +- **Parallel** fans every chunk out at once onto an executor and joins them back in order. +- **Batched** processes fixed-size windows: each window runs concurrently, then fully completes + before the next starts. This is the rate-limit lever — it caps how many extractions (and so how + many model calls) are in flight at a time. + +Every strategy honours the same two-part contract, and the contract is what lets the rest of the +pipeline stay simple. **Input order is preserved**: the result has one slot per input chunk, in +order. **A failure is a `null` slot, never an exception and never a dropped chunk** — when an +extraction throws, that slot comes back `null`, and Phase 2 turns it into a typed `Failed` result. +Nothing is silently reordered or lost, so the `result.size == chunks.size` invariant always holds. + +```mermaid +flowchart TB + subgraph serial [Serial · default] + direction TB + SA[chunk 0] --> SB[chunk 1] --> SC[chunk 2] + end + subgraph parallel [Parallel · all at once] + direction TB + PA[chunk 0] + PB[chunk 1] + PC[chunk 2] + end + subgraph batched [Batched · windows of N] + direction TB + BA["chunk 0 + 1
(window 1)"] --> BB["chunk 2 + 3
(window 2)"] + end +``` + +### Turning concurrency on is gated + +Going faster than serial is opt-in for a reason: it is only safe if the `PropositionExtractor` +behind the pipeline — usually a language-model client — is itself safe to call from several threads +at once. That is a property of the extractor, not something the pipeline can guarantee, so the +default stays serial and a deployment is expected to verify thread-safety before switching. Until +then, `BatchedExtractionStrategy(batchSize = 1)` is the documented degrade-to-serial path: it goes +through the concurrent code path but never actually overlaps two calls, producing byte-identical +output to the serial strategy. + +### Who owns the executor + +A strategy either borrows an executor or makes its own, and the rule is symmetric with ownership. A +caller-supplied executor is never shut down by the strategy — its lifecycle belongs to whoever +created it. A strategy that creates its own pool (the no-executor constructors) owns that pool, and +both concurrent strategies are `AutoCloseable` so `close()` shuts down a self-created pool and +leaves a borrowed one alone. The practical guidance: inject and manage your own executor in a +long-lived application; use the no-arg constructor as a `use { }` resource for a one-off. + +## Failure is isolated per chunk + +The batch path treats every chunk as independent, so one bad chunk can't sink a whole document. A +chunk that throws during extraction becomes a `null` slot; a chunk that throws during resolution, +revision, or even event emission is caught in Phase 2 and turned into a `Failed` result for that +chunk alone. Either way the run finishes and the failures are visible — `PropositionResults` +exposes the ids of chunks that failed, and the REST layer surfaces them rather than pretending +everything succeeded. + +Single-chunk processing is deliberately stricter. `processChunk` runs the two phases serially on +the calling thread and lets an extraction exception propagate, because a caller handling one chunk +wants the error, not a quietly empty result. Only the batch `process` converts failures into +`Failed` slots, because that is where swallowing one chunk to save the other forty is the right +trade. + +```mermaid +flowchart TB + C[Chunk] --> EX{Phase 1
extraction} + EX -->|throws| NULL[null slot] + EX -->|ok| RES{Phase 2
resolve / revise} + NULL --> FAILED["Failed result
(carries chunk id + cause)"] + RES -->|throws| FAILED + RES -->|ok| SUCCESS["Success result
(propositions + resolutions)"] + FAILED --> AGG[Aggregated results] + SUCCESS --> AGG +``` + +## The result, and who persists it + +The pipeline returns **unsaved** results on purpose. `process`, `processChunk`, and `processOnce` +all write nothing to any repository; the returned value is discarded if the caller ignores it. The +reason is transaction ownership — DICE is a substrate a host application embeds, and that +application owns its database transaction. Baking writes into the pipeline would force its +persistence model onto every consumer; handing back what *should* be saved leaves the boundary, and +the rollback story, with the caller. + +What to save is itself a small decision, captured in `propositionsToPersist()`. With no revision, +that's just the freshly extracted propositions. With revision, it's the set the reviser says to +keep — which crucially includes *modified existing* propositions, not only new ones. A contradiction, +for example, returns both the new fact and the existing one whose confidence was cut and status set +to `CONTRADICTED`; persisting only the new fact would silently leave the old one believed (see +[proposition-lifecycle](proposition-lifecycle.md)). + +```mermaid +flowchart TB + R[Result to persist] --> Q{revision ran?} + Q -->|no| RAW[the extracted propositions] + Q -->|yes| REV[the revised set the reviser kept] + REV --> N[new / merged / reinforced / generalized] + REV --> O["contradicted: the new fact
AND the demoted original"] + RAW --> SAVE[(persist + structural edges)] + N --> SAVE + O --> SAVE +``` + +The convenience `persist()` helper writes the right propositions, saves only the entities those +propositions actually reference, and wires up the structural relationships between chunks, +propositions, and entities in one step — but it's still the caller who chooses to call it, inside +its own transaction. + +## Revision, events, and provenance + +When a reviser and a repository are configured, Phase 2 also reconciles each new proposition against +what's stored and classifies the relationship — new, merged, reinforced, contradicted, or +generalized. Those classifications drive the lifecycle transitions described in +[proposition-lifecycle](proposition-lifecycle.md), and each one emits a *pre-persistence* event so a +consumer can watch reconciliation happen; the durable record is still the `PropositionPersisted` +the store emits on save (see [events](events.md)). `processOnce` adds a hash check so re-ingesting +the same text is a no-op, and lets a caller attach extra grounding ids — the source records a fact +came from — that later become provenance edges. + +## Configurable behavior + +The extractor, the optional reviser and mention filter, the event listener, and the execution +strategy are all pluggable through the pipeline's fluent builders. What ships is intentionally +conservative — serial execution, revision off, no persistence — so the safe, predictable behaviour +is the default, and a deployment opts into concurrency, reconciliation, and its own write boundary +as it needs them. From e3f507be3bdadfaea650687d2287bf7eab00cda1 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 23:11:00 -0400 Subject: [PATCH 15/27] docs(pipeline): drop a stray planning-item reference from a KDoc The processChunk KDoc ended with an internal planning-item tag that shouldn't ship in committed code. The sentence reads the same without it. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt index a603045c..a6d95ad1 100644 --- a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt +++ b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt @@ -285,7 +285,7 @@ class PropositionPipeline private constructor( * * Single-chunk semantics are unchanged: this runs Phase 1 then Phase 2 serially on the * calling thread and propagates any extraction exception (it never produces a - * [ChunkPropositionResult.Failed] — only the batch [process] does, per A3). + * [ChunkPropositionResult.Failed] — only the batch [process] does). * * @param chunk The chunk to process * @param context Configuration including schema and entity resolver From 04168e8dfdf2d59b514088efa75368cfce1138e9 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 23:18:31 -0400 Subject: [PATCH 16/27] docs: reframe the extraction pipeline as named stages and link the design notes Reword the extraction pipeline note to describe its two stages by name (extraction and resolution) rather than numbered phases, so it reads as a self-contained architecture overview instead of a sequence of delivery steps. Also surface the design notes from the README, which previously linked none of them, so the conceptual docs are discoverable from the entry point. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- README.md | 13 +++++ docs/design/extraction-pipeline.md | 82 +++++++++++++++--------------- 2 files changed, 55 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 8c2d53d4..ee210f9e 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,19 @@ flowchart TB style Projections fill:#d4f5d4,stroke:#3fd73c,color:#1e1e1e ``` +### Design notes + +For the reasoning behind how DICE behaves — the conceptual model and the decisions you can't +recover by reading a single class — see the design notes in [`docs/design/`](docs/design/): + +- [Extraction pipeline](docs/design/extraction-pipeline.md) — the two-stage extract/resolve flow, + optional concurrency, and why the pipeline returns unsaved results for the caller to persist. +- [Proposition lifecycle](docs/design/proposition-lifecycle.md) — trust scoring, source authority, + conflict classification, supersession versus contradiction, and decay instead of deletion. +- [Knowledge hygiene](docs/design/knowledge-hygiene.md) — admission gates, mark-and-sweep + reclamation, and the consolidation dream loop. +- [Events](docs/design/events.md) — the domain-event model the store and pipeline emit. + ## Real-World Example: Impromptu **[Impromptu](https://github.com/embabel/impromptu)** is a classical music exploration chatbot that uses DICE diff --git a/docs/design/extraction-pipeline.md b/docs/design/extraction-pipeline.md index a22a71bc..8a8276e4 100644 --- a/docs/design/extraction-pipeline.md +++ b/docs/design/extraction-pipeline.md @@ -1,11 +1,11 @@ -# Extraction pipeline: two phases, optional concurrency, and unsaved results +# Extraction pipeline: two stages, optional concurrency, and unsaved results DICE turns raw text into grounded propositions through one pipeline. It chunks nothing and stores nothing itself — it takes chunks in, extracts statements, resolves the entities those statements mention, optionally reconciles them against what's already known, and hands back a result the -caller persists. This note is about three design decisions behind that flow: why extraction is -split into two phases, why only the first phase is allowed to run concurrently, and why the -pipeline deliberately writes nothing. +caller persists. This note is about three design decisions behind that flow: why extraction and +resolution are kept as separate stages, why only the extraction stage is allowed to run +concurrently, and why the pipeline deliberately writes nothing. The pipeline produces the propositions that everything downstream then governs — the admission gates decide what to keep, the lifecycle decides how each one ages, and the dream loop tidies them @@ -15,12 +15,12 @@ later (see [knowledge-hygiene](knowledge-hygiene.md) and ```mermaid flowchart LR CHUNKS[Chunks] --> P1 - subgraph P1 [Phase 1 · extraction · concurrency-safe] + subgraph P1 ["Extraction stage · concurrency-safe"] direction TB EX["extract propositions
+ suggest entities"] end P1 --> P2 - subgraph P2 [Phase 2 · resolution · always serial] + subgraph P2 ["Resolution stage · always serial"] direction TB RES["resolve entities
through the shared resolver"] --> REV["optional revision
against the store"] end @@ -28,20 +28,20 @@ flowchart LR RESULT -.caller persists.-> STORE[(Proposition store)] ``` -## Why two phases +## Why extraction and resolution are separate -Extraction is two jobs wearing one coat. The first reads a chunk and produces *candidate* -statements and the entities they mention — pure work that depends only on the chunk in front of -it. The second takes those candidates and decides *which real entities* they refer to, which means -consulting a resolver that accumulates identity across the whole run. +Extraction is two jobs wearing one coat. One reads a chunk and produces *candidate* statements and +the entities they mention — pure work that depends only on the chunk in front of it. The other +takes those candidates and decides *which real entities* they refer to, which means consulting a +resolver that accumulates identity across the whole run. -Splitting them is what makes safe concurrency possible at all. Phase 1 (`extractPhase`) touches no -resolver and shares no state between chunks, so two chunks can be extracted at the same time -without stepping on each other. Phase 2 (`resolvePhase`) runs every chunk through one shared -cross-chunk resolver so that "Brahms" mentioned in chunk 3 lands on the same entity as "Brahms" in -chunk 17 — and that shared, mutating identity map is exactly the thing you cannot touch from -several threads at once. So the line between the phases is also the line between "parallelizable" -and "must stay serial." +Keeping them as distinct stages is what makes safe concurrency possible at all. The extraction +stage (`extractPhase`) touches no resolver and shares no state between chunks, so two chunks can be +extracted at the same time without stepping on each other. The resolution stage (`resolvePhase`) +runs every chunk through one shared cross-chunk resolver so that "Brahms" mentioned in one chunk +lands on the same entity as "Brahms" in another — and that shared, mutating identity map is exactly +the thing you cannot touch from several threads at once. So the line between the two stages is also +the line between "parallelizable" and "must stay serial." ```mermaid sequenceDiagram @@ -51,10 +51,10 @@ sequenceDiagram participant Strategy as Execution strategy participant Resolver as Shared cross-chunk resolver Caller->>Pipeline: process(chunks, context) - Note over Pipeline,Strategy: Phase 1 — fan out, order preserved, a failure becomes a null slot + Note over Pipeline,Strategy: Extraction — fan out, order preserved, a failure becomes a null slot Pipeline->>Strategy: extract each chunk (resolver-free) Strategy-->>Pipeline: per-chunk extractions (nulls where extraction threw) - Note over Pipeline,Resolver: Phase 2 — strictly serial, one shared entity identity + Note over Pipeline,Resolver: Resolution — strictly serial, one shared entity identity loop each extraction, in input order alt extraction present Pipeline->>Resolver: resolve entities, then optionally revise @@ -68,11 +68,12 @@ sequenceDiagram ## Execution strategies -How Phase 1 is dispatched is a pluggable choice, separated from the pipeline behind a small SPI so -that turning on concurrency is a one-line swap rather than a rewrite. Three strategies ship: +How the extraction stage is dispatched is a pluggable choice, separated from the pipeline behind a +small SPI so that turning on concurrency is a one-line swap rather than a rewrite. Three strategies +ship: -- **Serial** (the default) runs each chunk one at a time on the calling thread. It is the exact - pre-concurrency behaviour, so the default pipeline changes nothing. +- **Serial** (the default) runs each chunk one at a time on the calling thread. It is the + simplest, fully sequential behaviour, so the default pipeline has no concurrency to reason about. - **Parallel** fans every chunk out at once onto an executor and joins them back in order. - **Batched** processes fixed-size windows: each window runs concurrently, then fully completes before the next starts. This is the rate-limit lever — it caps how many extractions (and so how @@ -81,22 +82,23 @@ that turning on concurrency is a one-line swap rather than a rewrite. Three stra Every strategy honours the same two-part contract, and the contract is what lets the rest of the pipeline stay simple. **Input order is preserved**: the result has one slot per input chunk, in order. **A failure is a `null` slot, never an exception and never a dropped chunk** — when an -extraction throws, that slot comes back `null`, and Phase 2 turns it into a typed `Failed` result. -Nothing is silently reordered or lost, so the `result.size == chunks.size` invariant always holds. +extraction throws, that slot comes back `null`, and the resolution stage turns it into a typed +`Failed` result. Nothing is silently reordered or lost, so the `result.size == chunks.size` +invariant always holds. ```mermaid flowchart TB - subgraph serial [Serial · default] + subgraph serial ["Serial · default"] direction TB SA[chunk 0] --> SB[chunk 1] --> SC[chunk 2] end - subgraph parallel [Parallel · all at once] + subgraph parallel ["Parallel · all at once"] direction TB PA[chunk 0] PB[chunk 1] PC[chunk 2] end - subgraph batched [Batched · windows of N] + subgraph batched ["Batched · windows of N"] direction TB BA["chunk 0 + 1
(window 1)"] --> BB["chunk 2 + 3
(window 2)"] end @@ -125,22 +127,22 @@ long-lived application; use the no-arg constructor as a `use { }` resource for a The batch path treats every chunk as independent, so one bad chunk can't sink a whole document. A chunk that throws during extraction becomes a `null` slot; a chunk that throws during resolution, -revision, or even event emission is caught in Phase 2 and turned into a `Failed` result for that -chunk alone. Either way the run finishes and the failures are visible — `PropositionResults` -exposes the ids of chunks that failed, and the REST layer surfaces them rather than pretending -everything succeeded. +revision, or even event emission is caught in the resolution stage and turned into a `Failed` +result for that chunk alone. Either way the run finishes and the failures are visible — +`PropositionResults` exposes the ids of chunks that failed, and the REST layer surfaces them rather +than pretending everything succeeded. -Single-chunk processing is deliberately stricter. `processChunk` runs the two phases serially on -the calling thread and lets an extraction exception propagate, because a caller handling one chunk +Single-chunk processing is deliberately stricter. `processChunk` runs both stages serially on the +calling thread and lets an extraction exception propagate, because a caller handling one chunk wants the error, not a quietly empty result. Only the batch `process` converts failures into `Failed` slots, because that is where swallowing one chunk to save the other forty is the right trade. ```mermaid flowchart TB - C[Chunk] --> EX{Phase 1
extraction} + C[Chunk] --> EX{Extraction
stage} EX -->|throws| NULL[null slot] - EX -->|ok| RES{Phase 2
resolve / revise} + EX -->|ok| RES{Resolution
stage} NULL --> FAILED["Failed result
(carries chunk id + cause)"] RES -->|throws| FAILED RES -->|ok| SUCCESS["Success result
(propositions + resolutions)"] @@ -183,9 +185,9 @@ its own transaction. ## Revision, events, and provenance -When a reviser and a repository are configured, Phase 2 also reconciles each new proposition against -what's stored and classifies the relationship — new, merged, reinforced, contradicted, or -generalized. Those classifications drive the lifecycle transitions described in +When a reviser and a repository are configured, the resolution stage also reconciles each new +proposition against what's stored and classifies the relationship — new, merged, reinforced, +contradicted, or generalized. Those classifications drive the lifecycle transitions described in [proposition-lifecycle](proposition-lifecycle.md), and each one emits a *pre-persistence* event so a consumer can watch reconciliation happen; the durable record is still the `PropositionPersisted` the store emits on save (see [events](events.md)). `processOnce` adds a hash check so re-ingesting From b56f16eea6b24f5ce1bb29c8fc329e9889863585 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 23:30:59 -0400 Subject: [PATCH 17/27] docs: remove delivery-phase wording from code comments and identifiers Replace numbered "Phase 1/2/3/4" comments and N-phase pipeline phrasing with descriptive stage wording across the extraction pipeline and maintenance orchestrator, and rename the pipeline's private extractPhase/resolvePhase/ ExtractionPhaseResult to the stage spelling the design note already uses. The mark-and-sweep collector keeps its canonical mark phase / sweep phase terms, which name an algorithm rather than a delivery step. No behaviour change. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../pipeline/ExtractionExecutionStrategy.kt | 4 +- .../dice/pipeline/PropositionPipeline.kt | 44 +++++++++---------- .../DefaultMemoryMaintenanceOrchestrator.kt | 14 +++--- .../memory/MemoryMaintenanceOrchestrator.kt | 8 ++-- .../embabel/dice/proposition/Proposition.kt | 2 +- .../com/embabel/dice/spi/TrustScorer.kt | 2 +- .../dice/pipeline/PropositionPipelineTest.kt | 8 ++-- .../MemoryMaintenanceOrchestratorTest.kt | 22 +++++----- docs/design/extraction-pipeline.md | 4 +- 9 files changed, 54 insertions(+), 54 deletions(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategy.kt index 9ceba359..0fac8501 100644 --- a/dice/src/main/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategy.kt +++ b/dice/src/main/kotlin/com/embabel/dice/pipeline/ExtractionExecutionStrategy.kt @@ -21,8 +21,8 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors /** - * Controls how the [PropositionPipeline] dispatches the stateless per-chunk extraction phase - * (Phase 1). Implementations choose serial or concurrent execution; Phase 2 (entity resolution) + * Controls how the [PropositionPipeline] dispatches the stateless per-chunk extraction stage. + * Implementations choose serial or concurrent execution; the resolution stage (entity resolution) * always runs serially regardless of the strategy, preserving cross-chunk entity identity. * * ## Contract diff --git a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt index a6d95ad1..ddf41d05 100644 --- a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt +++ b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt @@ -57,7 +57,7 @@ import java.time.Instant * - [withRevision] — compare new propositions against existing ones * - [withMentionFilter] — drop low-quality entity mentions before resolution * - [withEventListener] — observe pipeline events - * - [withExecutionStrategy] — control how the extraction phase is dispatched + * - [withExecutionStrategy] — control how the extraction stage is dispatched * * Example usage: * ```kotlin @@ -157,11 +157,11 @@ class PropositionPipeline private constructor( /** * Set the [ExtractionExecutionStrategy] used to dispatch the stateless per-chunk - * extraction phase of [process]. + * extraction stage of [process]. * - * Defaults to [SerialExtractionStrategy] (zero behavior change vs. the pre-strategy - * pipeline). [ParallelExtractionStrategy] and [BatchedExtractionStrategy] only affect - * Phase 1 (extraction); the resolver-touching Phase 2 always stays serial, so + * Defaults to [SerialExtractionStrategy] (the fully sequential default). + * [ParallelExtractionStrategy] and [BatchedExtractionStrategy] only affect + * the extraction stage; the resolver-touching resolution stage always stays serial, so * cross-chunk entity identity is preserved regardless of strategy. * * Enabling Parallel/Batched(`batchSize > 1`) in production is gated on verifying extractor @@ -174,20 +174,20 @@ class PropositionPipeline private constructor( PropositionPipeline(extractor, reviser, propositionRepository, mentionFilter, rawEventListener, strategy) /** - * Carries the resolver-free output of Phase 1 for a single chunk so it can be produced - * concurrently. Phase 2 picks this up serially to do entity resolution. See [process]. + * Carries the resolver-free output of the extraction stage for a single chunk so it can be produced + * concurrently. The resolution stage picks this up serially to do entity resolution. See [process]. */ - private data class ExtractionPhaseResult( + private data class ExtractionStageResult( val chunk: Chunk, val suggestedPropositions: SuggestedPropositions, val suggestedEntities: SuggestedEntities, ) /** - * Phase 1 — extract propositions and convert mentions to suggested entities. + * Extraction stage — extract propositions and convert mentions to suggested entities. * Touches no entity resolver, so it is safe to run concurrently across chunks. */ - private fun extractPhase(chunk: Chunk, context: SourceAnalysisContext): ExtractionPhaseResult { + private fun extractStage(chunk: Chunk, context: SourceAnalysisContext): ExtractionStageResult { logger.debug("Processing chunk: {}", chunk.id) // Step 1: Extract propositions from chunk @@ -203,16 +203,16 @@ class PropositionPipeline private constructor( ) logger.debug("Created {} suggested entities", suggestedEntities.suggestedEntities.size) - return ExtractionPhaseResult(chunk, suggestedPropositions, suggestedEntities) + return ExtractionStageResult(chunk, suggestedPropositions, suggestedEntities) } /** - * Phase 2 — resolve entities through the shared cross-chunk resolver, apply resolutions, + * Resolution stage — resolve entities through the shared cross-chunk resolver, apply resolutions, * and optionally revise against existing propositions. Must stay serial so all chunks share * the same entity identity within a single [process] run. */ - private fun resolvePhase( - extraction: ExtractionPhaseResult, + private fun resolveStage( + extraction: ExtractionStageResult, context: SourceAnalysisContext, ): ChunkPropositionResult.Success { val (chunk, suggestedPropositions, suggestedEntities) = extraction @@ -283,7 +283,7 @@ class PropositionPipeline private constructor( * [PersistablePropositions.persist][PersistablePropositions.persist] within its own * transaction scope; nothing is written to any repository otherwise. * - * Single-chunk semantics are unchanged: this runs Phase 1 then Phase 2 serially on the + * Single-chunk semantics are unchanged: this runs extraction then resolution serially on the * calling thread and propagates any extraction exception (it never produces a * [ChunkPropositionResult.Failed] — only the batch [process] does). * @@ -295,7 +295,7 @@ class PropositionPipeline private constructor( chunk: Chunk, context: SourceAnalysisContext, ): ChunkPropositionResult = - resolvePhase(extractPhase(chunk, context), context) + resolveStage(extractStage(chunk, context), context) /** * Process a text once, with hash-based deduplication. @@ -380,13 +380,13 @@ class PropositionPipeline private constructor( ) val crossChunkContext = context.copy(entityResolver = crossChunkResolver) - // Phase 1 — resolver-free extraction, dispatched via the execution strategy (parallelizable). + // Extraction stage — resolver-free extraction, dispatched via the execution strategy (parallelizable). // A failed extraction yields a null slot; input order is preserved by the strategy. - // Capture the cause per chunk so we can produce a typed Failed result in Phase 2. + // Capture the cause per chunk so we can produce a typed Failed result in the resolution stage. val failures = HashMap() - val extractions: List = + val extractions: List = executionStrategy.execute(chunks) { chunk -> - runCatching { extractPhase(chunk, crossChunkContext) } + runCatching { extractStage(chunk, crossChunkContext) } .getOrElse { e -> logger.warn("Extraction failed for chunk {}: {}", chunk.id, e.message, e) synchronized(failures) { failures[chunk.id] = e } @@ -394,7 +394,7 @@ class PropositionPipeline private constructor( } } - // Phase 2 — always serial, so all chunks share the same entity identity. + // Resolution stage — always serial, so all chunks share the same entity identity. // Resolve each non-null extraction through the shared crossChunkResolver; map null // slots to typed ChunkPropositionResult.Failed. val chunkResults: List = extractions.mapIndexed { i, extraction -> @@ -402,7 +402,7 @@ class PropositionPipeline private constructor( if (extraction != null) { // Isolate per-chunk: a throw in resolution, revision, or event emission surfaces // as a Failed result for this chunk only, never aborting the whole run. - runCatching { resolvePhase(extraction, crossChunkContext) } + runCatching { resolveStage(extraction, crossChunkContext) } .getOrElse { e -> logger.warn("Resolution failed for chunk {}: {}", chunkId, e.message, e) ChunkPropositionResult.failed(chunkId, e) diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt index f3a5c9df..a9f84cc1 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt @@ -30,7 +30,7 @@ private val logger = LoggerFactory.getLogger("com.embabel.dice.projection.memory /** * Default [MemoryMaintenanceOrchestrator]: runs consolidation, abstraction, retirement, and - * optionally the mark-and-sweep collector as a four-phase pipeline. + * optionally the mark-and-sweep collector as a four-stage pipeline. * * The consolidation and abstraction logic delegates to [SessionConsolidationPass] and * [AbstractionPass] respectively — they own the details of what changes. This orchestrator @@ -52,7 +52,7 @@ private val logger = LoggerFactory.getLogger("com.embabel.dice.projection.memory * @param abstractionTargetCount How many abstractions to generate per group. * @param retireBelow Hard-delete propositions whose effective confidence falls below this; null disables retirement. * @param retireDecayK Decay-rate multiplier used when computing effective confidence for retirement. - * @param collector Optional mark-and-sweep collector run as the final phase; null disables it. + * @param collector Optional mark-and-sweep collector run as the final stage; null disables it. */ data class DefaultMemoryMaintenanceOrchestrator( private val repository: PropositionRepository, @@ -69,17 +69,17 @@ data class DefaultMemoryMaintenanceOrchestrator( contextId: ContextId, sessionPropositions: List, ): MaintenanceResult { - // Phase 1: Consolidate session propositions (single source of truth: SessionConsolidationPass) + // Consolidate session propositions (single source of truth: SessionConsolidationPass) val consolidation = consolidate(contextId, sessionPropositions) - // Phase 2: Abstract entity groups (single source of truth: AbstractionPass) + // Abstract entity groups (single source of truth: AbstractionPass) val (abstractions, superseded) = abstract(contextId) - // Phase 3: Retire expired propositions (legacy hard delete; see class KDoc) + // Retire expired propositions (legacy hard delete; see class KDoc) val retired = retire(contextId) - // Phase 4 (optional): run the mark-and-sweep collector when one is configured. - // Default off; when null the behavior is identical to the three-phase pipeline. + // Optionally run the mark-and-sweep collector when one is configured. + // Default off; when null the behavior is identical to the three-stage pipeline. val collectorResult = collector?.run(contextId) logger.info( diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestrator.kt index 630db95b..85619781 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestrator.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestrator.kt @@ -22,11 +22,11 @@ import com.embabel.dice.proposition.PropositionRepository /** * Result of a memory maintenance run. * - * @property consolidation Result from the consolidation phase, null if no session propositions were provided + * @property consolidation Result from the consolidation stage, null if no session propositions were provided * @property abstractions New abstract propositions generated from entity groups * @property superseded Source propositions that were marked SUPERSEDED after abstraction * @property retired Propositions that were deleted because their effective confidence fell below the threshold - * @property collectorResult Result of the optional collector phase, or null when no collector was configured + * @property collectorResult Result of the optional collector stage, or null when no collector was configured */ data class MaintenanceResult( val consolidation: ConsolidationResult?, @@ -48,7 +48,7 @@ data class MaintenanceResult( * Orchestrates memory maintenance — consolidating, abstracting, retiring, and persisting — * as a single entry point suitable for end-of-session or scheduled batch runs. * - * Three-phase pipeline: + * Three-stage pipeline: * 1. **Consolidate** — Promote/reinforce/merge/discard session propositions against existing long-term memories. * 2. **Abstract** — Synthesize higher-level insights from groups of related propositions (grouped by entity). * 3. **Retire expired** — Delete ACTIVE propositions whose effective confidence has fallen below a threshold. @@ -71,7 +71,7 @@ data class MaintenanceResult( interface MemoryMaintenanceOrchestrator { /** - * Run all maintenance phases for the given context. + * Run all maintenance stages for the given context. * * @param contextId The context to maintain * @param sessionPropositions Propositions from the current session to consolidate; empty for scheduled maintenance diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/Proposition.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/Proposition.kt index c936a937..0d5eb4e3 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/Proposition.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/Proposition.kt @@ -69,7 +69,7 @@ enum class PropositionStatus { * Each proposition should express a single fact with at most two entity mentions * (SUBJECT and OBJECT). This maps cleanly to graph relationships during promotion. * Complex sentences with multiple relationships should be extracted as multiple - * propositions during the extraction phase. + * propositions during the extraction stage. * * @property id Unique identifier for this proposition * @property contextId The context in which this proposition is relevant diff --git a/dice/src/main/kotlin/com/embabel/dice/spi/TrustScorer.kt b/dice/src/main/kotlin/com/embabel/dice/spi/TrustScorer.kt index 356f3ee2..340ac138 100644 --- a/dice/src/main/kotlin/com/embabel/dice/spi/TrustScorer.kt +++ b/dice/src/main/kotlin/com/embabel/dice/spi/TrustScorer.kt @@ -24,7 +24,7 @@ import com.embabel.dice.proposition.Proposition * weight propositions by how much they trust the source ([authorityTier]) and how * a clash with existing knowledge was classified ([conflictType]), without DICE * core hard-coding any particular trust model. The cached score is later consumed - * by query filters and gates (a downstream phase). + * by query filters and gates (a downstream stage). * * The returned score is a value in `[0.0, 1.0]`, where `1.0` is fully trusted and * `0.0` is untrusted. Out-of-range scores from custom implementations are advisory diff --git a/dice/src/test/kotlin/com/embabel/dice/pipeline/PropositionPipelineTest.kt b/dice/src/test/kotlin/com/embabel/dice/pipeline/PropositionPipelineTest.kt index ebf2cab9..2d99a34b 100644 --- a/dice/src/test/kotlin/com/embabel/dice/pipeline/PropositionPipelineTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/pipeline/PropositionPipelineTest.kt @@ -92,7 +92,7 @@ class PropositionPipelineTest { override fun extract(chunk: Chunk, context: SourceAnalysisContext): SuggestedPropositions { // Sentinel: a "fail:"-prefixed chunk simulates an extraction failure so the - // execution strategy's runCatching yields a null slot -> Phase 2 produces + // execution strategy's runCatching yields a null slot -> the resolution stage produces // ChunkPropositionResult.Failed. if (chunk.text.startsWith("fail:")) { throw IllegalStateException("sentinel failure for ${chunk.id}") @@ -1488,8 +1488,8 @@ class PropositionPipelineTest { @Test fun `a chunk that fails during resolution is isolated as Failed`() { - // Phase 1 (extraction) succeeds for every chunk; the resolver throws for chunk-2 - // during Phase 2 resolution. The run must still produce one result per chunk with + // extraction succeeds for every chunk; the resolver throws for chunk-2 + // during resolution. The run must still produce one result per chunk with // chunk-2 surfaced as Failed — not abort and discard the good chunks' results. val resolverThatThrowsOnBoom = object : EntityResolver { override fun resolve( @@ -1557,7 +1557,7 @@ class PropositionPipelineTest { val result = newPipeline(strategy).process(chunks, context()) - // 3 unique entities; Alice appears once in newEntities (resolved serially in Phase 2). + // 3 unique entities; Alice appears once in newEntities (resolved serially in the resolution stage). val newEntities = result.newEntities() assertEquals(3, newEntities.size, "Found: ${newEntities.map { it.name }}") assertEquals(1, newEntities.count { it.name == "Alice" }) diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestratorTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestratorTest.kt index 684ad5a3..5575abc1 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestratorTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/MemoryMaintenanceOrchestratorTest.kt @@ -110,7 +110,7 @@ class MemoryMaintenanceOrchestratorTest { } @Nested - inner class ConsolidationPhaseTests { + inner class ConsolidationStageTests { @Test fun `skips consolidation when no session propositions`() { @@ -214,7 +214,7 @@ class MemoryMaintenanceOrchestratorTest { } @Nested - inner class AbstractionPhaseTests { + inner class AbstractionStageTests { @Test fun `skips abstraction when no abstractor configured`() { @@ -564,7 +564,7 @@ class MemoryMaintenanceOrchestratorTest { orchestrator.maintain(contextId) - // The abstraction phase query should filter to level 0 + // The abstraction stage query should filter to level 0 val queries = mutableListOf() verify { repository.query(capture(queries)) } val abstractionQuery = queries.find { it.maxLevel == 0 } @@ -574,7 +574,7 @@ class MemoryMaintenanceOrchestratorTest { } @Nested - inner class RetirementPhaseTests { + inner class RetirementStageTests { @Test fun `skips retirement when retireBelow is null`() { @@ -712,7 +712,7 @@ class MemoryMaintenanceOrchestratorTest { } @Nested - inner class CollectorPhaseTests { + inner class CollectorStageTests { @Test fun `no collector configured yields null collectorResult`() { @@ -730,7 +730,7 @@ class MemoryMaintenanceOrchestratorTest { val old = Instant.now().minus(365, ChronoUnit.DAYS) val decayed = proposition("old fact", confidence = 0.5, decay = 0.5, revised = old) - // Only the collector phase consumes candidates here: consolidation is skipped (no session), + // Only the collector stage consumes candidates here: consolidation is skipped (no session), // abstraction is skipped (no abstractor), and retirement is skipped (no retireBelow). every { repository.query(any()) } returns listOf(decayed) val saved = mutableListOf() @@ -759,7 +759,7 @@ class MemoryMaintenanceOrchestratorTest { inner class FullPipelineTests { @Test - fun `runs all three phases in order`() { + fun `runs all three stages in order`() { val session = listOf(proposition("session fact")) val aliceProps = (1..5).map { proposition("alice fact $it", entityId = "alice") } @@ -775,9 +775,9 @@ class MemoryMaintenanceOrchestratorTest { ) // Repository returns different results per query context - // First call: consolidation phase (existing ACTIVE props) - // Second call: abstraction phase (level 0 ACTIVE props) - // Third call: retirement phase (ACTIVE props) + // First call: consolidation stage (existing ACTIVE props) + // Second call: abstraction stage (level 0 ACTIVE props) + // Third call: retirement stage (ACTIVE props) val queryResults = mutableListOf( emptyList(), // consolidation: existing aliceProps, // abstraction: level 0 active @@ -807,7 +807,7 @@ class MemoryMaintenanceOrchestratorTest { val result = orchestrator.maintain(contextId, session) - // All phases ran + // All stages ran assertNotNull(result.consolidation) assertEquals(1, result.consolidation!!.promoted.size) assertEquals(1, result.abstractions.size) diff --git a/docs/design/extraction-pipeline.md b/docs/design/extraction-pipeline.md index 8a8276e4..7e695e4b 100644 --- a/docs/design/extraction-pipeline.md +++ b/docs/design/extraction-pipeline.md @@ -36,8 +36,8 @@ takes those candidates and decides *which real entities* they refer to, which me resolver that accumulates identity across the whole run. Keeping them as distinct stages is what makes safe concurrency possible at all. The extraction -stage (`extractPhase`) touches no resolver and shares no state between chunks, so two chunks can be -extracted at the same time without stepping on each other. The resolution stage (`resolvePhase`) +stage (`extractStage`) touches no resolver and shares no state between chunks, so two chunks can be +extracted at the same time without stepping on each other. The resolution stage (`resolveStage`) runs every chunk through one shared cross-chunk resolver so that "Brahms" mentioned in one chunk lands on the same entity as "Brahms" in another — and that shared, mutating identity map is exactly the thing you cannot touch from several threads at once. So the line between the two stages is also From d3da68844c95e4605a5b57743b6569be2142822a Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 23:40:43 -0400 Subject: [PATCH 18/27] fix(concurrency): lock dream-loop cycles per context and tighten thread-safety contracts The change-volume gate in DefaultDreamLoopOrchestrator did a check-then-act on lastActiveCount, so two triggers for the same context could both pass the gate on the same stale count and run overlapping cycles. consolidate/consolidateNow now hold a per-context lock (reentrant; different contexts still run in parallel). The pipeline's per-chunk failure map is a ConcurrentHashMap rather than a HashMap guarded by an external lock with an undocumented post-join read. Also document the thread-safety contracts surfaced by an audit: InMemoryEntityResolver is single-run-scoped and not thread-safe; DefaultCollectorRunner is safe across contexts but double-sweeps a context called concurrently; the maintenance orchestrator's per-step persistence is non-transactional but self-heals on the next idempotent run; and dream-loop copy() isolation comes from the field initializer. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../common/resolver/InMemoryEntityResolver.kt | 5 ++++ .../dice/pipeline/PropositionPipeline.kt | 7 +++-- .../memory/DefaultCollectorRunner.kt | 6 +++++ .../memory/DefaultDreamLoopOrchestrator.kt | 26 +++++++++++++++---- .../DefaultMemoryMaintenanceOrchestrator.kt | 5 ++++ 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/common/resolver/InMemoryEntityResolver.kt b/dice/src/main/kotlin/com/embabel/dice/common/resolver/InMemoryEntityResolver.kt index 8f018320..343cb1bb 100644 --- a/dice/src/main/kotlin/com/embabel/dice/common/resolver/InMemoryEntityResolver.kt +++ b/dice/src/main/kotlin/com/embabel/dice/common/resolver/InMemoryEntityResolver.kt @@ -40,6 +40,11 @@ import kotlin.math.min * - Partial name matching (e.g., "Holmes" matches "Sherlock Holmes") * - Levenshtein distance for fuzzy matching * + * Not thread-safe: it keeps a plain mutable map of remembered entities, so a single instance must + * not be shared across concurrent callers. This is by design — `PropositionPipeline.process` builds + * a fresh resolver per run and only calls it from the serial resolution stage, so the memory stays + * scoped to one run and never sees concurrent access. + * * @param config Configuration for matching thresholds */ class InMemoryEntityResolver( diff --git a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt index ddf41d05..ed82b8a5 100644 --- a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt +++ b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt @@ -43,6 +43,7 @@ import com.embabel.dice.proposition.revision.PropositionReviser import com.embabel.dice.proposition.revision.RevisionResult import org.slf4j.LoggerFactory import java.time.Instant +import java.util.concurrent.ConcurrentHashMap /** * Pipeline for extracting propositions from chunks. @@ -383,13 +384,15 @@ class PropositionPipeline private constructor( // Extraction stage — resolver-free extraction, dispatched via the execution strategy (parallelizable). // A failed extraction yields a null slot; input order is preserved by the strategy. // Capture the cause per chunk so we can produce a typed Failed result in the resolution stage. - val failures = HashMap() + // ConcurrentHashMap so a concurrent execution strategy can record failures safely without an + // external lock; the resolution stage reads it only after every extraction has been joined. + val failures = ConcurrentHashMap() val extractions: List = executionStrategy.execute(chunks) { chunk -> runCatching { extractStage(chunk, crossChunkContext) } .getOrElse { e -> logger.warn("Extraction failed for chunk {}: {}", chunk.id, e.message, e) - synchronized(failures) { failures[chunk.id] = e } + failures[chunk.id] = e throw e // let the strategy's runCatching map this to a null slot } } diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt index 9eba434f..dcc5f4cd 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt @@ -44,6 +44,12 @@ import java.util.UUID * - [run] with `dryRun = false` applies each decision, saves the run record, then emits a * [PropositionStatusChanged] per applied transition. * + * Concurrency: holds no shared mutable state — every [run] call works with its own locals — so + * runs for different contexts are safe in parallel. Two runs for the *same* context at once are + * not corrupting but are wasteful: both read the same ACTIVE set and the second simply re-applies + * or skips already-transitioned propositions. Serialize per context at the scheduling layer if that + * matters (the [DefaultDreamLoopOrchestrator] that normally drives this already locks per context). + * * @param repository Proposition store to read candidates from and write transitions to. * @param strategies Mark strategies run during the mark phase. * @param policy Policy deciding each marked proposition's fate. diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt index f8ea6a3d..fb8dba68 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt @@ -56,12 +56,25 @@ data class DefaultDreamLoopOrchestrator( * Tracks the last-seen ACTIVE count per context, used by the change-volume trigger. * * Kept outside the primary constructor on purpose: it's mutable runtime state, not part of - * this orchestrator's value identity. That way [copy]-built instances each start with their - * own clean baseline (no shared state), and two orchestrators whose counters have diverged - * still compare as equal. Thread-safe via [ConcurrentHashMap]. + * this orchestrator's value identity. That isolation comes from [copy] re-running this field + * initializer (it copies constructor properties only), so a copy starts with its own clean + * baseline rather than sharing this map; two orchestrators whose counters have diverged still + * compare as equal. Thread-safe via [ConcurrentHashMap]. */ private val lastActiveCount: ConcurrentHashMap = ConcurrentHashMap() + /** + * One lock per context, so a full cycle for a given context runs start-to-finish without + * another cycle for the same context interleaving. Without this, two triggers for the same + * context could both pass the change-volume gate on the same stale count and run overlapping + * cycles. Different contexts lock independently and still run concurrently. Like + * [lastActiveCount], this is per-instance runtime state — serialization holds within a single + * shared orchestrator instance, which is the intended deployment. + */ + private val cycleLocks: ConcurrentHashMap = ConcurrentHashMap() + + private fun lockFor(contextId: ContextId): Any = cycleLocks.computeIfAbsent(contextId) { Any() } + override fun withPass(pass: ConsolidationPass): DefaultDreamLoopOrchestrator = copy(passes = passes + pass) @@ -73,7 +86,9 @@ data class DefaultDreamLoopOrchestrator( fun withChangeVolumeThreshold(threshold: Int): DefaultDreamLoopOrchestrator = copy(changeVolumeThreshold = threshold) - override fun consolidate(contextId: ContextId): DreamLoopReport? { + override fun consolidate(contextId: ContextId): DreamLoopReport? = synchronized(lockFor(contextId)) { + // Hold the per-context lock across the gate check and the cycle so the threshold decision + // and the count update can't race a concurrent trigger for the same context. val currentActive = activeSnapshot(contextId).size val delta = currentActive - (lastActiveCount[contextId] ?: 0) if (delta < changeVolumeThreshold) { @@ -85,10 +100,11 @@ data class DefaultDreamLoopOrchestrator( ) return null } + // Reentrant: consolidateNow takes the same per-context lock. return consolidateNow(contextId) } - override fun consolidateNow(contextId: ContextId): DreamLoopReport { + override fun consolidateNow(contextId: ContextId): DreamLoopReport = synchronized(lockFor(contextId)) { val cycleStarted = Instant.now() // (1) Fetch the ACTIVE snapshot once. diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt index a9f84cc1..0f86cc32 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt @@ -45,6 +45,11 @@ private val logger = LoggerFactory.getLogger("com.embabel.dice.projection.memory * dream-loop which soft-transitions them to STALE by default. This preserves the original * behavior for backward compatibility. * + * Persistence is per-step, not transactional: the abstraction step saves each entity group as it + * goes, so if a later group throws, earlier groups are already written. That's safe here because + * [maintain] re-runs abstraction from the level-0 snapshot on every call, so the next call simply + * re-processes whatever didn't finish — partial state heals itself rather than corrupting. + * * @param repository Storage backend for propositions. * @param consolidator Decides how session propositions are folded into long-term memory. * @param abstractor Optional; synthesizes higher-level insights from entity groups. From 49dfad0060c9add65478030dd32bfe54a2313e2c Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 23:45:41 -0400 Subject: [PATCH 19/27] feat(observability): log consolidation passes and sweep decisions The abstraction, contradiction-resolution, and session-consolidation passes did real work but emitted nothing of their own, and the default sweep policy skipped pinned/unmarked propositions silently. Add debug logging per pass summarizing what it decided (counts, per context) and trace logging for each sweep decision, so a maintenance cycle is observable end to end rather than only at the orchestrator. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../consolidation/AbstractionPass.kt | 7 +++++++ .../ContradictionResolutionPass.kt | 7 +++++++ .../consolidation/SessionConsolidationPass.kt | 8 ++++++++ .../dice/projection/memory/SweepPolicy.kt | 18 ++++++++++++++---- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt index 8367c42f..ced28cd4 100644 --- a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt @@ -20,6 +20,7 @@ import com.embabel.dice.operations.PropositionGroup import com.embabel.dice.operations.abstraction.PropositionAbstractor import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionStatus +import org.slf4j.LoggerFactory /** * Consolidation pass that synthesizes higher-level propositions from groups of related ground-level @@ -53,6 +54,8 @@ class AbstractionPass @JvmOverloads constructor( override val name: String = "abstraction" + private val logger = LoggerFactory.getLogger(AbstractionPass::class.java) + override fun run(contextId: ContextId, propositions: List): ConsolidationPassResult { return try { val level0Active = propositions.filter { @@ -86,6 +89,10 @@ class AbstractionPass @JvmOverloads constructor( abstractedGroups++ } + logger.debug( + "Abstraction over {} level-0 active proposition(s) for {}: {} group(s) abstracted, {} skipped (already covered), {} proposition(s) to save", + level0Active.size, contextId, abstractedGroups, skipped, toSave.size, + ) if (toSave.isEmpty()) { ConsolidationPassResult.NoOp(name, "no groups above threshold or all covered") } else { diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt index e829efbc..bbf9d758 100644 --- a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt @@ -20,6 +20,7 @@ import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionStatus import com.embabel.dice.proposition.revision.PropositionRelation import com.embabel.dice.proposition.revision.PropositionReviser +import org.slf4j.LoggerFactory /** * Consolidation pass that resolves contradictions among ACTIVE propositions by delegating @@ -49,6 +50,8 @@ class ContradictionResolutionPass @JvmOverloads constructor( override val name: String = "contradiction-resolution" + private val logger = LoggerFactory.getLogger(ContradictionResolutionPass::class.java) + override fun run(contextId: ContextId, propositions: List): ConsolidationPassResult { return try { val active = propositions.filter { it.status == PropositionStatus.ACTIVE } @@ -78,6 +81,10 @@ class ContradictionResolutionPass @JvmOverloads constructor( } } val deduped = toSave.distinctBy { it.id } + logger.debug( + "Contradiction resolution over {} active proposition(s) for {}: {} retired to CONTRADICTED", + active.size, contextId, deduped.size, + ) if (deduped.isEmpty()) { ConsolidationPassResult.NoOp(name, "no contradictions found") } else { diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/SessionConsolidationPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/SessionConsolidationPass.kt index 760bd4dd..1c539622 100644 --- a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/SessionConsolidationPass.kt +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/SessionConsolidationPass.kt @@ -18,6 +18,7 @@ package com.embabel.dice.operations.consolidation import com.embabel.agent.core.ContextId import com.embabel.dice.projection.memory.MemoryConsolidator import com.embabel.dice.proposition.Proposition +import org.slf4j.LoggerFactory /** * Consolidation pass that folds a session's propositions into long-term memory by delegating @@ -40,10 +41,17 @@ class SessionConsolidationPass @JvmOverloads constructor( override val name: String = "session-consolidation" + private val logger = LoggerFactory.getLogger(SessionConsolidationPass::class.java) + override fun run(contextId: ContextId, propositions: List): ConsolidationPassResult { return try { val result = consolidator.consolidate(sessionPropositions, propositions) val toSave = result.promoted + result.reinforced + result.merged.map { it.result } + logger.debug( + "Session consolidation for {}: {} session vs {} long-term proposition(s) -> {} promoted, {} reinforced, {} merged, {} discarded", + contextId, sessionPropositions.size, propositions.size, + result.promoted.size, result.reinforced.size, result.merged.size, result.discarded.size, + ) if (toSave.isEmpty()) { ConsolidationPassResult.NoOp(name, "nothing to promote, reinforce, or merge") } else { diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepPolicy.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepPolicy.kt index 2c88a5fc..0383befe 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepPolicy.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepPolicy.kt @@ -17,6 +17,7 @@ package com.embabel.dice.projection.memory import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionStatus +import org.slf4j.LoggerFactory /** * The "sweep" half of the mark-and-sweep collector: given a proposition and its marks, decides @@ -56,9 +57,18 @@ class StatusTransitionSweepPolicy @JvmOverloads constructor( private val targetStatus: PropositionStatus = PropositionStatus.STALE, ) : SweepPolicy { - override fun decide(proposition: Proposition, marks: List): SweepAction = when { - proposition.pinned -> SweepAction.Skip - marks.isEmpty() -> SweepAction.Skip - else -> SweepAction.TransitionStatus(targetStatus) + private val logger = LoggerFactory.getLogger(StatusTransitionSweepPolicy::class.java) + + override fun decide(proposition: Proposition, marks: List): SweepAction { + val action = when { + proposition.pinned -> SweepAction.Skip + marks.isEmpty() -> SweepAction.Skip + else -> SweepAction.TransitionStatus(targetStatus) + } + logger.trace( + "Sweep decision for {} (pinned={}, marks={}): {}", + proposition.id.take(8), proposition.pinned, marks.size, action, + ) + return action } } From e3c6ea2d430dacb1dbd1dc35e1cda47befed1503 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 23:48:54 -0400 Subject: [PATCH 20/27] docs(agents): map the maintenance, collector, consolidation, gate, and strategy subsystems The AGENTS navigation guides predated the extraction-strategy, gate, collector, dream-loop, and consolidation-pass subsystems. Add the missing package rows and a knowledge-hygiene orientation section (admission -> reclamation -> consolidation), and point both guides at the design notes, so the pipelines are navigable end to end. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- AGENTS.md | 11 ++++++++--- dice/AGENTS.md | 20 +++++++++++++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 33e9cd7b..d70cbef4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,12 +48,14 @@ The `dice` module is organized by responsibility: | `com.embabel.dice.proposition.extraction` | `PropositionExtractor` SPI, `LlmPropositionExtractor` | | `com.embabel.dice.proposition.revision` | `PropositionReviser` SPI, `LlmPropositionReviser` | | `com.embabel.dice.proposition.store` | In-memory and JSON-file implementations of `PropositionRepository` and `DecayManager` | -| `com.embabel.dice.pipeline` | `PropositionPipeline` — the orchestrating entry point that chains extractor → entity resolver → reviser | +| `com.embabel.dice.proposition.gate` | Admission gates that route pipeline output before persistence (`ExtractionGate`, `ExtractionGatePipeline`, `StandardGates`, `EvidenceFloorGate`) | +| `com.embabel.dice.pipeline` | `PropositionPipeline` — the two-stage entry point (concurrent extraction → serial entity resolution); pluggable `ExtractionExecutionStrategy` (serial / parallel / batched) | | `com.embabel.dice.spi` | Policy extension points consumers plug into: `TrustScorer`, `AuthorityResolver`/`AuthorityTier`, `AuthorityWeightedTrustScorer`, `ConflictDetector`, `ConflictType`, `StatusTransitionPolicy`, and their shipped defaults | | `com.embabel.dice.common` | Shared types: `SourceAnalysisContext`, `EntityResolver`, `Relations`, `ContentHasher`, `KnowledgeType`, events, validation rules, mention filters, resolver implementations | | `com.embabel.dice.entity` | Entity-only pipeline (`EntityPipeline`, `EntityIncrementalAnalyzer`, `EntityResolutionService`, `EntityResolutionTools`) | | `com.embabel.dice.incremental` | `AbstractIncrementalAnalyzer`, `ChunkHistoryStore`, `ConversationSource`, windowed processing | -| `com.embabel.dice.projection` | Projectors: `GraphProjector`, `PrologProjector`, `MemoryProjector`; lineage tracking (`ProjectionRecordStore`, `Reconciler`) | +| `com.embabel.dice.projection` | Projectors (`GraphProjector`, `PrologProjector`, `MemoryProjector`); memory maintenance and the dream loop (`MemoryMaintenanceOrchestrator`, `DreamLoopOrchestrator`); the mark-and-sweep collector (`CollectorRunner`, `CollectorStrategy`, `SweepPolicy`) and its audit trail (`CollectorRecordStore`); lineage tracking (`ProjectionRecordStore`, `Reconciler`) | +| `com.embabel.dice.operations` | Higher-level proposition operations (`PropositionAbstractor`, `PropositionContraster`) and, under `operations.consolidation`, the passes that compose the dream loop (`SessionConsolidationPass`, `AbstractionPass`, `ContradictionResolutionPass`, `DecaySweepPass`) | | `com.embabel.dice.text2graph` | Text-to-graph builder pipeline (`KnowledgeGraphBuilder`, `SourceAnalyzer`, entity merge/relationship resolution policies) | | `com.embabel.dice.provenance` | `ProvenanceEntry`, `SourceLocator` — rich evidence links from propositions back to source material | | `com.embabel.dice.query.oracle` | `Oracle`, `LlmOracle`, `PrologTools` — natural language question answering over propositions | @@ -80,4 +82,7 @@ The `dice` module is organized by responsibility: - **Understanding the proposition data model** → `Proposition.kt` in `com.embabel.dice.proposition`. Every field is documented inline. - **Adding a new entity resolver strategy** → implement `CandidateSearcher` in `com.embabel.dice.common.resolver.searcher`, then compose it into an `EscalatingEntityResolver`. - **Writing integration tests against Neo4j** → look at `DrivinePropositionStoreIntegrationTest` in `dice-storage/src/test`; it shows the `@SpringBootTest` + Testcontainers pattern in use. -- **Understanding *why* the system behaves as it does** → `docs/design/` holds the design-decision notes — the conceptual model and the reasoning you can't recover by reading a class (e.g. the proposition lifecycle: trust, authority, supersession, decay). +- **Tuning what gets into the store** → admission gates in `com.embabel.dice.proposition.gate` (`ExtractionGatePipeline`, `StandardGates`); they run on pipeline output before the caller persists. +- **Running maintenance / consolidation** → `DefaultDreamLoopOrchestrator` (threshold-gated consolidation passes) or `DefaultMemoryMaintenanceOrchestrator` (the legacy four-step pipeline), both in `com.embabel.dice.projection.memory`. +- **Reclaiming stale or duplicate propositions** → `DefaultCollectorRunner` plus its `CollectorStrategy`/`SweepPolicy` in `com.embabel.dice.projection.memory`; runs are auditable via `CollectorRecordStore`. +- **Understanding *why* the system behaves as it does** → [`docs/design/`](docs/design/) holds the design-decision notes — the conceptual model and the reasoning you can't recover by reading a class: the extraction pipeline, the proposition lifecycle (trust, authority, supersession, decay), knowledge hygiene (gates, reclamation, consolidation), and the event model. diff --git a/dice/AGENTS.md b/dice/AGENTS.md index bb7eca64..2e73b619 100644 --- a/dice/AGENTS.md +++ b/dice/AGENTS.md @@ -33,6 +33,18 @@ Kotlin 2.1.10, Java 21. `embabel-agent-api` and `embabel-agent-rag-core` are `pr **`SourceAnalysisContext`** (`com.embabel.dice.common.SourceAnalysisContext`) — carries everything a pipeline run needs: schema (`DataDictionary`), entity resolver, `contextId`, optional known entities and template model. In Kotlin use infix builders; in Java use the `withXxx` builder chain. +## Knowledge hygiene: admission, reclamation, consolidation + +Three seams keep the store healthy at three different moments; all ship conservative, pluggable defaults. + +**Admission — gates.** `ExtractionGate` (`com.embabel.dice.proposition.gate`) runs on pipeline output *before* the caller persists. `ExtractionGatePipeline` chains gates; the first non-`Persist` decision wins. `StandardGates` ships confidence/trust/conflict/evidence-floor gates; `EvidenceFloorGate` demotes a weak relation rather than dropping the fact. + +**Reclamation — the mark-and-sweep collector.** `CollectorRunner`/`DefaultCollectorRunner` (`com.embabel.dice.projection.memory`) fetch ACTIVE candidates, gather marks from each `CollectorStrategy` (stale via `DecayCollectorStrategy`, duplicates via `DuplicateCollectorStrategy`), then let a `SweepPolicy` decide each one's fate (`TransitionStatus`/`HardDelete`/`Skip`). `run(dryRun = true)` previews without writing; `collect()` marks only. Every run is recorded in a `CollectorRecordStore` for audit. + +**Consolidation — the dream loop.** `DreamLoopOrchestrator`/`DefaultDreamLoopOrchestrator` run `ConsolidationPass`es (`operations.consolidation`) as a threshold-gated cycle: `SessionConsolidationPass`, `AbstractionPass` (drives supersession), `ContradictionResolutionPass` (drives contradiction), `DecaySweepPass` (drives stale). The orchestrator owns all writes and locks per `contextId` so cycles for one context never overlap. + +These map onto the lifecycle in [proposition-lifecycle](../docs/design/proposition-lifecycle.md); the design notes in [`docs/design/`](../docs/design/) explain the *why* behind each. + ## Package map | Package | Contents | @@ -41,7 +53,8 @@ Kotlin 2.1.10, Java 21. `embabel-agent-api` and `embabel-agent-rag-core` are `pr | `proposition.extraction` | `PropositionExtractor`, `LlmPropositionExtractor`, extraction config | | `proposition.revision` | `PropositionReviser`, `LlmPropositionReviser` | | `proposition.store` | `InMemoryPropositionRepository`, `JsonFilePropositionRepository`, `InMemoryDecayManager` | -| `pipeline` | `PropositionPipeline`, `PropositionResults`, persistence helpers | +| `proposition.gate` | Admission gates applied to pipeline output before persistence: `ExtractionGate`, `ExtractionGatePipeline`, `StandardGates`, `ObservableGate`, `GateDecision`, `GatedPropositionResult`, `EvidenceFloorGate` | +| `pipeline` | `PropositionPipeline` (two-stage extract→resolve), `PropositionResults`, `PersistablePropositions`; execution strategies: `ExtractionExecutionStrategy` + `SerialExtractionStrategy`/`ParallelExtractionStrategy`/`BatchedExtractionStrategy` | | `spi` | Policy extension points: `TrustScorer`, `AuthorityResolver`/`AuthorityTier`, `AuthorityWeightedTrustScorer`, `ConflictDetector`, `ConflictType`, `StatusTransitionPolicy`, and their shipped defaults | | `common` | `SourceAnalysisContext`, `EntityResolver`, `Relations`, `ContentHasher`, `KnowledgeType`, events, `SchemaAdherence`, validation rules | | `common.filter` | `MentionFilter`, `SchemaValidatedMentionFilter`, `ObservableMentionFilter`, context-aware filters | @@ -51,9 +64,9 @@ Kotlin 2.1.10, Java 21. `embabel-agent-api` and `embabel-agent-rag-core` are `pr | `incremental` | `AbstractIncrementalAnalyzer`, `ChunkHistoryStore`, `InMemoryChunkHistoryStore`, `ConversationSource`, `IncrementalSource` | | `incremental.proposition` | `PropositionIncrementalAnalyzer` | | `projection.graph` | `GraphProjector`, `RelationBasedGraphProjector`, `LlmGraphProjector`, `GraphProjectionService`, `ProjectionPolicy` | -| `projection.memory` | `MemoryProjector`, `MemoryRetriever`, `MemoryConsolidator`, `MemoryMaintenanceOrchestrator`, `DefaultMemoryProjector` | +| `projection.memory` | `MemoryProjector`/`DefaultMemoryProjector`, `MemoryRetriever`, `MemoryConsolidator`; **maintenance & dream loop**: `MemoryMaintenanceOrchestrator`/`DefaultMemoryMaintenanceOrchestrator`, `DreamLoopOrchestrator`/`DefaultDreamLoopOrchestrator`, `DreamLoopReport`; **mark-and-sweep collector**: `CollectorRunner`/`DefaultCollectorRunner`, `CollectorStrategy` (`DecayCollectorStrategy`, `DuplicateCollectorStrategy`), `SweepPolicy`/`StatusTransitionSweepPolicy`, `SweepAction`, `PropositionMark`, `MarkReason`, `CollectorRunResult` | | `projection.prolog` | `PrologProjector`, `PrologEngine`, Prolog type wrappers | -| `projection.lineage` | `ProjectionRecordStore`, `InMemoryProjectionRecordStore`, `Reconciler`, `ProjectionRecord`, `ProjectionRun` | +| `projection.lineage` | `ProjectionRecordStore`, `InMemoryProjectionRecordStore`, `Reconciler`, `ProjectionRecord`, `ProjectionRun`; **collector audit trail**: `CollectorRecordStore`/`InMemoryCollectorRecordStore`, `CollectorRecord`, `CollectorRun`, `CollectorOutcome` | | `projection.grounding` | `GroundingResolver`, `GroundingWiringService` | | `text2graph` | `KnowledgeGraphBuilder`, `SourceAnalyzer`, `LlmSourceAnalyzer`, `MultiPassKnowledgeGraphBuilder`, merge policies, relationship resolution | | `provenance` | `ProvenanceEntry`, `SourceLocator`, `UriLocator` | @@ -62,6 +75,7 @@ Kotlin 2.1.10, Java 21. `embabel-agent-api` and `embabel-agent-rag-core` are `pr | `agent` | `Memory`, `MemoryRetriever` (agent-facing view), `ProvenanceResolver` | | `web.rest` | `PropositionPipelineController`, `MemoryController`, API key security — optional, activated by `spring-webmvc` | | `operations` | `PropositionAbstractor`, `PropositionContraster` — higher-level proposition management | +| `operations.consolidation` | The dream-loop steps as composable passes: `ConsolidationPass`/`ConsolidationPassResult`, `SessionConsolidationPass`, `AbstractionPass`, `ContradictionResolutionPass`, `DecaySweepPass` | ## Gotchas From d5f785e0776d59de34b50d2dbd93c49cce74ff79 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 23:52:54 -0400 Subject: [PATCH 21/27] docs(design): add consolidation/dream-loop and reclamation/collector deep dives The knowledge-hygiene note explains why admission, reclamation, and consolidation exist; these two new notes explain how the latter two are built so a newcomer can follow them end to end. Consolidation-and-dream-loop covers the ConsolidationPass abstraction, the four passes, cycle composition, and threshold-gated/locked triggering. Reclamation-and-collector covers the mark strategies (including the union-find duplicate detection), the sweep policy, collect/dry-run/live entry points, and the audit trail. Cross-linked from knowledge-hygiene and the README; all diagrams parse. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- README.md | 6 +- docs/design/consolidation-and-dream-loop.md | 137 ++++++++++++++++++++ docs/design/knowledge-hygiene.md | 6 + docs/design/reclamation-and-collector.md | 130 +++++++++++++++++++ 4 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 docs/design/consolidation-and-dream-loop.md create mode 100644 docs/design/reclamation-and-collector.md diff --git a/README.md b/README.md index ee210f9e..b31558de 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,11 @@ recover by reading a single class — see the design notes in [`docs/design/`](d - [Proposition lifecycle](docs/design/proposition-lifecycle.md) — trust scoring, source authority, conflict classification, supersession versus contradiction, and decay instead of deletion. - [Knowledge hygiene](docs/design/knowledge-hygiene.md) — admission gates, mark-and-sweep - reclamation, and the consolidation dream loop. + reclamation, and the consolidation dream loop (the "why" overview of the three interventions). +- [Consolidation and the dream loop](docs/design/consolidation-and-dream-loop.md) — the pass + abstraction, the four consolidation passes, and how a cycle composes and is triggered. +- [Reclamation and the collector](docs/design/reclamation-and-collector.md) — the mark-and-sweep + internals: strategies, sweep policy, dry-run vs. live, and the audit trail. - [Events](docs/design/events.md) — the domain-event model the store and pipeline emit. ## Real-World Example: Impromptu diff --git a/docs/design/consolidation-and-dream-loop.md b/docs/design/consolidation-and-dream-loop.md new file mode 100644 index 00000000..271178a0 --- /dev/null +++ b/docs/design/consolidation-and-dream-loop.md @@ -0,0 +1,137 @@ +# Consolidation and the dream loop: passes, composition, and the cycle + +Admission and reclamation keep bad data out and reclaim what's no longer useful, but they don't make +good data *better*. That's consolidation's job: fold a session's raw facts into long-term memory, +abstract clusters of related facts into higher-level ones, resolve lingering contradictions, and +retire what has decayed. The [knowledge-hygiene](knowledge-hygiene.md) note explains *why* +consolidation is a separate intervention; this note is about *how* it's built — the pass abstraction, +the four passes that ship, and how the dream loop composes them into one repeatable cycle. + +```mermaid +flowchart LR + SNAP["ACTIVE snapshot
(fetched once)"] --> ORCH + subgraph ORCH ["Dream-loop cycle · one write at the end"] + direction TB + P1[SessionConsolidationPass] --> P2[AbstractionPass] + P2 --> P3[ContradictionResolutionPass] + P3 --> P4[DecaySweepPass] + end + ORCH --> AGG["aggregate every pass's
propositionsToSave"] + AGG --> STORE[(Proposition store)] + ORCH --> REPORT["DreamLoopReport
(counts per cycle)"] +``` + +## The pass abstraction + +Every consolidation step is a `ConsolidationPass`: it takes the context's snapshot of propositions, +decides what should change, and returns a `ConsolidationPassResult` — **it never writes anything +itself**. That purity contract is the whole design. The orchestrator fetches the snapshot once, runs +each pass over it, and applies the accumulated changes in a single write per cycle, so passes stay +small, independently testable, and reorderable. + +A result is one of three things, sealed so the orchestrator handles every case: + +- **`Changed`** — propositions to upsert (`propositionsToSave`), ids to hard-delete + (`propositionsToDelete`, honored only when the orchestrator opts in), a `skipped` count, and an + `externallyApplied` count for transitions a pass already wrote through its own machinery. +- **`NoOp`** — nothing to do; the snapshot is already settled for this pass. Returning `NoOp` rather + than an empty `Changed` is how the loop detects convergence. +- **`Failed`** — the pass threw; the orchestrator records it and continues with the remaining passes + rather than aborting the cycle. + +Passes must be **idempotent**: running one twice over an unchanged snapshot must yield `NoOp`, because +the loop runs passes repeatedly until the snapshot settles. A pass that always reports `Changed` would +loop forever. + +## The four passes that ship + +**`SessionConsolidationPass`** folds a session's propositions into long-term memory by delegating to a +`MemoryConsolidator` — promoting, reinforcing, or merging against the existing ACTIVE set. The session +is supplied at construction; the snapshot it runs against is the long-term set, not the session. + +**`AbstractionPass`** groups level-0 ACTIVE propositions by resolved entity, hands each group above a +size threshold to a `PropositionAbstractor`, and marks the consumed sources `SUPERSEDED` — this is +what drives **supersession** in the lifecycle. Two guards keep it from inflating forever: an +*idempotency guard* skips a group already covered by a higher-level proposition +(`sourceIds.containsAll(memberIds)`), and a `maxLevel` cap drops any abstraction above the ceiling. + +**`ContradictionResolutionPass`** asks a `PropositionReviser` to classify each ACTIVE proposition +against its entity-overlapping peers; for each pair the reviser calls `CONTRADICTORY`, the lower +effective confidence loses and is transitioned to `CONTRADICTED` — this drives **contradiction**. It +prunes candidates to entity-overlapping peers (bounding the classify fan-out) and resolves each +unordered pair once, so a symmetric reviser can't retire both members and leave no survivor. + +**`DecaySweepPass`** delegates to the mark-and-sweep collector (see +[reclamation-and-collector](reclamation-and-collector.md)) to move decayed propositions toward +`STALE`. It is **report-only**: the collector writes the transitions itself, so the pass returns empty +save/delete lists and reports the count via `externallyApplied`, avoiding a double write. + +## How a cycle composes them + +`consolidateNow` is the cycle. It does six things in order, and the shape is deliberate: + +```mermaid +sequenceDiagram + autonumber + participant Orch as DreamLoopOrchestrator + participant Store + participant Pass as Each ConsolidationPass + Orch->>Store: fetch the ACTIVE snapshot once + loop passes in registration order + Orch->>Pass: run(contextId, snapshot) + Pass-->>Orch: Changed / NoOp / Failed (a Failed pass is logged, not fatal) + end + Orch->>Orch: aggregate every Changed pass's propositionsToSave + Orch->>Store: one saveAll for the whole cycle + opt allowHardDelete + Orch->>Store: delete aggregated delete-ids + end + Orch->>Orch: record post-cycle ACTIVE count, then build DreamLoopReport +``` + +Aggregating into a single write is the reason passes don't write themselves: one cycle, one +`saveAll`, regardless of how many passes contributed. Hard deletes are off by default — a pass can +always *express* a delete intent without risking data loss, and only an orchestrator built with +`allowHardDelete` acts on it. + +The `DreamLoopReport` rolls up what happened: `totalExamined` (snapshot size), `totalNewPropositions` +(propositions whose id wasn't in the snapshot — genuinely new, like a fresh abstraction), and +`totalTransitioned` (re-saved snapshot members plus deletes plus each pass's `externallyApplied`). + +## Triggering: threshold-gated vs. unconditional + +There are two ways in. `consolidateNow` always runs a cycle. `consolidate` is the cheap, +run-it-often entry point: it only does the expensive work when enough has accumulated since the last +cycle. + +```mermaid +flowchart TB + C["consolidate(contextId)"] --> D{"active-count delta
≥ changeVolumeThreshold?"} + D -->|no| SKIP["record the count, return null
(cheap when nothing changed)"] + D -->|yes| RUN["consolidateNow(contextId)"] + RUN --> REPORT[DreamLoopReport] +``` + +The trigger tracks the ACTIVE count per context (`lastActiveCount`) and fires when the delta reaches +`changeVolumeThreshold`. Both entry points hold a **per-context lock** for the whole gate-plus-cycle, +so two triggers for the same context can't both pass the gate on the same stale count and run +overlapping cycles. Different contexts lock independently and still run in parallel; the lock lives on +the orchestrator instance, so a single shared instance is the unit of serialization. + +## Where this sits relative to revision + +Contradiction shows up in two places, and they're not the same moment. At **ingest**, the +`PropositionReviser` (see [proposition-lifecycle](proposition-lifecycle.md)) classifies a *newly +arriving* proposition against what's stored and demotes the loser immediately. The +`ContradictionResolutionPass` runs later, during a quiet consolidation cycle, sweeping for +contradictions *among already-stored* propositions that no single ingest caught. Same lifecycle +transition (`CONTRADICTED`), different trigger — one reactive, one background. Likewise, abstraction +is where most **supersession** happens, long after the source facts were ingested. + +## Configurable behavior + +The pass list, the consolidator and abstractor and reviser each pass delegates to, the +`changeVolumeThreshold`, and whether hard deletes are allowed are all configurable through the +orchestrator's copy-builders. What ships is conservative — soft transitions over deletes, consolidate +only past a threshold — so the safe behavior is the default and a deployment opts into anything more +aggressive. diff --git a/docs/design/knowledge-hygiene.md b/docs/design/knowledge-hygiene.md index b3d421cb..d2d3e04a 100644 --- a/docs/design/knowledge-hygiene.md +++ b/docs/design/knowledge-hygiene.md @@ -52,6 +52,9 @@ Reclamation borrows the shape of a tracing garbage collector on purpose: one sta looks like garbage*, a separate stage decides *what to do about it*, and every action leaves a record. +The internals — the strategies, the sweep policy, the entry points, and the audit trail — are in +[reclamation-and-collector](reclamation-and-collector.md); this section is the why. + Splitting "mark" from "sweep" keeps two independent judgments independent. A marker flags a proposition with a reason — it's gone **stale**, it's a **duplicate** of a survivor, or some domain-specific reason — without committing to a fate. The sweep policy then chooses the fate: @@ -100,6 +103,9 @@ Admission and reclamation keep the store from filling with bad data, but they do *better*. That's the job of the dream loop: a set of consolidation passes that run as repeatable cycles, ideally during idle time, to tidy what's already there. +How the passes compose into a cycle, and how the loop is triggered and locked, is in +[consolidation-and-dream-loop](consolidation-and-dream-loop.md); this section is the why. + The passes are composable and each does one thing: fold a session's raw facts together, abstract a cluster of related facts into a higher-level proposition, resolve lingering contradictions, and sweep decayed entries. Composability is the design decision — consolidation isn't one monolithic diff --git a/docs/design/reclamation-and-collector.md b/docs/design/reclamation-and-collector.md new file mode 100644 index 00000000..dfce4228 --- /dev/null +++ b/docs/design/reclamation-and-collector.md @@ -0,0 +1,130 @@ +# Reclamation and the collector: mark, sweep, and the audit trail + +Some propositions stop being worth keeping — they decay until no one believes them, or a newer +duplicate makes them redundant. Reclamation is the intervention that finds and retires them, and it's +built as a tracing garbage collector on purpose: one stage decides *what looks like garbage*, a +separate stage decides *what to do about it*, and every action leaves a record. The +[knowledge-hygiene](knowledge-hygiene.md) note covers *why* mark and sweep are kept apart; this note +is about *how* the collector is built — the strategies, the sweep policy, the entry points, and the +audit trail. + +```mermaid +flowchart LR + Q["ACTIVE candidates
(queried once)"] --> MARK + subgraph MARK ["Mark phase · descriptive, no writes"] + direction TB + S1[DecayCollectorStrategy] --> M["PropositionMarks
(reason: stale / duplicate / custom)"] + S2[DuplicateCollectorStrategy] --> M + end + M --> SWEEP{"SweepPolicy.decide"} + SWEEP -->|TransitionStatus| T["→ STALE (reversible default)"] + SWEEP -->|HardDelete| D["remove (opt-in only)"] + SWEEP -->|Skip| K["leave untouched"] + T --> REC[CollectorRecordStore] + D --> REC + K --> REC +``` + +## The mark phase: strategies and marks + +A `CollectorStrategy` inspects the candidate set and flags propositions, producing `PropositionMark`s +— each carries the proposition id, the strategy name, and a typed `MarkReason` (`Stale`, `Duplicate`, +or a domain-specific `Custom`). Marking is **purely descriptive**: a strategy never mutates anything, +it only reports what looks reclaimable. Two strategies ship: + +**`DecayCollectorStrategy`** marks a proposition whose `effectiveConfidence()` has fallen below a +retirement threshold — the decay path into staleness (decay itself is covered in +[proposition-lifecycle](proposition-lifecycle.md)). It reads only; the fate is the sweep's call. + +**`DuplicateCollectorStrategy`** finds redundant propositions by treating "is a duplicate of" as edges +in a graph and collapsing each connected component to one survivor. It uses union-find so that +overlapping pairs (A≈B, B≈C) resolve to a single cluster {A,B,C} rather than fighting over who merges +with whom — the result is deterministic regardless of the order pairs are discovered. Within a cluster +the survivor is the strongest member (highest effective confidence, ties broken by id), and everyone +else is marked `Duplicate`. + +```mermaid +flowchart TB + subgraph found ["Discovered near-duplicate pairs"] + AB["A ≈ B"] + BC["B ≈ C"] + DE["D ≈ E"] + end + found --> UF["union-find:
connected components"] + UF --> C1["cluster {A, B, C}"] + UF --> C2["cluster {D, E}"] + C1 --> W1["survivor = strongest
(conf, then id); mark the rest Duplicate"] + C2 --> W2["survivor = strongest;
mark the rest Duplicate"] +``` + +## The sweep phase: policy and actions + +A `SweepPolicy` looks at a proposition and its marks and returns a `SweepAction` — `TransitionStatus`, +`HardDelete`, or `Skip`. The policy only *decides*; applying the action is the runner's job, which +keeps "what counts as garbage" (strategies) independent from "what happens to it" (policy). + +The default `StatusTransitionSweepPolicy` is deliberately safe: it skips pinned propositions no matter +what marks they carry, skips anything unmarked, and otherwise transitions to `STALE`. It **never** +returns `HardDelete` — the shipped default is reversible, and destructive removal is something a +deployment opts into with a different policy. + +## Entry points: collect, dry run, live run + +The `CollectorRunner` has two entry points and the run has two modes, separating "what would happen" +from "make it happen": + +```mermaid +flowchart TB + START["query ACTIVE candidates, run the mark phase"] --> WHICH{entry point} + WHICH -->|"collect()"| COLLECT["return marks only
no writes, no record"] + WHICH -->|"run(dryRun = true)"| DRY["decide each fate, write nothing,
record outcomes as MARKED (preview)"] + WHICH -->|"run(dryRun = false)"| LIVE["apply each action, emit a
PropositionStatusChanged, record outcomes"] +``` + +`collect()` is mark-only — useful when a caller wants to see candidates without touching anything. A +**dry run** decides every fate and writes the audit trail but mutates nothing, so a policy can be +previewed against real data before it's let loose. A **live run** applies each decision, emits a +`PropositionStatusChanged` per transition (the same event the store emits, so a collector transition +looks identical to any other — see [events](events.md)), and records every outcome. A +`CollectorRunResult` partitions what happened into `marks`, `applied`, `skipped`, and `hardDeleted`. + +## The audit trail + +Reclamation is built to be explainable after the fact. When a `CollectorRecordStore` is configured, +each run writes one `CollectorRun` header and a `CollectorRecord` per acted-upon proposition. A record +carries the typed reason, the `CollectorOutcome`, and the before/after status, so a reviewer can trace +exactly why each proposition was touched and what happened to it. + +The store is **append-only**, and a run header is written even for a zero-mark run, so "the collector +ran and found nothing" is a retrievable fact rather than silence. The outcome vocabulary keeps a +preview honest: a dry run records `MARKED` (would-be), a live transition records `TRANSITIONED`, a +removal records `HARD_DELETED`, and an exempt proposition records `SKIPPED` — and the `CollectorRun`'s +`dryRun` flag is the authoritative discriminator between a preview and the real thing. + +```mermaid +flowchart LR + RUN["run"] --> HDR["CollectorRun header
(runId, dryRun, timing)"] + RUN --> RECS["one CollectorRecord per proposition"] + RECS --> O{CollectorOutcome} + O --> MK["MARKED · dry-run preview"] + O --> TR["TRANSITIONED · live status change"] + O --> HD["HARD_DELETED · live removal"] + O --> SK["SKIPPED · exempt / unmarked"] +``` + +## How decay reclamation rejoins consolidation + +The decay sweep is where reclamation and consolidation meet: `DecaySweepPass` is a thin consolidation +pass that drives a collector run for its context (see +[consolidation-and-dream-loop](consolidation-and-dream-loop.md)). Because the collector writes the +`STALE` transitions itself, the pass reports them as `externallyApplied` rather than handing +propositions back to the orchestrator — one transition, written once, counted once. So the same +mark-and-sweep machinery serves both an on-demand reclamation run and the decay step of a dream-loop +cycle. + +## Configurable behavior + +The strategy list, the sweep policy, and whether an audit store is attached are all pluggable. What +ships is cautious — mark on decay and duplication, sweep to a reversible `STALE`, never hard-delete by +default, and record everything — so the safe behavior is the default and a deployment opts into +custom strategies, hard deletion, or domain-specific mark reasons as it needs them. From d1a4b997c821d5cd5fb9659719000489d3b34907 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 22 Jun 2026 23:59:03 -0400 Subject: [PATCH 22/27] test: cover the dream-loop lock, decay policy, and an end-to-end reclamation flow Add the gaps the subsystems lacked: two concurrency regression tests proving the dream-loop per-context lock serializes same-context cycles while letting different contexts run in parallel; a direct unit test for DecayStatusPolicy.evaluate (the hysteresis band and the importance/reinforce utility weights, which only config defaults exercised before); and a no-mocks end-to-end test that runs a real decay sweep through the orchestrator, collector, store, events, and audit trail together. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../memory/DreamLoopOrchestratorTest.kt | 49 ++++++++ .../DreamLoopReclamationEndToEndTest.kt | 107 ++++++++++++++++++ .../embabel/dice/spi/DecayStatusPolicyTest.kt | 103 +++++++++++++++++ 3 files changed, 259 insertions(+) create mode 100644 dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopReclamationEndToEndTest.kt create mode 100644 dice/src/test/kotlin/com/embabel/dice/spi/DecayStatusPolicyTest.kt diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestratorTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestratorTest.kt index 73771f7f..18f7a3a0 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestratorTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestratorTest.kt @@ -266,4 +266,53 @@ class DreamLoopOrchestratorTest { assertEquals(1, after.runCount) assertFalse(report.passResults.none { it.passName == "after" }) } + + @Test + fun `concurrent cycles for the same context never overlap`() { + every { repository.query(any()) } returns listOf(proposition("p1")) + val active = java.util.concurrent.atomic.AtomicInteger(0) + val maxObserved = java.util.concurrent.atomic.AtomicInteger(0) + // A pass that widens the window where two cycles could overlap, recording the peak + // concurrency it ever observes. With the per-context lock, that peak must stay at 1. + val probe = object : ConsolidationPass { + override val name = "concurrency-probe" + override fun run(contextId: ContextId, propositions: List): ConsolidationPassResult { + val now = active.incrementAndGet() + maxObserved.updateAndGet { m -> maxOf(m, now) } + Thread.sleep(20) + active.decrementAndGet() + return ConsolidationPassResult.NoOp(name) + } + } + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository).withPass(probe) + + val threads = (1..8).map { Thread { orchestrator.consolidateNow(contextId) } } + threads.forEach { it.start() } + threads.forEach { it.join() } + + assertEquals(1, maxObserved.get(), "the per-context lock must serialize cycles for one context") + } + + @Test + fun `cycles for different contexts are not serialized against each other`() { + every { repository.query(any()) } returns listOf(proposition("p1")) + // Both contexts must be inside the pass at the same time for the barrier to release. A + // global lock would block the second context and time the barrier out; per-context locks + // let both proceed, proving the locking is per context, not global. + val barrier = java.util.concurrent.CyclicBarrier(2) + val rendezvous = object : ConsolidationPass { + override val name = "rendezvous" + override fun run(contextId: ContextId, propositions: List): ConsolidationPassResult { + barrier.await(2, java.util.concurrent.TimeUnit.SECONDS) + return ConsolidationPassResult.NoOp(name) + } + } + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository).withPass(rendezvous) + val errors = java.util.concurrent.CopyOnWriteArrayList() + val a = Thread { runCatching { orchestrator.consolidateNow(ContextId("ctx-a")) }.onFailure { errors.add(it) } } + val b = Thread { runCatching { orchestrator.consolidateNow(ContextId("ctx-b")) }.onFailure { errors.add(it) } } + a.start(); b.start(); a.join(); b.join() + + assertTrue(errors.isEmpty(), "different contexts must run concurrently, not block each other: $errors") + } } diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopReclamationEndToEndTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopReclamationEndToEndTest.kt new file mode 100644 index 00000000..6c62ab37 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopReclamationEndToEndTest.kt @@ -0,0 +1,107 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import com.embabel.dice.common.PropositionStatusChanged +import com.embabel.dice.common.RecordingDiceEventListener +import com.embabel.dice.operations.consolidation.DecaySweepPass +import com.embabel.dice.projection.lineage.CollectorOutcome +import com.embabel.dice.projection.lineage.InMemoryCollectorRecordStore +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionQuery +import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.proposition.store.InMemoryPropositionRepository +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant +import java.time.temporal.ChronoUnit + +/** + * End-to-end integration test for the decay path through the maintenance stack, wiring real + * components with no mocks: a real proposition store, the mark-and-sweep collector with its + * decay strategy and default sweep policy, an audit record store, an event listener, and the + * dream-loop orchestrator driving a [DecaySweepPass]. It proves the whole chain — orchestrator → + * pass → collector → strategy → sweep policy → store writes → lifecycle events → audit trail → + * cycle report — behaves coherently together, not just in isolation. + */ +class DreamLoopReclamationEndToEndTest { + + private val contextId = ContextId("e2e-context") + + private fun proposition( + text: String, + confidence: Double, + decay: Double, + contentRevised: Instant, + ): Proposition = Proposition( + contextId = contextId, + text = text, + mentions = emptyList(), + confidence = confidence, + decay = decay, + contentRevised = contentRevised, + ) + + @Test + fun `a dream-loop decay sweep retires decayed propositions to STALE across the whole stack`() { + val repository = InMemoryPropositionRepository() + val recordStore = InMemoryCollectorRecordStore() + val listener = RecordingDiceEventListener() + + val now = Instant.now() + val longAgo = now.minus(365, ChronoUnit.DAYS) + // Fresh and confident: stays ACTIVE. Two old, heavily-decayed facts: should retire. + val fresh = proposition("fresh fact", confidence = 0.9, decay = 0.1, contentRevised = now) + val decayedA = proposition("old fact A", confidence = 0.5, decay = 0.5, contentRevised = longAgo) + val decayedB = proposition("old fact B", confidence = 0.5, decay = 0.5, contentRevised = longAgo) + repository.saveAll(listOf(fresh, decayedA, decayedB)) + + val collector = CollectorRunner + .withRepository(repository) + .withStrategy(DecayCollectorStrategy(retireBelow = 0.3)) + .withRecordStore(recordStore) + .withEventListener(listener) + .build() + val orchestrator = DefaultDreamLoopOrchestrator + .withRepository(repository) + .withPass(DecaySweepPass(collector)) + + val report = orchestrator.consolidateNow(contextId) + + // The store reflects the transitions: only the fresh fact is still ACTIVE. + val active = repository.query(PropositionQuery.forContextId(contextId).withStatus(PropositionStatus.ACTIVE)) + val stale = repository.query(PropositionQuery.forContextId(contextId).withStatus(PropositionStatus.STALE)) + assertEquals(setOf("fresh fact"), active.map { it.text }.toSet()) + assertEquals(setOf("old fact A", "old fact B"), stale.map { it.text }.toSet()) + + // Each retirement emitted the standard lifecycle event, so observers see collector + // transitions identically to any other status change. + val events = listener.eventsOfType() + assertEquals(2, events.size) + assertTrue(events.all { it.newStatus == PropositionStatus.STALE }) + + // The run is auditable: one run header plus a TRANSITIONED record per swept proposition. + assertEquals(1, recordStore.runs().size) + val records = recordStore.all() + assertEquals(2, records.size) + assertTrue(records.all { it.outcome == CollectorOutcome.TRANSITIONED }) + + // The cycle report counts the externally-applied decay transitions. + assertEquals(2, report.totalTransitioned) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/spi/DecayStatusPolicyTest.kt b/dice/src/test/kotlin/com/embabel/dice/spi/DecayStatusPolicyTest.kt new file mode 100644 index 00000000..cfc5d3cb --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/spi/DecayStatusPolicyTest.kt @@ -0,0 +1,103 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.spi + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test + +/** + * Direct tests for [DecayStatusPolicy.evaluate] — the hysteresis band and the importance/reinforce + * utility weighting. Propositions are created with `decay = 0.0`, so a fresh proposition's effective + * confidence equals its raw confidence and the utility math is deterministic regardless of timing. + */ +class DecayStatusPolicyTest { + + private val contextId = ContextId("test-context") + + private fun proposition( + confidence: Double, + status: PropositionStatus = PropositionStatus.ACTIVE, + importance: Double = 0.5, + reinforceCount: Int = 0, + pinned: Boolean = false, + ): Proposition = Proposition( + contextId = contextId, + text = "p", + mentions = emptyList(), + confidence = confidence, + decay = 0.0, + importance = importance, + reinforceCount = reinforceCount, + pinned = pinned, + status = status, + ) + + // Default thresholds: staleness 0.1, recovery 0.2; default weights are 0.0. + + @Test + fun `pinned propositions never transition`() { + assertNull(DecayStatusPolicy().evaluate(proposition(confidence = 0.01, pinned = true))) + } + + @Test + fun `ACTIVE below the staleness floor goes STALE`() { + assertEquals(PropositionStatus.STALE, DecayStatusPolicy().evaluate(proposition(confidence = 0.05))) + } + + @Test + fun `ACTIVE inside the hysteresis band does not transition`() { + // 0.15 is above the staleness floor (0.1) but below recovery (0.2): no change. + assertNull(DecayStatusPolicy().evaluate(proposition(confidence = 0.15))) + } + + @Test + fun `STALE above the recovery ceiling returns to ACTIVE`() { + assertEquals( + PropositionStatus.ACTIVE, + DecayStatusPolicy().evaluate(proposition(confidence = 0.9, status = PropositionStatus.STALE)), + ) + } + + @Test + fun `STALE inside the hysteresis band does not revive`() { + assertNull(DecayStatusPolicy().evaluate(proposition(confidence = 0.15, status = PropositionStatus.STALE))) + } + + @Test + fun `importance weight can lift utility above the staleness floor`() { + // With default weights, 0.08 < 0.1 retires. + assertEquals(PropositionStatus.STALE, DecayStatusPolicy().evaluate(proposition(confidence = 0.08, importance = 1.0))) + // With importance weighting, utility = 0.08 * (1 + 0.5 * 1.0) = 0.12 >= 0.1, so it stays ACTIVE. + assertNull(DecayStatusPolicy(importanceWeight = 0.5).evaluate(proposition(confidence = 0.08, importance = 1.0))) + } + + @Test + fun `reinforce weight can lift utility above the staleness floor`() { + // With default weights, 0.095 < 0.1 retires. + assertEquals(PropositionStatus.STALE, DecayStatusPolicy().evaluate(proposition(confidence = 0.095, reinforceCount = 10))) + // ln(1 + 10) ~= 2.40, so utility = 0.095 * (1 + 0.1 * 2.40) ~= 0.118 >= 0.1, staying ACTIVE. + assertNull(DecayStatusPolicy(reinforceWeight = 0.1).evaluate(proposition(confidence = 0.095, reinforceCount = 10))) + } + + @Test + fun `a status other than ACTIVE or STALE never transitions`() { + assertNull(DecayStatusPolicy().evaluate(proposition(confidence = 0.01, status = PropositionStatus.CONTRADICTED))) + } +} From 62543042e24aadb379c028b74f932d8294ffc00c Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Tue, 23 Jun 2026 00:26:28 -0400 Subject: [PATCH 23/27] refactor(spi): move the sweep-policy cluster into the spi package SweepPolicy/StatusTransitionSweepPolicy and their SweepAction/PropositionMark/ MarkReason vocabulary move from projection.memory into com.embabel.dice.spi, sitting alongside StatusTransitionPolicy. Deciding a marked proposition's lifecycle fate is a policy seam, the same kind of plug-point as the trust, authority, conflict, and status policies already in spi. The cluster depends only on the core proposition model, so spi keeps its 'depends on core only' layering; the collector in projection.memory now imports these from spi (the correct feature -> spi direction). Pure move, no behaviour change. Also relocate StatusTransitionSweepPolicyTest into the spi test package to match the DecayStatusPolicyTest convention. Note: moving these public types is a source-incompatible change for any external importer of the old package (acceptable pre-1.0). Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- AGENTS.md | 6 +++--- dice/AGENTS.md | 6 +++--- .../embabel/dice/operations/consolidation/DecaySweepPass.kt | 4 +++- .../com/embabel/dice/projection/lineage/CollectorRecord.kt | 2 +- .../embabel/dice/projection/memory/CollectorRunResult.kt | 1 + .../com/embabel/dice/projection/memory/CollectorRunner.kt | 2 ++ .../com/embabel/dice/projection/memory/CollectorStrategy.kt | 2 ++ .../dice/projection/memory/DecayCollectorStrategy.kt | 2 ++ .../dice/projection/memory/DefaultCollectorRunner.kt | 3 +++ .../dice/projection/memory/DuplicateCollectorStrategy.kt | 2 ++ .../embabel/dice/{projection/memory => spi}/MarkReason.kt | 2 +- .../dice/{projection/memory => spi}/PropositionMark.kt | 2 +- .../embabel/dice/{projection/memory => spi}/SweepAction.kt | 2 +- .../embabel/dice/{projection/memory => spi}/SweepPolicy.kt | 2 +- .../dice/operations/consolidation/DecaySweepPassTest.kt | 6 +++--- .../embabel/dice/projection/lineage/CollectorRecordTest.kt | 2 +- .../projection/lineage/InMemoryCollectorRecordStoreTest.kt | 2 +- .../dice/projection/memory/CollectorRunnerBuilderTest.kt | 1 + .../embabel/dice/projection/memory/CollectorStrategyTest.kt | 2 ++ .../dice/projection/memory/DecayCollectorStrategyTest.kt | 1 + .../dice/projection/memory/DefaultCollectorRunnerTest.kt | 5 +++++ .../projection/memory/DuplicateCollectorStrategyTest.kt | 1 + .../memory => spi}/StatusTransitionSweepPolicyTest.kt | 2 +- 23 files changed, 42 insertions(+), 18 deletions(-) rename dice/src/main/kotlin/com/embabel/dice/{projection/memory => spi}/MarkReason.kt (97%) rename dice/src/main/kotlin/com/embabel/dice/{projection/memory => spi}/PropositionMark.kt (97%) rename dice/src/main/kotlin/com/embabel/dice/{projection/memory => spi}/SweepAction.kt (97%) rename dice/src/main/kotlin/com/embabel/dice/{projection/memory => spi}/SweepPolicy.kt (98%) rename dice/src/test/kotlin/com/embabel/dice/{projection/memory => spi}/StatusTransitionSweepPolicyTest.kt (98%) diff --git a/AGENTS.md b/AGENTS.md index d70cbef4..36338089 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,11 +50,11 @@ The `dice` module is organized by responsibility: | `com.embabel.dice.proposition.store` | In-memory and JSON-file implementations of `PropositionRepository` and `DecayManager` | | `com.embabel.dice.proposition.gate` | Admission gates that route pipeline output before persistence (`ExtractionGate`, `ExtractionGatePipeline`, `StandardGates`, `EvidenceFloorGate`) | | `com.embabel.dice.pipeline` | `PropositionPipeline` — the two-stage entry point (concurrent extraction → serial entity resolution); pluggable `ExtractionExecutionStrategy` (serial / parallel / batched) | -| `com.embabel.dice.spi` | Policy extension points consumers plug into: `TrustScorer`, `AuthorityResolver`/`AuthorityTier`, `AuthorityWeightedTrustScorer`, `ConflictDetector`, `ConflictType`, `StatusTransitionPolicy`, and their shipped defaults | +| `com.embabel.dice.spi` | Policy extension points consumers plug into: `TrustScorer`, `AuthorityResolver`/`AuthorityTier`, `AuthorityWeightedTrustScorer`, `ConflictDetector`, `ConflictType`, `StatusTransitionPolicy`, the mark-and-sweep `SweepPolicy`/`StatusTransitionSweepPolicy` (with its `SweepAction`/`PropositionMark`/`MarkReason` vocabulary), and their shipped defaults | | `com.embabel.dice.common` | Shared types: `SourceAnalysisContext`, `EntityResolver`, `Relations`, `ContentHasher`, `KnowledgeType`, events, validation rules, mention filters, resolver implementations | | `com.embabel.dice.entity` | Entity-only pipeline (`EntityPipeline`, `EntityIncrementalAnalyzer`, `EntityResolutionService`, `EntityResolutionTools`) | | `com.embabel.dice.incremental` | `AbstractIncrementalAnalyzer`, `ChunkHistoryStore`, `ConversationSource`, windowed processing | -| `com.embabel.dice.projection` | Projectors (`GraphProjector`, `PrologProjector`, `MemoryProjector`); memory maintenance and the dream loop (`MemoryMaintenanceOrchestrator`, `DreamLoopOrchestrator`); the mark-and-sweep collector (`CollectorRunner`, `CollectorStrategy`, `SweepPolicy`) and its audit trail (`CollectorRecordStore`); lineage tracking (`ProjectionRecordStore`, `Reconciler`) | +| `com.embabel.dice.projection` | Projectors (`GraphProjector`, `PrologProjector`, `MemoryProjector`); memory maintenance and the dream loop (`MemoryMaintenanceOrchestrator`, `DreamLoopOrchestrator`); the mark-and-sweep collector (`CollectorRunner`, `CollectorStrategy`; the `SweepPolicy` lives in `spi`) and its audit trail (`CollectorRecordStore`); lineage tracking (`ProjectionRecordStore`, `Reconciler`) | | `com.embabel.dice.operations` | Higher-level proposition operations (`PropositionAbstractor`, `PropositionContraster`) and, under `operations.consolidation`, the passes that compose the dream loop (`SessionConsolidationPass`, `AbstractionPass`, `ContradictionResolutionPass`, `DecaySweepPass`) | | `com.embabel.dice.text2graph` | Text-to-graph builder pipeline (`KnowledgeGraphBuilder`, `SourceAnalyzer`, entity merge/relationship resolution policies) | | `com.embabel.dice.provenance` | `ProvenanceEntry`, `SourceLocator` — rich evidence links from propositions back to source material | @@ -84,5 +84,5 @@ The `dice` module is organized by responsibility: - **Writing integration tests against Neo4j** → look at `DrivinePropositionStoreIntegrationTest` in `dice-storage/src/test`; it shows the `@SpringBootTest` + Testcontainers pattern in use. - **Tuning what gets into the store** → admission gates in `com.embabel.dice.proposition.gate` (`ExtractionGatePipeline`, `StandardGates`); they run on pipeline output before the caller persists. - **Running maintenance / consolidation** → `DefaultDreamLoopOrchestrator` (threshold-gated consolidation passes) or `DefaultMemoryMaintenanceOrchestrator` (the legacy four-step pipeline), both in `com.embabel.dice.projection.memory`. -- **Reclaiming stale or duplicate propositions** → `DefaultCollectorRunner` plus its `CollectorStrategy`/`SweepPolicy` in `com.embabel.dice.projection.memory`; runs are auditable via `CollectorRecordStore`. +- **Reclaiming stale or duplicate propositions** → `DefaultCollectorRunner` and its `CollectorStrategy` in `com.embabel.dice.projection.memory` (the `SweepPolicy` that decides each fate lives in `com.embabel.dice.spi`); runs are auditable via `CollectorRecordStore`. - **Understanding *why* the system behaves as it does** → [`docs/design/`](docs/design/) holds the design-decision notes — the conceptual model and the reasoning you can't recover by reading a class: the extraction pipeline, the proposition lifecycle (trust, authority, supersession, decay), knowledge hygiene (gates, reclamation, consolidation), and the event model. diff --git a/dice/AGENTS.md b/dice/AGENTS.md index 2e73b619..8fbbd8a8 100644 --- a/dice/AGENTS.md +++ b/dice/AGENTS.md @@ -39,7 +39,7 @@ Three seams keep the store healthy at three different moments; all ship conserva **Admission — gates.** `ExtractionGate` (`com.embabel.dice.proposition.gate`) runs on pipeline output *before* the caller persists. `ExtractionGatePipeline` chains gates; the first non-`Persist` decision wins. `StandardGates` ships confidence/trust/conflict/evidence-floor gates; `EvidenceFloorGate` demotes a weak relation rather than dropping the fact. -**Reclamation — the mark-and-sweep collector.** `CollectorRunner`/`DefaultCollectorRunner` (`com.embabel.dice.projection.memory`) fetch ACTIVE candidates, gather marks from each `CollectorStrategy` (stale via `DecayCollectorStrategy`, duplicates via `DuplicateCollectorStrategy`), then let a `SweepPolicy` decide each one's fate (`TransitionStatus`/`HardDelete`/`Skip`). `run(dryRun = true)` previews without writing; `collect()` marks only. Every run is recorded in a `CollectorRecordStore` for audit. +**Reclamation — the mark-and-sweep collector.** `CollectorRunner`/`DefaultCollectorRunner` (`com.embabel.dice.projection.memory`) fetch ACTIVE candidates, gather marks from each `CollectorStrategy` (stale via `DecayCollectorStrategy`, duplicates via `DuplicateCollectorStrategy`), then let a `SweepPolicy` (in `spi` with `StatusTransitionPolicy`, since deciding a lifecycle fate is a policy seam) decide each one's fate (`TransitionStatus`/`HardDelete`/`Skip`). `run(dryRun = true)` previews without writing; `collect()` marks only. Every run is recorded in a `CollectorRecordStore` for audit. **Consolidation — the dream loop.** `DreamLoopOrchestrator`/`DefaultDreamLoopOrchestrator` run `ConsolidationPass`es (`operations.consolidation`) as a threshold-gated cycle: `SessionConsolidationPass`, `AbstractionPass` (drives supersession), `ContradictionResolutionPass` (drives contradiction), `DecaySweepPass` (drives stale). The orchestrator owns all writes and locks per `contextId` so cycles for one context never overlap. @@ -55,7 +55,7 @@ These map onto the lifecycle in [proposition-lifecycle](../docs/design/propositi | `proposition.store` | `InMemoryPropositionRepository`, `JsonFilePropositionRepository`, `InMemoryDecayManager` | | `proposition.gate` | Admission gates applied to pipeline output before persistence: `ExtractionGate`, `ExtractionGatePipeline`, `StandardGates`, `ObservableGate`, `GateDecision`, `GatedPropositionResult`, `EvidenceFloorGate` | | `pipeline` | `PropositionPipeline` (two-stage extract→resolve), `PropositionResults`, `PersistablePropositions`; execution strategies: `ExtractionExecutionStrategy` + `SerialExtractionStrategy`/`ParallelExtractionStrategy`/`BatchedExtractionStrategy` | -| `spi` | Policy extension points: `TrustScorer`, `AuthorityResolver`/`AuthorityTier`, `AuthorityWeightedTrustScorer`, `ConflictDetector`, `ConflictType`, `StatusTransitionPolicy`, and their shipped defaults | +| `spi` | Policy extension points: `TrustScorer`, `AuthorityResolver`/`AuthorityTier`, `AuthorityWeightedTrustScorer`, `ConflictDetector`, `ConflictType`, `StatusTransitionPolicy`, the mark-and-sweep `SweepPolicy`/`StatusTransitionSweepPolicy` with its `SweepAction`/`PropositionMark`/`MarkReason` vocabulary, and their shipped defaults | | `common` | `SourceAnalysisContext`, `EntityResolver`, `Relations`, `ContentHasher`, `KnowledgeType`, events, `SchemaAdherence`, validation rules | | `common.filter` | `MentionFilter`, `SchemaValidatedMentionFilter`, `ObservableMentionFilter`, context-aware filters | | `common.resolver` | `EscalatingEntityResolver`, `InMemoryEntityResolver`, `KnownEntityResolver`, `ChainedEntityResolver`, `LlmCandidateBakeoff`, `ContextCompressor` | @@ -64,7 +64,7 @@ These map onto the lifecycle in [proposition-lifecycle](../docs/design/propositi | `incremental` | `AbstractIncrementalAnalyzer`, `ChunkHistoryStore`, `InMemoryChunkHistoryStore`, `ConversationSource`, `IncrementalSource` | | `incremental.proposition` | `PropositionIncrementalAnalyzer` | | `projection.graph` | `GraphProjector`, `RelationBasedGraphProjector`, `LlmGraphProjector`, `GraphProjectionService`, `ProjectionPolicy` | -| `projection.memory` | `MemoryProjector`/`DefaultMemoryProjector`, `MemoryRetriever`, `MemoryConsolidator`; **maintenance & dream loop**: `MemoryMaintenanceOrchestrator`/`DefaultMemoryMaintenanceOrchestrator`, `DreamLoopOrchestrator`/`DefaultDreamLoopOrchestrator`, `DreamLoopReport`; **mark-and-sweep collector**: `CollectorRunner`/`DefaultCollectorRunner`, `CollectorStrategy` (`DecayCollectorStrategy`, `DuplicateCollectorStrategy`), `SweepPolicy`/`StatusTransitionSweepPolicy`, `SweepAction`, `PropositionMark`, `MarkReason`, `CollectorRunResult` | +| `projection.memory` | `MemoryProjector`/`DefaultMemoryProjector`, `MemoryRetriever`, `MemoryConsolidator`; **maintenance & dream loop**: `MemoryMaintenanceOrchestrator`/`DefaultMemoryMaintenanceOrchestrator`, `DreamLoopOrchestrator`/`DefaultDreamLoopOrchestrator`, `DreamLoopReport`; **mark-and-sweep collector**: `CollectorRunner`/`DefaultCollectorRunner`, `CollectorStrategy` (`DecayCollectorStrategy`, `DuplicateCollectorStrategy`), `CollectorRunResult` — the `SweepPolicy` and the `SweepAction`/`PropositionMark`/`MarkReason` vocabulary live in `spi` | | `projection.prolog` | `PrologProjector`, `PrologEngine`, Prolog type wrappers | | `projection.lineage` | `ProjectionRecordStore`, `InMemoryProjectionRecordStore`, `Reconciler`, `ProjectionRecord`, `ProjectionRun`; **collector audit trail**: `CollectorRecordStore`/`InMemoryCollectorRecordStore`, `CollectorRecord`, `CollectorRun`, `CollectorOutcome` | | `projection.grounding` | `GroundingResolver`, `GroundingWiringService` | diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPass.kt index 5bbb2bef..62e5294d 100644 --- a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPass.kt +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPass.kt @@ -18,6 +18,8 @@ package com.embabel.dice.operations.consolidation import com.embabel.agent.core.ContextId import com.embabel.dice.projection.memory.CollectorRunner import com.embabel.dice.proposition.Proposition +import com.embabel.dice.spi.SweepAction +import com.embabel.dice.spi.SweepPolicy /** * The decay pass in a consolidation cycle: a thin wrapper that delegates entirely to an injected @@ -32,7 +34,7 @@ import com.embabel.dice.proposition.Proposition * The behavior depends on the [SweepPolicy] the runner was built with. The default policy retires * propositions softly to `STALE` and never hard-deletes, so a [DecaySweepPass] over a default * runner is non-destructive. For hard-delete, build the runner with a policy that returns - * `SweepAction.HardDelete` for stale marks — this pass surfaces those deletes in its summary + * [SweepAction.HardDelete] for stale marks — this pass surfaces those deletes in its summary * but never puts ids in [ConsolidationPassResult.Changed.propositionsToDelete]. * * ## Dual-threshold hysteresis diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRecord.kt b/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRecord.kt index 1f35c2c8..f5d1c3fc 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRecord.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/lineage/CollectorRecord.kt @@ -15,8 +15,8 @@ */ package com.embabel.dice.projection.lineage -import com.embabel.dice.projection.memory.MarkReason import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.spi.MarkReason import java.time.Instant /** diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunResult.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunResult.kt index 34aa6ea0..4d87fa97 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunResult.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunResult.kt @@ -15,6 +15,7 @@ */ package com.embabel.dice.projection.memory +import com.embabel.dice.spi.PropositionMark import java.time.Instant /** diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunner.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunner.kt index 105cf67b..314689a5 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunner.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorRunner.kt @@ -19,6 +19,8 @@ import com.embabel.agent.core.ContextId import com.embabel.dice.common.DiceEventListener import com.embabel.dice.projection.lineage.CollectorRecordStore import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.spi.StatusTransitionSweepPolicy +import com.embabel.dice.spi.SweepPolicy /** * Runs the mark-and-sweep collector for a context: selects ACTIVE propositions, hands them to diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorStrategy.kt index e746a00a..5c85a743 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorStrategy.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/CollectorStrategy.kt @@ -18,6 +18,8 @@ package com.embabel.dice.projection.memory import com.embabel.agent.core.ContextId import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.spi.PropositionMark +import com.embabel.dice.spi.SweepPolicy /** * The "mark" half of the mark-and-sweep collector: looks at candidate propositions and diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt index ff9907cc..02b8ec97 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt @@ -18,6 +18,8 @@ package com.embabel.dice.projection.memory import com.embabel.agent.core.ContextId import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.spi.MarkReason +import com.embabel.dice.spi.PropositionMark import org.slf4j.LoggerFactory /** diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt index dcc5f4cd..349e5cb6 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt @@ -26,6 +26,9 @@ import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionQuery import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.spi.PropositionMark +import com.embabel.dice.spi.SweepAction +import com.embabel.dice.spi.SweepPolicy import org.slf4j.LoggerFactory import java.time.Instant import java.util.UUID diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategy.kt index 2ccde537..bdf1fed7 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategy.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategy.kt @@ -20,6 +20,8 @@ import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionQuery import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.spi.MarkReason +import com.embabel.dice.spi.PropositionMark import org.slf4j.LoggerFactory /** diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/MarkReason.kt b/dice/src/main/kotlin/com/embabel/dice/spi/MarkReason.kt similarity index 97% rename from dice/src/main/kotlin/com/embabel/dice/projection/memory/MarkReason.kt rename to dice/src/main/kotlin/com/embabel/dice/spi/MarkReason.kt index b3c0806e..0ef3dfa1 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/MarkReason.kt +++ b/dice/src/main/kotlin/com/embabel/dice/spi/MarkReason.kt @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.embabel.dice.projection.memory +package com.embabel.dice.spi /** * Why a proposition was marked for collection by a [CollectorStrategy]. diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/PropositionMark.kt b/dice/src/main/kotlin/com/embabel/dice/spi/PropositionMark.kt similarity index 97% rename from dice/src/main/kotlin/com/embabel/dice/projection/memory/PropositionMark.kt rename to dice/src/main/kotlin/com/embabel/dice/spi/PropositionMark.kt index 38ee289f..ddf30842 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/PropositionMark.kt +++ b/dice/src/main/kotlin/com/embabel/dice/spi/PropositionMark.kt @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.embabel.dice.projection.memory +package com.embabel.dice.spi import java.time.Instant diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepAction.kt b/dice/src/main/kotlin/com/embabel/dice/spi/SweepAction.kt similarity index 97% rename from dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepAction.kt rename to dice/src/main/kotlin/com/embabel/dice/spi/SweepAction.kt index 5481ee2c..97391776 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepAction.kt +++ b/dice/src/main/kotlin/com/embabel/dice/spi/SweepAction.kt @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.embabel.dice.projection.memory +package com.embabel.dice.spi import com.embabel.dice.proposition.PropositionStatus diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepPolicy.kt b/dice/src/main/kotlin/com/embabel/dice/spi/SweepPolicy.kt similarity index 98% rename from dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepPolicy.kt rename to dice/src/main/kotlin/com/embabel/dice/spi/SweepPolicy.kt index 0383befe..9c64a26b 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/SweepPolicy.kt +++ b/dice/src/main/kotlin/com/embabel/dice/spi/SweepPolicy.kt @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.embabel.dice.projection.memory +package com.embabel.dice.spi import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionStatus diff --git a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPassTest.kt b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPassTest.kt index 3f9dc8d6..49c95b96 100644 --- a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPassTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/DecaySweepPassTest.kt @@ -18,13 +18,13 @@ package com.embabel.dice.operations.consolidation import com.embabel.agent.core.ContextId import com.embabel.dice.projection.memory.CollectorRunner import com.embabel.dice.projection.memory.DecayCollectorStrategy -import com.embabel.dice.projection.memory.PropositionMark -import com.embabel.dice.projection.memory.SweepAction -import com.embabel.dice.projection.memory.SweepPolicy import com.embabel.dice.proposition.EntityMention import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.spi.PropositionMark +import com.embabel.dice.spi.SweepAction +import com.embabel.dice.spi.SweepPolicy import io.mockk.every import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertEquals diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/lineage/CollectorRecordTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/lineage/CollectorRecordTest.kt index 723c5510..0a47d109 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/lineage/CollectorRecordTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/lineage/CollectorRecordTest.kt @@ -15,8 +15,8 @@ */ package com.embabel.dice.projection.lineage -import com.embabel.dice.projection.memory.MarkReason import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.spi.MarkReason import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertThrows diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStoreTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStoreTest.kt index 5ecce1ab..3355203d 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStoreTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStoreTest.kt @@ -15,7 +15,7 @@ */ package com.embabel.dice.projection.lineage -import com.embabel.dice.projection.memory.MarkReason +import com.embabel.dice.spi.MarkReason import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorRunnerBuilderTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorRunnerBuilderTest.kt index f2dc6b30..122d74e9 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorRunnerBuilderTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorRunnerBuilderTest.kt @@ -16,6 +16,7 @@ package com.embabel.dice.projection.memory import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.spi.StatusTransitionSweepPolicy import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorStrategyTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorStrategyTest.kt index ef2fbcd5..c5de8942 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorStrategyTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorStrategyTest.kt @@ -18,6 +18,8 @@ package com.embabel.dice.projection.memory import com.embabel.agent.core.ContextId import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.spi.MarkReason +import com.embabel.dice.spi.PropositionMark import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertThrows diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategyTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategyTest.kt index 5e229209..30eb8343 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategyTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategyTest.kt @@ -18,6 +18,7 @@ package com.embabel.dice.projection.memory import com.embabel.agent.core.ContextId import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.spi.MarkReason import io.mockk.mockk import io.mockk.verify import org.junit.jupiter.api.Assertions.assertEquals diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt index 0d147258..719b4117 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt @@ -24,6 +24,11 @@ import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionQuery import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.spi.MarkReason +import com.embabel.dice.spi.PropositionMark +import com.embabel.dice.spi.StatusTransitionSweepPolicy +import com.embabel.dice.spi.SweepAction +import com.embabel.dice.spi.SweepPolicy import io.mockk.every import io.mockk.mockk import io.mockk.slot diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategyTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategyTest.kt index b4f38f60..3f6043c3 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategyTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DuplicateCollectorStrategyTest.kt @@ -20,6 +20,7 @@ import com.embabel.agent.rag.service.Cluster import com.embabel.common.core.types.SimilarityResult import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.spi.MarkReason import io.mockk.every import io.mockk.mockk import io.mockk.verify diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/StatusTransitionSweepPolicyTest.kt b/dice/src/test/kotlin/com/embabel/dice/spi/StatusTransitionSweepPolicyTest.kt similarity index 98% rename from dice/src/test/kotlin/com/embabel/dice/projection/memory/StatusTransitionSweepPolicyTest.kt rename to dice/src/test/kotlin/com/embabel/dice/spi/StatusTransitionSweepPolicyTest.kt index 510619b4..349f60f6 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/StatusTransitionSweepPolicyTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/spi/StatusTransitionSweepPolicyTest.kt @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.embabel.dice.projection.memory +package com.embabel.dice.spi import com.embabel.agent.core.ContextId import com.embabel.dice.proposition.Proposition From c5c34b5abe31777e415c990880d5ec7b763e7432 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Tue, 23 Jun 2026 01:00:43 -0400 Subject: [PATCH 24/27] fix(consolidation): stop AbstractionPass from retiring sources it can't replace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sources were marked SUPERSEDED unconditionally. Two cases lost facts: when every candidate abstraction exceeded maxLevel (no replacement at all), and when a source was pinned (eviction-immune). Now skip a group whose abstractions are all over the cap, and never supersede a pinned source — keep it ACTIVE alongside the abstraction. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../consolidation/AbstractionPass.kt | 17 +++++-- .../consolidation/AbstractionPassTest.kt | 44 ++++++++++++++++++- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt index ced28cd4..4ec8b128 100644 --- a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt @@ -83,24 +83,33 @@ class AbstractionPass @JvmOverloads constructor( } val abstractions = abstractor.abstract(PropositionGroup.of(entityId, props), abstractionTargetCount) .filter { it.level <= maxLevel } + if (abstractions.isEmpty()) { + // Every candidate abstraction exceeded maxLevel. Don't retire the sources to + // SUPERSEDED with nothing to replace them — that would silently lose the facts. + // Skip the group and leave the sources ACTIVE. + skipped += props.size + continue + } toSave += abstractions // withStatus preserves the contentRevised decay anchor (only metadataRevised moves). - toSave += props.map { it.withStatus(PropositionStatus.SUPERSEDED) } + // Pinned sources are eviction-immune: keep them ACTIVE alongside the abstraction + // rather than retiring them to SUPERSEDED. + toSave += props.filter { !it.pinned }.map { it.withStatus(PropositionStatus.SUPERSEDED) } abstractedGroups++ } logger.debug( - "Abstraction over {} level-0 active proposition(s) for {}: {} group(s) abstracted, {} skipped (already covered), {} proposition(s) to save", + "Abstraction over {} level-0 active proposition(s) for {}: {} group(s) abstracted, {} skipped (already covered or over the level cap), {} proposition(s) to save", level0Active.size, contextId, abstractedGroups, skipped, toSave.size, ) if (toSave.isEmpty()) { - ConsolidationPassResult.NoOp(name, "no groups above threshold or all covered") + ConsolidationPassResult.NoOp(name, "no groups above threshold, all covered, or all abstractions over the level cap") } else { ConsolidationPassResult.Changed( passName = name, propositionsToSave = toSave, skipped = skipped, - summary = "abstracted $abstractedGroups groups, $skipped propositions skipped (already covered)", + summary = "abstracted $abstractedGroups groups, $skipped propositions skipped (already covered or over the level cap)", ) } } catch (e: Throwable) { diff --git a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/AbstractionPassTest.kt b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/AbstractionPassTest.kt index a7092657..d0827eb6 100644 --- a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/AbstractionPassTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/AbstractionPassTest.kt @@ -41,6 +41,7 @@ class AbstractionPassTest { status: PropositionStatus = PropositionStatus.ACTIVE, level: Int = 0, sourceIds: List = emptyList(), + pinned: Boolean = false, ): Proposition = Proposition( id = id, @@ -52,6 +53,7 @@ class AbstractionPassTest { status = status, level = level, sourceIds = sourceIds, + pinned = pinned, ) private fun group(entityId: String, count: Int): List = @@ -103,6 +105,25 @@ class AbstractionPassTest { } } + @Test + fun `a pinned source is kept ACTIVE, not superseded, when its group is abstracted`() { + val members = group("bob", 4) + proposition("pinned-1", "bob", pinned = true) + val abstraction = proposition("abs-1", "bob", level = 1, sourceIds = members.map { it.id }) + val abstractor = mockk() + every { abstractor.abstract(any(), any()) } returns listOf(abstraction) + + val changed = assertInstanceOf( + ConsolidationPassResult.Changed::class.java, + AbstractionPass(abstractor, abstractionThreshold = 5).run(contextId, members), + ) + + // The abstraction plus the 4 unpinned sources superseded; the pinned source is eviction- + // immune, so it is neither superseded nor otherwise touched (stays ACTIVE, absent from saves). + assertTrue(changed.propositionsToSave.contains(abstraction)) + assertEquals(4, changed.propositionsToSave.count { it.status == PropositionStatus.SUPERSEDED }) + assertTrue(changed.propositionsToSave.none { it.id == "pinned-1" }) + } + @Test fun `level-inflation guard skips a group already covered by an existing higher-level proposition`() { val members = group("bob", 5) @@ -146,18 +167,37 @@ class AbstractionPassTest { } @Test - fun `abstractions exceeding maxLevel are filtered out`() { + fun `when every abstraction exceeds maxLevel the sources are kept, not superseded`() { val members = group("bob", 5) val tooHigh = proposition("abs-high", "bob", level = 4, sourceIds = members.map { it.id }) val abstractor = mockk() every { abstractor.abstract(any(), any()) } returns listOf(tooHigh) + val result = AbstractionPass(abstractor, abstractionThreshold = 5, maxLevel = 3) + .run(contextId, members) + + // The over-cap abstraction is dropped, and crucially the sources are NOT retired to + // SUPERSEDED — there would be no surviving abstraction to replace them, so retiring them + // would silently lose the facts. The group is skipped and the pass is a NoOp. + assertInstanceOf(ConsolidationPassResult.NoOp::class.java, result) + } + + @Test + fun `a surviving abstraction still supersedes the sources even when an over-cap sibling is dropped`() { + val members = group("bob", 5) + val ok = proposition("abs-ok", "bob", level = 2, sourceIds = members.map { it.id }) + val tooHigh = proposition("abs-high", "bob", level = 4, sourceIds = members.map { it.id }) + val abstractor = mockk() + every { abstractor.abstract(any(), any()) } returns listOf(ok, tooHigh) + val result = AbstractionPass(abstractor, abstractionThreshold = 5, maxLevel = 3) .run(contextId, members) val changed = assertInstanceOf(ConsolidationPassResult.Changed::class.java, result) - // the over-cap abstraction is dropped; only the 5 superseded sources remain + // Over-cap sibling dropped, the within-cap abstraction kept, and the sources superseded + // because there IS now a replacement. assertTrue(changed.propositionsToSave.none { it.level > 3 }) + assertTrue(changed.propositionsToSave.contains(ok)) assertEquals(5, changed.propositionsToSave.count { it.status == PropositionStatus.SUPERSEDED }) } } From e36fa0c46a522c237dcbe01a6985d547182550b8 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Tue, 23 Jun 2026 01:00:43 -0400 Subject: [PATCH 25/27] test: fill remaining unit-test gaps Cover MarkReason.Custom (consumer key/description flowing through a mark), CollectorRecordStore.findRun (hit and miss), and the DreamLoopReport data-class contract (totals carry through, cycleCompleted defaults to construction time). Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../InMemoryCollectorRecordStoreTest.kt | 13 +++++ .../memory/CollectorStrategyTest.kt | 11 +++++ .../projection/memory/DreamLoopReportTest.kt | 47 +++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopReportTest.kt diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStoreTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStoreTest.kt index 3355203d..257db7e0 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStoreTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/lineage/InMemoryCollectorRecordStoreTest.kt @@ -17,8 +17,10 @@ package com.embabel.dice.projection.lineage import com.embabel.dice.spi.MarkReason import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import java.time.Instant class InMemoryCollectorRecordStoreTest { @@ -86,4 +88,15 @@ class InMemoryCollectorRecordStoreTest { assertTrue(store.findByProposition("nope").isEmpty()) } + + @Test + fun `findRun returns the recorded run header and null for an unknown id`() { + val store = InMemoryCollectorRecordStore() + store.recordRun(CollectorRun(runId = "r1", startedAt = Instant.now(), dryRun = true)) + + val found = store.findRun("r1") + assertEquals("r1", found?.runId) + assertTrue(found?.dryRun == true) + assertNull(store.findRun("nope")) + } } diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorStrategyTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorStrategyTest.kt index c5de8942..531a5da8 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorStrategyTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/CollectorStrategyTest.kt @@ -94,4 +94,15 @@ class CollectorStrategyTest { assertEquals(2, marks.size) assertTrue(marks.all { it.reason == MarkReason.Stale }) } + + @Test + fun `a Custom mark reason carries its consumer key and description`() { + val reason = MarkReason.Custom(key = "schema-version", description = "extracted under an old schema") + assertEquals("schema-version", reason.key) + assertEquals("extracted under an old schema", reason.description) + + // A custom reason flows through a mark like the shipped reasons do. + val mark = PropositionMark(propositionId = "p1", reason = reason, strategyName = "custom") + assertEquals("schema-version", mark.reason.key) + } } diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopReportTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopReportTest.kt new file mode 100644 index 00000000..6e99a31e --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopReportTest.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.projection.memory + +import com.embabel.agent.core.ContextId +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +/** Data-class contract for the cycle report: totals carry through and cycleCompleted defaults to now. */ +class DreamLoopReportTest { + + @Test + fun `report carries its totals and defaults cycleCompleted to construction time`() { + val started = Instant.now().minusSeconds(5) + val report = DreamLoopReport( + contextId = ContextId("c"), + cycleStarted = started, + passResults = emptyList(), + totalExamined = 10, + totalTransitioned = 3, + totalNewPropositions = 2, + triggered = true, + ) + + assertEquals(10, report.totalExamined) + assertEquals(3, report.totalTransitioned) + assertEquals(2, report.totalNewPropositions) + assertTrue(report.triggered) + // cycleCompleted is defaulted at construction, so it's at or after the cycle start. + assertTrue(!report.cycleCompleted.isBefore(started)) + } +} From 81032524e930c5bfa99afbffa59622d45621dc47 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Tue, 23 Jun 2026 01:00:57 -0400 Subject: [PATCH 26/27] feat: pinned eviction immunity across the lifecycle Make Proposition.pinned a real 'must retain' guarantee end to end: the decay collector strategy never marks a pinned proposition, contradiction resolution never auto-retires one, and the reviser keeps a pinned original intact on contradiction (leaving the clash for explicit resolution) instead of demoting it. Adds a pinned filter to PropositionQuery and pin/unpin/findPinned helpers on PropositionStore. Decay-status and sweep-policy already exempted pinned; this closes the remaining eviction paths. Budget-priority injection of pinned propositions remains with the token-budget work (#7). Closes #9 Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../ContradictionResolutionPass.kt | 6 +- .../memory/DecayCollectorStrategy.kt | 3 +- .../dice/proposition/PropositionQuery.kt | 6 ++ .../dice/proposition/PropositionStore.kt | 25 +++++++ .../revision/LlmPropositionReviser.kt | 33 ++++++--- .../revision/PropositionReviser.kt | 6 +- .../ContradictionResolutionPassTest.kt | 22 ++++++ .../memory/DecayCollectorStrategyTest.kt | 15 ++++ .../memory/DefaultCollectorRunnerTest.kt | 7 +- .../embabel/dice/proposition/PinningTest.kt | 71 +++++++++++++++++++ 10 files changed, 177 insertions(+), 17 deletions(-) create mode 100644 dice/src/test/kotlin/com/embabel/dice/proposition/PinningTest.kt diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt index bbf9d758..6bdb4545 100644 --- a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt @@ -74,8 +74,10 @@ class ContradictionResolutionPass @JvmOverloads constructor( val weaker = if (p.effectiveConfidence() < c.proposition.effectiveConfidence()) p else c.proposition - if (weaker.status == PropositionStatus.ACTIVE) { - // withStatus preserves the contentRevised decay anchor. + if (weaker.status == PropositionStatus.ACTIVE && !weaker.pinned) { + // withStatus preserves the contentRevised decay anchor. A pinned + // proposition is conflict-protected — never auto-retired — so a + // contradiction against it is left for explicit resolution. toSave += weaker.withStatus(PropositionStatus.CONTRADICTED) } } diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt index 02b8ec97..f0138c7a 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategy.kt @@ -49,7 +49,8 @@ class DecayCollectorStrategy @JvmOverloads constructor( contextId: ContextId, ): List { val marks = candidates - .filter { it.effectiveConfidence(retireDecayK) < retireBelow } + // Pinned propositions are decay-immune: never mark them, even below the threshold. + .filter { !it.pinned && it.effectiveConfidence(retireDecayK) < retireBelow } .map { PropositionMark(propositionId = it.id, reason = MarkReason.Stale, strategyName = STRATEGY_NAME) } logger.debug("DecayCollectorStrategy: marked {} of {} candidates as stale (threshold={})", marks.size, candidates.size, retireBelow) return marks diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionQuery.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionQuery.kt index e7fb8763..59fc63ed 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionQuery.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionQuery.kt @@ -77,6 +77,9 @@ data class PropositionQuery( // Trust filter (reads the cached trust score from proposition metadata) val minTrustScore: Double? = null, + // Pinned filter: true = only pinned, false = only unpinned, null = either + val pinned: Boolean? = null, + // Ordering and limits val orderBy: OrderBy = OrderBy.NONE, val limit: Int? = null, @@ -200,6 +203,9 @@ data class PropositionQuery( fun withMinTrustScore(threshold: Double): PropositionQuery = copy(minTrustScore = threshold) + /** Restrict to pinned (`true`) or unpinned (`false`) propositions. */ + fun withPinned(pinned: Boolean = true): PropositionQuery = copy(pinned = pinned) + fun withOrderBy(orderBy: OrderBy): PropositionQuery = copy(orderBy = orderBy) fun orderedByEffectiveConfidence(): PropositionQuery = diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionStore.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionStore.kt index 9bfcafb7..6a0ad712 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionStore.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionStore.kt @@ -79,6 +79,7 @@ internal fun PropositionQuery.matchesFilters( } if (minImportance != null && prop.importance < minImportance) return false if (minReinforceCount != null && prop.reinforceCount < minReinforceCount) return false + if (pinned != null && prop.pinned != pinned) return false return prop.passesMinTrust(minTrustScore) } @@ -170,6 +171,30 @@ interface PropositionStore { */ fun count(): Int + // ======================================================================== + // Pinning — "must retain" propositions that resist eviction and decay + // ======================================================================== + + /** + * Pin a proposition so it resists reclamation: pinned propositions are skipped by the decay + * collector and the sweep policy, are decay-exempt in the default status policy, and are not + * auto-retired by contradiction resolution. + * + * @return the saved pinned proposition, or `null` if no proposition has [id]. + */ + fun pin(id: String): Proposition? = findById(id)?.let { save(it.withPinned(true)) } + + /** + * Clear a proposition's pin, returning it to normal reclamation. + * + * @return the saved proposition, or `null` if no proposition has [id]. + */ + fun unpin(id: String): Proposition? = findById(id)?.let { save(it.withPinned(false)) } + + /** All pinned propositions in [contextId]. */ + fun findPinned(contextId: ContextId): List = + query(PropositionQuery.forContextId(contextId).withPinned(true)) + // ======================================================================== // Composable query - consolidates filtering, ordering, limiting // ======================================================================== diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/revision/LlmPropositionReviser.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/revision/LlmPropositionReviser.kt index f8e0fe39..3997a65a 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/revision/LlmPropositionReviser.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/revision/LlmPropositionReviser.kt @@ -513,17 +513,28 @@ data class LlmPropositionReviser( contradictory != null -> { val original = repository.findById(contradictory.proposition.id) ?: contradictory.proposition - val reducedConfidence = (original.confidence * 0.3).coerceAtLeast(0.05) - // Accelerate decay so contradicted propositions fade faster - val acceleratedDecay = (original.decay + 0.15).coerceAtMost(1.0) - val contradicted = original - .withConfidence(reducedConfidence) - .withStatus(PropositionStatus.CONTRADICTED) - .copy(decay = acceleratedDecay, lastAccessed = Instant.now()) - logger.debug( - "Contradicted: {} (conf: {}, decay: {}) vs new: {}", - original.text, reducedConfidence, acceleratedDecay, newProposition.text - ) + // A pinned original is conflict-protected: a contradicting fact never auto-demotes + // it. Keep it untouched and store the new proposition alongside, leaving the + // conflict for explicit resolution. + val contradicted = if (original.pinned) { + logger.debug( + "Contradiction with pinned original kept intact: {} vs new: {}", + original.text, newProposition.text, + ) + original + } else { + val reducedConfidence = (original.confidence * 0.3).coerceAtLeast(0.05) + // Accelerate decay so contradicted propositions fade faster + val acceleratedDecay = (original.decay + 0.15).coerceAtMost(1.0) + logger.debug( + "Contradicted: {} (conf: {}, decay: {}) vs new: {}", + original.text, reducedConfidence, acceleratedDecay, newProposition.text + ) + original + .withConfidence(reducedConfidence) + .withStatus(PropositionStatus.CONTRADICTED) + .copy(decay = acceleratedDecay, lastAccessed = Instant.now()) + } // Contradicted is deliberately NOT trust-scored: it keeps its existing // confidence-reduction trajectory. A wired detector refines only its classification. val detector = conflictDetector diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/revision/PropositionReviser.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/revision/PropositionReviser.kt index 963232bf..d85499c3 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/revision/PropositionReviser.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/revision/PropositionReviser.kt @@ -67,7 +67,11 @@ sealed class RevisionResult { val revised: Proposition, ) : RevisionResult() - /** Contradicted an existing proposition (both stored, old with reduced confidence) */ + /** + * Contradicted an existing proposition. Both are stored; the [original] normally has its + * confidence reduced and status set to CONTRADICTED — except a pinned original is kept intact + * (conflict-protected), leaving the clash for explicit resolution. + */ data class Contradicted( val original: Proposition, val new: Proposition, diff --git a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt index 39f940f7..26573820 100644 --- a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt @@ -41,6 +41,7 @@ class ContradictionResolutionPassTest { confidence: Double, status: PropositionStatus = PropositionStatus.ACTIVE, contentRevised: Instant = Instant.now(), + pinned: Boolean = false, ): Proposition = Proposition( id = id, @@ -51,6 +52,7 @@ class ContradictionResolutionPassTest { decay = 0.1, status = status, contentRevised = contentRevised, + pinned = pinned, ) @Test @@ -121,6 +123,26 @@ class ContradictionResolutionPassTest { assertEquals(PropositionStatus.CONTRADICTED, transitioned.status) } + @Test + fun `a pinned proposition is never retired by contradiction resolution`() { + // The pinned one has the LOWER confidence, so it would normally lose — but pinned + // propositions are conflict-protected and must survive. + val pinned = proposition("a", "bob", 0.2, pinned = true) + val strong = proposition("b", "bob", 0.9) + val reviser = mockk() + every { reviser.classify(pinned, listOf(strong)) } returns listOf( + ClassifiedProposition(strong, PropositionRelation.CONTRADICTORY, 0.5), + ) + every { reviser.classify(strong, listOf(pinned)) } returns listOf( + ClassifiedProposition(pinned, PropositionRelation.CONTRADICTORY, 0.5), + ) + + val result = ContradictionResolutionPass(reviser).run(contextId, listOf(pinned, strong)) + + // Nothing retired: the only loser would have been the pinned proposition. + assertInstanceOf(ConsolidationPassResult.NoOp::class.java, result) + } + @Test fun `no contradictions yields NoOp`() { val a = proposition("a", "bob", 0.9) diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategyTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategyTest.kt index 30eb8343..8abd5c80 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategyTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DecayCollectorStrategyTest.kt @@ -34,6 +34,7 @@ class DecayCollectorStrategyTest { text: String = "p", confidence: Double, decay: Double = 0.0, + pinned: Boolean = false, ): Proposition = Proposition( contextId = contextId, @@ -41,8 +42,22 @@ class DecayCollectorStrategyTest { mentions = emptyList(), confidence = confidence, decay = decay, + pinned = pinned, ) + @Test + fun `a pinned proposition below the threshold is decay-immune and not marked`() { + val pinnedLow = proposition("pinned-low", confidence = 0.1, pinned = true) + val unpinnedLow = proposition("low", confidence = 0.1) + val repository = mockk(relaxed = true) + + val marks = DecayCollectorStrategy(retireBelow = 0.5) + .mark(listOf(pinnedLow, unpinnedLow), repository, contextId) + + assertEquals(1, marks.size) + assertEquals(unpinnedLow.id, marks[0].propositionId) + } + @Test fun `marks candidates below the threshold with Stale and strategyName decay`() { val low = proposition("low", confidence = 0.2) diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt index 719b4117..d84f231a 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt @@ -159,14 +159,17 @@ class DefaultCollectorRunnerTest { } @Test - fun `live run skips a pinned proposition`() { + fun `a live run leaves a pinned proposition untouched (decay-immune)`() { val pinned = decayedProp("pinned old fact", pinned = true) every { repository.query(any()) } returns listOf(pinned) val result = runner().run(contextId, dryRun = false) + // The decay strategy never marks a pinned proposition, so it isn't applied, skipped, or even + // marked — fully decay-immune — and nothing is written or emitted. assertTrue(result.applied.isEmpty()) - assertTrue(result.skipped.isNotEmpty()) + assertTrue(result.skipped.isEmpty()) + assertTrue(result.marks.isEmpty()) verify(exactly = 0) { repository.save(any()) } verify(exactly = 0) { repository.delete(any()) } assertTrue(listener.eventsOfType().isEmpty()) diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/PinningTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/PinningTest.kt new file mode 100644 index 00000000..de912bc0 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/PinningTest.kt @@ -0,0 +1,71 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.store.InMemoryPropositionRepository +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +/** Store-level pinning: the pin/unpin/findPinned helpers and the `pinned` query filter. */ +class PinningTest { + + private val ctx = ContextId("ctx") + + private fun prop(id: String, pinned: Boolean = false): Proposition = + Proposition(id = id, contextId = ctx, text = id, mentions = emptyList(), confidence = 0.8, pinned = pinned) + + @Test + fun `pin and unpin toggle the flag and persist it`() { + val repo = InMemoryPropositionRepository() + repo.save(prop("p1")) + + val pinned = repo.pin("p1") + assertNotNull(pinned) + assertTrue(pinned!!.pinned) + assertTrue(repo.findById("p1")!!.pinned) + + val unpinned = repo.unpin("p1") + assertFalse(unpinned!!.pinned) + assertFalse(repo.findById("p1")!!.pinned) + } + + @Test + fun `pin returns null for a missing id`() { + assertNull(InMemoryPropositionRepository().pin("nope")) + } + + @Test + fun `findPinned returns only the pinned propositions in the context`() { + val repo = InMemoryPropositionRepository() + repo.saveAll(listOf(prop("p1", pinned = true), prop("p2"), prop("p3", pinned = true))) + + assertEquals(setOf("p1", "p3"), repo.findPinned(ctx).map { it.id }.toSet()) + } + + @Test + fun `the pinned query filter selects pinned or unpinned`() { + val repo = InMemoryPropositionRepository() + repo.saveAll(listOf(prop("p1", pinned = true), prop("p2"))) + + assertEquals(setOf("p1"), repo.query(PropositionQuery.forContextId(ctx).withPinned(true)).map { it.id }.toSet()) + assertEquals(setOf("p2"), repo.query(PropositionQuery.forContextId(ctx).withPinned(false)).map { it.id }.toSet()) + } +} From 1f1e8d23a7f8fee7dce845dfb1f06eaa27277b7e Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Wed, 24 Jun 2026 02:05:30 -0400 Subject: [PATCH 27/27] fix: address PR #48 review findings Tighten the "skip / protect / keep" branches the review flagged for dropping information without a durable trace: - Dream-loop trigger: stop advancing the change-volume baseline on a skipped call so growth accumulates since the last cycle, not the last call (a steadily-growing context was never consolidating). - Reconcile cross-pass saves by id with a defined status precedence, so a proposition retired by two passes in one cycle has a deterministic final status instead of one decided by saveAll ordering. - Count deletes as transitions only when allowHardDelete actually applies them. - Dedupe abstraction's superseded sources by id (pass and maintenance twin) so a source mentioning two entities is retired once. - Collector: persist the partial audit trail on a mid-run failure, and reorder to persist -> record -> emit so a throwing listener can't leave a transition recorded-but-not, persisted-but-unrecorded. - Surface a PropositionRoutedToReview signal when a contradiction lands on a pinned fact, on both the ingest and consolidation paths, instead of swallowing it silently. - Route the REST file-extract endpoint through process() so one bad chunk yields partial results instead of failing the whole upload and the execution strategies are actually exercised. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../consolidation/AbstractionPass.kt | 11 ++- .../ContradictionResolutionPass.kt | 34 ++++++++-- .../dice/pipeline/PropositionPipeline.kt | 13 ++++ .../memory/DefaultCollectorRunner.kt | 33 +++++++-- .../memory/DefaultDreamLoopOrchestrator.kt | 46 +++++++++++-- .../DefaultMemoryMaintenanceOrchestrator.kt | 19 +++--- .../web/rest/PropositionPipelineController.kt | 22 +++--- .../consolidation/AbstractionPassTest.kt | 32 +++++++++ .../ContradictionResolutionPassTest.kt | 24 +++++++ .../memory/DefaultCollectorRunnerTest.kt | 44 ++++++++++++ .../memory/DreamLoopOrchestratorTest.kt | 67 +++++++++++++++++++ .../rest/PropositionPipelineControllerTest.kt | 65 ++++++++++++++++++ 12 files changed, 371 insertions(+), 39 deletions(-) diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt index 4ec8b128..89821425 100644 --- a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/AbstractionPass.kt @@ -98,16 +98,21 @@ class AbstractionPass @JvmOverloads constructor( abstractedGroups++ } + // A source that mentions two qualifying entities lands in both groups and would be + // marked SUPERSEDED once per group. Collapse to one save per id so a single source is + // never written — or counted — twice (the abstractions all carry fresh ids and pass + // through untouched). Mirrors the dedup ContradictionResolutionPass already does. + val deduped = toSave.distinctBy { it.id } logger.debug( "Abstraction over {} level-0 active proposition(s) for {}: {} group(s) abstracted, {} skipped (already covered or over the level cap), {} proposition(s) to save", - level0Active.size, contextId, abstractedGroups, skipped, toSave.size, + level0Active.size, contextId, abstractedGroups, skipped, deduped.size, ) - if (toSave.isEmpty()) { + if (deduped.isEmpty()) { ConsolidationPassResult.NoOp(name, "no groups above threshold, all covered, or all abstractions over the level cap") } else { ConsolidationPassResult.Changed( passName = name, - propositionsToSave = toSave, + propositionsToSave = deduped, skipped = skipped, summary = "abstracted $abstractedGroups groups, $skipped propositions skipped (already covered or over the level cap)", ) diff --git a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt index 6bdb4545..7b2f69fa 100644 --- a/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt +++ b/dice/src/main/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPass.kt @@ -16,6 +16,8 @@ package com.embabel.dice.operations.consolidation import com.embabel.agent.core.ContextId +import com.embabel.dice.common.DiceEventListener +import com.embabel.dice.common.PropositionRoutedToReview import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionStatus import com.embabel.dice.proposition.revision.PropositionRelation @@ -35,6 +37,13 @@ import org.slf4j.LoggerFactory * Classification uses [PropositionReviser.classify] — NOT a contraster — because resolving a * contradiction is a lifecycle transition, not an articulation of differences. * + * ## Pinned losers are surfaced, not silently dropped + * + * A pinned proposition is conflict-protected and is never auto-retired. Rather than skip it in + * silence — which would let a wrong pin accumulate contradicting evidence invisibly — the pass + * emits a [PropositionRoutedToReview] for it via [eventListener], so a human (or a later pass) is + * prompted to resolve the conflict or unpin it. + * * ## Idempotency * * Only ACTIVE propositions are compared and only an ACTIVE loser is transitioned, so once a conflict @@ -43,9 +52,12 @@ import org.slf4j.LoggerFactory * are pruned to those sharing at least one resolved entity, bounding `classify()` fan-out. * * @property reviser The delegate that classifies the relation between propositions. + * @property eventListener Notified when a contradiction lands on a pinned proposition; defaults to + * a no-op listener so callers that don't care about the signal need not wire one. */ class ContradictionResolutionPass @JvmOverloads constructor( private val reviser: PropositionReviser, + private val eventListener: DiceEventListener = DiceEventListener.DEV_NULL, ) : ConsolidationPass { override val name: String = "contradiction-resolution" @@ -74,11 +86,23 @@ class ContradictionResolutionPass @JvmOverloads constructor( val weaker = if (p.effectiveConfidence() < c.proposition.effectiveConfidence()) p else c.proposition - if (weaker.status == PropositionStatus.ACTIVE && !weaker.pinned) { - // withStatus preserves the contentRevised decay anchor. A pinned - // proposition is conflict-protected — never auto-retired — so a - // contradiction against it is left for explicit resolution. - toSave += weaker.withStatus(PropositionStatus.CONTRADICTED) + if (weaker.status == PropositionStatus.ACTIVE) { + if (weaker.pinned) { + // A pinned proposition is conflict-protected — never auto-retired — + // so we don't transition it. But surface the contradiction as a + // review signal instead of dropping it silently, so the pin can be + // explicitly resolved or unpinned rather than quietly going wrong. + val stronger = if (weaker.id == p.id) c.proposition else p + eventListener.onEvent( + PropositionRoutedToReview( + weaker, + reason = "pinned proposition contradicted by '${stronger.id}'; resolve the conflict or unpin", + ) + ) + } else { + // withStatus preserves the contentRevised decay anchor. + toSave += weaker.withStatus(PropositionStatus.CONTRADICTED) + } } } } diff --git a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt index ed82b8a5..0cf21d11 100644 --- a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt +++ b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt @@ -24,6 +24,7 @@ import com.embabel.dice.common.PropositionDiscovered import com.embabel.dice.common.PropositionGeneralized import com.embabel.dice.common.PropositionMerged import com.embabel.dice.common.PropositionReinforced +import com.embabel.dice.common.PropositionRoutedToReview import com.embabel.dice.common.SafeDiceEventListener import com.embabel.dice.common.SourceAnalysisContext import com.embabel.dice.common.SuggestedEntities @@ -257,6 +258,18 @@ class PropositionPipeline private constructor( ) } eventListener.onEvent(event) + // A contradiction against a pinned original leaves the pin intact (conflict + // protection). Surface a review signal alongside the contradiction event so the + // contested pin can be explicitly resolved or unpinned, rather than silently + // accumulating contradicting evidence. + if (revision is RevisionResult.Contradicted && revision.original.pinned) { + eventListener.onEvent( + PropositionRoutedToReview( + revision.original, + reason = "pinned proposition contradicted by newer evidence; resolve the conflict or unpin", + ) + ) + } } results } else { diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt index 349e5cb6..4e976d56 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunner.kt @@ -102,6 +102,7 @@ class DefaultCollectorRunner( val hardDeleted = mutableListOf() val records = mutableListOf() + try { for ((propositionId, propMarks) in marksByProposition) { val proposition = candidatesById[propositionId] ?: continue when (val action = policy.decide(proposition, propMarks)) { @@ -112,9 +113,15 @@ class DefaultCollectorRunner( // newStatus still carries what WOULD happen. records += records(propMarks, runId, CollectorOutcome.MARKED, proposition.status, action.newStatus) } else { - applyTransition(proposition, action.newStatus, propMarks) + // Order matters: persist, then buffer the audit record, then emit. The + // record lands between the durable write and the (inline, possibly throwing) + // listener call, so a transition can never be persisted without its record — + // even if a listener blows up before the run finishes. + val previousStatus = proposition.status + val saved = repository.save(proposition.withStatus(action.newStatus)) + records += records(propMarks, runId, CollectorOutcome.TRANSITIONED, previousStatus, action.newStatus) applied.addAll(propMarks) - records += records(propMarks, runId, CollectorOutcome.TRANSITIONED, proposition.status, action.newStatus) + emitStatusChanged(saved, previousStatus, action.newStatus, propMarks) } } @@ -136,6 +143,15 @@ class DefaultCollectorRunner( } } } + } catch (e: Throwable) { + // A mutation failed partway through. Earlier iterations have already saved/deleted and + // emitted their events, and their records are buffered in `records` — persist that + // partial trail before rethrowing so a HARD_DELETED proposition is never lost without an + // audit record. The failing proposition added no record (the throw preempts it). + logger.warn("Collector run {} aborted mid-run; persisting the {} record(s) gathered so far", runId, records.size, e) + persistRun(runId, startedAt, Instant.now(), dryRun, records) + throw e + } // Compute the finish instant once and thread it into both the persisted run header and // the returned result, so the audit object and the summary agree on the finish time. @@ -172,14 +188,17 @@ class DefaultCollectorRunner( return candidatesById to marks } - /** Persist the swept transition, then emit the lifecycle event (persist-then-emit). */ - private fun applyTransition( - proposition: Proposition, + /** + * Emit the lifecycle event for a transition that has already been persisted and recorded. + * Kept as the final step so the durable write and its audit record are both in place before any + * (inline, possibly throwing) listener runs. + */ + private fun emitStatusChanged( + saved: Proposition, + previousStatus: PropositionStatus, newStatus: PropositionStatus, propMarks: List, ) { - val previousStatus = proposition.status - val saved = repository.save(proposition.withStatus(newStatus)) // Multiple strategies may mark the same proposition; combine their distinct reason keys // (sorted for run-to-run determinism) so the emitted event is order-independent and never // silently drops a reason. `reason` stays a nullable String for backward compatibility. diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt index fb8dba68..921e7bf8 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultDreamLoopOrchestrator.kt @@ -92,8 +92,11 @@ data class DefaultDreamLoopOrchestrator( val currentActive = activeSnapshot(contextId).size val delta = currentActive - (lastActiveCount[contextId] ?: 0) if (delta < changeVolumeThreshold) { - // Not enough has changed to be worth a cycle; record the count and skip. - lastActiveCount[contextId] = currentActive + // Not enough has accumulated *since the last cycle* to be worth one. Crucially we leave + // the baseline untouched here: advancing it on a skip would measure the delta since the + // last call instead, so steady incremental growth (the normal scheduled case) would + // never accumulate to the threshold and a cycle would never fire. The baseline only + // moves after a real cycle (step 5 below). logger.debug( "Skipping consolidation for {}: active-count delta {} < threshold {}", contextId, delta, changeVolumeThreshold, @@ -114,10 +117,14 @@ data class DefaultDreamLoopOrchestrator( // (2) Run each pass in registration order, isolating failures into Failed results. val passResults = passes.map { pass -> runPass(pass, contextId, snapshot) } - // (3) Aggregate saves into a single write. + // (3) Aggregate saves into a single write, reconciling any proposition that more than one + // pass wants to re-save this cycle. Without this, the same id returned by two passes (e.g. + // SUPERSEDED from abstraction and CONTRADICTED from contradiction-resolution) would be + // written twice and the persisted status would depend on saveAll ordering. We collapse to + // one write per id with a deterministic winner — see [reconcileSaves]. val snapshotIds = snapshot.mapTo(HashSet()) { it.id } val changed = passResults.filterIsInstance() - val toSave: List = changed.flatMap { it.propositionsToSave } + val toSave: List = reconcileSaves(changed.flatMap { it.propositionsToSave }) if (toSave.isNotEmpty()) { repository.saveAll(toSave) } @@ -137,8 +144,12 @@ data class DefaultDreamLoopOrchestrator( // by their own pass, so they are counted via externallyApplied rather than the save list. val newPropositions = toSave.count { it.id !in snapshotIds } val savedTransitions = toSave.count { it.id in snapshotIds } - val transitioned = savedTransitions + - changed.sumOf { it.propositionsToDelete.size + it.externallyApplied } + // Deletes only count as transitions when they are actually applied. With allowHardDelete + // off (the default) delete-intents are ignored, so counting them would over-report work + // that never happened. + val appliedDeletes = if (allowHardDelete) changed.sumOf { it.propositionsToDelete.size } else 0 + val transitioned = savedTransitions + appliedDeletes + + changed.sumOf { it.externallyApplied } val report = DreamLoopReport( contextId = contextId, cycleStarted = cycleStarted, @@ -189,6 +200,29 @@ data class DefaultDreamLoopOrchestrator( ConsolidationPassResult.Failed(pass.name, e) } + /** + * Collapse the cycle's combined save list to one write per proposition id. + * + * When two passes each hand back a copy of the same proposition with a different target status, + * the order they happen to appear in the flat-mapped list must not decide what gets persisted. + * We pick a deterministic winner by status strength: a contradiction (the belief is now wrong) + * outranks a supersession (still true, just rolled up into an abstraction), which outranks a + * decay-to-STALE, and any retirement outranks leaving it ACTIVE. Freshly created propositions + * (new ids, e.g. abstractions) never collide, so they pass through untouched. + */ + private fun reconcileSaves(toSave: List): List = + toSave + .groupBy { it.id } + .map { (_, copies) -> copies.maxByOrNull { statusStrength(it.status) }!! } + + private fun statusStrength(status: PropositionStatus): Int = when (status) { + PropositionStatus.CONTRADICTED -> 4 + PropositionStatus.SUPERSEDED -> 3 + PropositionStatus.STALE -> 2 + PropositionStatus.PROMOTED -> 1 + PropositionStatus.ACTIVE -> 0 + } + private fun activeSnapshot(contextId: ContextId): List = repository.query( PropositionQuery.forContextId(contextId).withStatus(PropositionStatus.ACTIVE) diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt index 0f86cc32..e18c20cd 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/DefaultMemoryMaintenanceOrchestrator.kt @@ -153,11 +153,11 @@ data class DefaultMemoryMaintenanceOrchestrator( ) // Recover the per-entity grouping so abstraction runs per group, in the legacy granularity. - // Each group is abstracted in isolation through its OWN AbstractionPass invocation: a group's - // abstractions are exactly those returned by that group's abstract() call (no cross-group - // sourceId matching, so a source mentioning two entities is never saved or counted twice), - // persisted per group in the legacy call shape (one saveAll for the group's abstractions, one - // for its superseded sources). + // Each group is abstracted in isolation through its OWN AbstractionPass invocation, persisted + // per group in the legacy call shape (one saveAll for the group's abstractions, one for its + // superseded sources). A source that mentions two qualifying entities lands in both groups, so + // it would be superseded once per group — we dedup the superseded sources by id below so such + // a source is never saved or counted twice. val entityGroups = active .flatMap { prop -> prop.mentions.mapNotNull { it.resolvedId }.map { entityId -> entityId to prop } } .groupBy({ it.first }, { it.second }) @@ -166,6 +166,9 @@ data class DefaultMemoryMaintenanceOrchestrator( val allAbstractions = mutableListOf() val allSuperseded = mutableListOf() + // Ids already retired this call, so a source shared across two entity groups is superseded + // (saved and counted) exactly once. + val supersededIds = HashSet() // maxLevel = Int.MAX_VALUE preserves the pre-refactor "persist every abstraction the // abstractor returns" contract — maintain() applies no level ceiling. Any cap belongs only on @@ -187,9 +190,9 @@ data class DefaultMemoryMaintenanceOrchestrator( } val groupAbstractions = outcome.propositionsToSave.filter { it.level > 0 } - val groupSuperseded = outcome.propositionsToSave.filter { - it.status == PropositionStatus.SUPERSEDED - } + val groupSuperseded = outcome.propositionsToSave + .filter { it.status == PropositionStatus.SUPERSEDED } + .filter { supersededIds.add(it.id) } repository.saveAll(groupAbstractions) allAbstractions.addAll(groupAbstractions) diff --git a/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt b/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt index 8bb59494..a129e153 100644 --- a/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt +++ b/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt @@ -150,17 +150,19 @@ class PropositionPipelineController( )) } - // Process each chunk through the pipeline + // Process all chunks through the pipeline's batch entry point. process() isolates a failing + // chunk into a typed Failed result (so one bad chunk yields partial results instead of 500ing + // the whole upload), shares entity identity across chunks, and is the only path that honors + // the configured extraction execution strategy (Serial/Parallel/Batched). processChunk(), by + // contrast, propagates failures and runs one chunk in isolation. val context = buildContext(contextId, parseKnownEntities(knownEntitiesJson), schemaName) - val chunkResults = chunks.map { chunk -> - val result = propositionPipeline.processChunk(chunk, context) + val processResult = propositionPipeline.process(chunks, context) + val chunkResults = processResult.chunkResults - // Persist revised propositions too (e.g. a CONTRADICTED original), not just the new ones. - result.propositionsToPersist().forEach { proposition -> - propositionRepository.save(proposition) - } - - result + // Persist what revision says to keep across the whole batch — revised originals (e.g. a + // CONTRADICTED original) as well as the new propositions, not just the new ones. + processResult.propositionsToPersist().forEach { proposition -> + propositionRepository.save(proposition) } // Aggregate results @@ -168,7 +170,7 @@ class PropositionPipelineController( val allRevisions = chunkResults.flatMap { it.revisionResults } // Failed chunks contribute zero resolutions to the aggregate response: // entityResolutions is a Success-only field, and Failed returns empty propositions/ - // revisionResults via the interface, so the :143-144 flatMaps already exclude them. + // revisionResults via the interface, so the flat-maps above already exclude them. val allResolutions = chunkResults .filterIsInstance() .flatMap { it.entityResolutions.resolutions } diff --git a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/AbstractionPassTest.kt b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/AbstractionPassTest.kt index d0827eb6..9b0c6d22 100644 --- a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/AbstractionPassTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/AbstractionPassTest.kt @@ -124,6 +124,38 @@ class AbstractionPassTest { assertTrue(changed.propositionsToSave.none { it.id == "pinned-1" }) } + @Test + fun `a source mentioning two qualifying entities is superseded only once`() { + // `shared` mentions entities "a" and "b", so it lands in both group a = {a-1..a-4, shared} + // and group b = {b-1..b-4, shared}, each at the threshold of 5. Without dedup it would be + // marked SUPERSEDED once per group; the pass must collapse it to a single save. + val shared = Proposition( + id = "shared", + contextId = contextId, + text = "prop-shared", + mentions = listOf( + EntityMention(span = "e", type = "Person", resolvedId = "a"), + EntityMention(span = "e", type = "Person", resolvedId = "b"), + ), + confidence = 0.8, + ) + val members = group("a", 4) + group("b", 4) + shared + val abstractor = mockk() + // A distinct abstraction per group, so the dedup collapses only the shared source — never an + // abstraction. + every { abstractor.abstract(any(), any()) } answers { + val grp = firstArg() + listOf(proposition("abs-${grp.label}", grp.label, level = 1, sourceIds = grp.propositions.map { it.id })) + } + + val changed = assertInstanceOf( + ConsolidationPassResult.Changed::class.java, + AbstractionPass(abstractor, abstractionThreshold = 5).run(contextId, members), + ) + + assertEquals(1, changed.propositionsToSave.count { it.id == "shared" }) + } + @Test fun `level-inflation guard skips a group already covered by an existing higher-level proposition`() { val members = group("bob", 5) diff --git a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt index 26573820..c166e234 100644 --- a/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/operations/consolidation/ContradictionResolutionPassTest.kt @@ -16,6 +16,8 @@ package com.embabel.dice.operations.consolidation import com.embabel.agent.core.ContextId +import com.embabel.dice.common.PropositionRoutedToReview +import com.embabel.dice.common.RecordingDiceEventListener import com.embabel.dice.proposition.EntityMention import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionStatus @@ -143,6 +145,28 @@ class ContradictionResolutionPassTest { assertInstanceOf(ConsolidationPassResult.NoOp::class.java, result) } + @Test + fun `a contradicted pinned proposition is surfaced for review, not silently dropped`() { + // Same setup as above (the pinned one would lose), but now assert the contradiction is not + // swallowed: a review signal must be emitted so the contested pin can be resolved or unpinned. + val pinned = proposition("a", "bob", 0.2, pinned = true) + val strong = proposition("b", "bob", 0.9) + val reviser = mockk() + every { reviser.classify(pinned, listOf(strong)) } returns listOf( + ClassifiedProposition(strong, PropositionRelation.CONTRADICTORY, 0.5), + ) + every { reviser.classify(strong, listOf(pinned)) } returns listOf( + ClassifiedProposition(pinned, PropositionRelation.CONTRADICTORY, 0.5), + ) + val listener = RecordingDiceEventListener() + + ContradictionResolutionPass(reviser, listener).run(contextId, listOf(pinned, strong)) + + val routed = listener.eventsOfType() + assertEquals(1, routed.size) + assertEquals("a", routed.single().proposition.id) + } + @Test fun `no contradictions yields NoOp`() { val a = proposition("a", "bob", 0.9) diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt index d84f231a..c2aaf9ec 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DefaultCollectorRunnerTest.kt @@ -16,6 +16,7 @@ package com.embabel.dice.projection.memory import com.embabel.agent.core.ContextId +import com.embabel.dice.common.DiceEventListener import com.embabel.dice.common.PropositionStatusChanged import com.embabel.dice.common.RecordingDiceEventListener import com.embabel.dice.projection.lineage.CollectorRecordStore @@ -158,6 +159,49 @@ class DefaultCollectorRunnerTest { assertTrue(recordStore.findByRun(result.runId).isNotEmpty()) } + @Test + fun `a proposition transitioned before a mid-run failure still has an audit record`() { + // Two decayed candidates. The first transitions cleanly; the second's save throws partway + // through the run. The first proposition is already mutated and its event emitted, so its + // audit record must be persisted despite the abort — otherwise a transition (or a hard + // delete) is unrecoverable. + val first = decayedProp("first old fact") + val second = decayedProp("second old fact") + every { repository.query(any()) } returns listOf(first, second) + var saves = 0 + every { repository.save(any()) } answers { + saves++ + if (saves == 1) firstArg() else throw RuntimeException("storage failure") + } + + runCatching { runner().run(contextId, dryRun = false) } + + // The first really transitioned: exactly one event, and a durable audit record for it. + assertEquals(1, listener.eventsOfType().size) + assertTrue(recordStore.findByProposition(first.id).isNotEmpty()) + } + + @Test + fun `a transition is recorded even when a listener throws after the persist`() { + // persist-then-emit means a listener runs after the durable write. If it throws, the + // proposition is already transitioned — its audit record must still be captured rather than + // lost to the unhandled listener failure. + val decayed = decayedProp("old fact") + every { repository.query(any()) } returns listOf(decayed) + every { repository.save(any()) } answers { firstArg() } + val throwingListener = DiceEventListener { throw RuntimeException("listener boom") } + val runner = CollectorRunner + .withRepository(repository) + .withStrategy(DecayCollectorStrategy(retireBelow = 0.3)) + .withRecordStore(recordStore) + .withEventListener(throwingListener) + .build() + + runCatching { runner.run(contextId, dryRun = false) } + + assertTrue(recordStore.findByProposition(decayed.id).isNotEmpty()) + } + @Test fun `a live run leaves a pinned proposition untouched (decay-immune)`() { val pinned = decayedProp("pinned old fact", pinned = true) diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestratorTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestratorTest.kt index 18f7a3a0..75acbb1c 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestratorTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/DreamLoopOrchestratorTest.kt @@ -20,6 +20,7 @@ import com.embabel.dice.operations.consolidation.ConsolidationPass import com.embabel.dice.operations.consolidation.ConsolidationPassResult import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.proposition.PropositionStatus import io.mockk.every import io.mockk.mockk import io.mockk.slot @@ -293,6 +294,72 @@ class DreamLoopOrchestratorTest { assertEquals(1, maxObserved.get(), "the per-context lock must serialize cycles for one context") } + @Test + fun `consolidate accumulates growth across skipped calls until the threshold is reached`() { + // Growth arrives in increments of 3, each below the default threshold of 10. The trigger + // measures the delta since the last CYCLE, so it must keep accumulating across skipped calls + // rather than resetting the baseline on every call — otherwise a steadily-growing context + // never consolidates. + var activeCount = 0 + every { repository.query(any()) } answers { List(activeCount) { proposition("p$it") } } + val pass = FakePass("noop") { ConsolidationPassResult.NoOp("noop") } + val orchestrator = DefaultDreamLoopOrchestrator.withRepository(repository).withPass(pass) + + repeat(5) { activeCount += 3; orchestrator.consolidate(contextId) } // 3, 6, 9, 12, 15 + + assertTrue(pass.runCount >= 1) { + "consolidation never fired despite cumulative growth to $activeCount (threshold 10)" + } + } + + @Test + fun `aggregated saves reconcile to a single status per proposition id`() { + // Two passes return the same proposition id with different target statuses in one cycle. The + // persisted result must not depend on saveAll ordering: it is collapsed to one write whose + // status is the stronger of the two (CONTRADICTED outranks SUPERSEDED). + val original = proposition("shared") + every { repository.query(any()) } returns listOf(original) + val captured = slot>() + every { repository.saveAll(capture(captured)) } returns Unit + + DefaultDreamLoopOrchestrator.withRepository(repository) + .withPass(FakePass("abstraction") { + ConsolidationPassResult.Changed( + "abstraction", + propositionsToSave = listOf(original.withStatus(PropositionStatus.SUPERSEDED)), + ) + }) + .withPass(FakePass("contradiction") { + ConsolidationPassResult.Changed( + "contradiction", + propositionsToSave = listOf(original.withStatus(PropositionStatus.CONTRADICTED)), + ) + }) + .consolidateNow(contextId) + + assertEquals(1, captured.captured.count { it.id == original.id }) + assertEquals( + PropositionStatus.CONTRADICTED, + captured.captured.single { it.id == original.id }.status, + ) + } + + @Test + fun `transition count excludes deletes that were never applied`() { + // allowHardDelete is off by default, so delete-intents are ignored. The report must not + // count those phantom deletes as transitions. + every { repository.query(any()) } returns emptyList() + + val report = DefaultDreamLoopOrchestrator.withRepository(repository) + .withPass(FakePass("decay") { + ConsolidationPassResult.Changed("decay", propositionsToDelete = listOf("doomed-1", "doomed-2")) + }) + .consolidateNow(contextId) + + verify(exactly = 0) { repository.delete(any()) } + assertEquals(0, report.totalTransitioned) + } + @Test fun `cycles for different contexts are not serialized against each other`() { every { repository.query(any()) } returns listOf(proposition("p1")) diff --git a/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt b/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt index 979f8409..1df89616 100644 --- a/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt @@ -23,8 +23,13 @@ import com.embabel.dice.common.Resolutions import com.embabel.dice.common.SuggestedEntity import com.embabel.dice.common.resolver.AlwaysCreateEntityResolver import com.embabel.dice.common.support.InMemorySchemaRegistry +import com.embabel.agent.rag.ingestion.ContentChunker +import com.embabel.agent.rag.ingestion.HierarchicalContentReader +import com.embabel.agent.rag.model.Chunk +import com.embabel.agent.rag.model.NavigableDocument import com.embabel.dice.pipeline.ChunkPropositionResult import com.embabel.dice.pipeline.PropositionPipeline +import com.embabel.dice.pipeline.PropositionResults import com.embabel.dice.proposition.* import com.embabel.dice.proposition.revision.RevisionResult import com.fasterxml.jackson.annotation.JsonClassDescription @@ -33,11 +38,14 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.kotlin.KotlinModule import io.mockk.every import io.mockk.mockk +import io.mockk.verify import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.http.MediaType import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter +import org.springframework.mock.web.MockMultipartFile import org.springframework.test.web.servlet.MockMvc +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status @@ -255,4 +263,61 @@ class PropositionPipelineControllerTest { .content("""{"text": " "}""") ).andExpect(status().isBadRequest) } + + @Test + fun `POST extract file isolates a failing chunk and returns partial results`() { + // A two-chunk upload where the second chunk fails extraction. The endpoint must route through + // the batch process() — which isolates a failure into a typed Failed result and honors the + // execution strategy — rather than the failure-propagating processChunk() loop that would + // 500 the whole upload on one bad chunk. + val contextId = "test-context" + val reader = mockk() + val chunker = mockk() + val document = mockk(relaxed = true) + every { reader.parseContent(any(), any()) } returns document + every { chunker.chunk(any()) } returns listOf( + Chunk.create(text = "good chunk", parentId = "doc"), + Chunk.create(text = "bad chunk", parentId = "doc"), + ) + + val proposition = Proposition( + contextId = ContextId(contextId), + text = "User loves Brahms", + mentions = emptyList(), + confidence = 0.9, + ) + val success = ChunkPropositionResult.Success( + chunkId = "chunk-ok", + suggestedPropositions = SuggestedPropositions(chunkId = "chunk-ok", propositions = emptyList()), + entityResolutions = Resolutions(chunkIds = setOf("chunk-ok"), resolutions = emptyList()), + propositions = listOf(proposition), + revisionResults = emptyList(), + ) + val failed = ChunkPropositionResult.Failed("chunk-bad", "extraction blew up") + every { propositionPipeline.process(any(), any()) } returns + PropositionResults(chunkResults = listOf(success, failed), allPropositions = listOf(proposition)) + + val fileController = PropositionPipelineController( + propositionPipeline = propositionPipeline, + propositionRepository = propositionRepository, + entityResolver = entityResolver, + schemaRegistry = schemaRegistry, + contentReader = reader, + contentChunker = chunker, + ) + val mvc = MockMvcBuilders.standaloneSetup(fileController) + .setMessageConverters(MappingJackson2HttpMessageConverter(objectMapper)) + .build() + + mvc.perform( + multipart("/api/v1/contexts/$contextId/extract/file") + .file(MockMultipartFile("file", "doc.txt", "text/plain", "content".toByteArray())) + ) + .andExpect(status().isOk) + .andExpect(jsonPath("$.chunksProcessed").value(2)) + .andExpect(jsonPath("$.entities.failed[0]").value("chunk-bad")) + + verify(exactly = 1) { propositionPipeline.process(any(), any()) } + verify(exactly = 0) { propositionPipeline.processChunk(any(), any()) } + } }