diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/CollectorTraceRowMappers.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/CollectorTraceRowMappers.kt index 6070dc41..21beada9 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/CollectorTraceRowMappers.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/CollectorTraceRowMappers.kt @@ -117,6 +117,7 @@ object CollectorDecisionRowMapper { "foldedGrounding" to retired.foldedGrounding, "foldedProvenanceRefs" to retired.foldedProvenanceRefs, "foldedSourceIds" to retired.foldedSourceIds, + "foldedProvenanceEvidenceKeys" to retired.foldedProvenanceEvidenceKeys, ) fun fromRow(row: Map<*, *>, retired: List): CollectorDecision = CollectorDecision( @@ -133,6 +134,7 @@ object CollectorDecisionRowMapper { foldedGrounding = row.stringList("foldedGrounding"), foldedProvenanceRefs = row.stringList("foldedProvenanceRefs"), foldedSourceIds = row.stringList("foldedSourceIds"), + foldedProvenanceEvidenceKeys = row.stringList("foldedProvenanceEvidenceKeys"), ) } diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineCollectorTraceStore.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineCollectorTraceStore.kt index 65cdb44a..115a3f79 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineCollectorTraceStore.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineCollectorTraceStore.kt @@ -44,7 +44,8 @@ import java.time.Instant * - `(:CollectorDecision {id, runId, contextId, componentId, survivorId, action, createdAt})` * with `id = "runId|componentId"`, plus one child * `(:CollectorRetired {id, runId, contextId, propositionId, priorStatus, foldedGrounding, - * foldedProvenanceRefs, foldedSourceIds})-[:RETIRED_IN]->(:CollectorDecision)` per retired + * foldedProvenanceRefs, foldedSourceIds, foldedProvenanceEvidenceKeys}) + * -[:RETIRED_IN]->(:CollectorDecision)` per retired * proposition, so a reversal has everything a merging sweep folded onto the survivor. * * Every write is a single `UNWIND $rows AS r ...` round trip (see [com.embabel.dice.storage.CollectorTraceRowMappers] for @@ -227,7 +228,8 @@ class DrivineCollectorTraceStore( retired: [r IN retiredNodes WHERE r IS NOT NULL | { propositionId: r.propositionId, priorStatus: r.priorStatus, foldedGrounding: r.foldedGrounding, foldedProvenanceRefs: r.foldedProvenanceRefs, - foldedSourceIds: r.foldedSourceIds + foldedSourceIds: r.foldedSourceIds, + foldedProvenanceEvidenceKeys: r.foldedProvenanceEvidenceKeys }] } AS row """.trimIndent(), diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivinePropositionRepository.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivinePropositionRepository.kt index 1d35e7b0..ad98150d 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivinePropositionRepository.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivinePropositionRepository.kt @@ -30,7 +30,13 @@ import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.proposition.GraphQueryCapable import com.embabel.dice.proposition.PropositionStatus import com.embabel.dice.proposition.PropositionStoreType +import com.embabel.dice.provenance.ConnectorRef +import com.embabel.dice.provenance.ContentAddressedLocator +import com.embabel.dice.provenance.FileLocator import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.SourceLocator +import com.embabel.dice.provenance.SourceRevisionRef +import com.embabel.dice.provenance.UriLocator import com.embabel.dice.query.graph.GraphNeighborhood import com.embabel.dice.query.graph.GraphPath import com.embabel.dice.query.graph.PropositionLineage @@ -43,10 +49,46 @@ import org.drivine.query.QuerySpecification import org.drivine.query.dsl.* import org.slf4j.LoggerFactory import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.annotation.Propagation import org.springframework.transaction.annotation.Transactional +import org.springframework.transaction.support.TransactionSynchronizationManager import org.springframework.transaction.support.TransactionTemplate import java.time.Instant +internal object SourceProvenanceQueryStatements { + // The existing uniqueness index is composite `(contextId, text)`. Every repository proposition + // has non-null text, so the empty-string lower bound preserves membership while letting Neo4j + // seek the tenant prefix before expanding provenance relationships. + val bySourceKey = """ + MATCH (p:Proposition {contextId: ${'$'}contextId}) + USING INDEX SEEK p:Proposition(contextId, text) + WHERE p.text >= '' AND EXISTS { + MATCH (p)-[:DERIVED_FROM]->(:Source {key: ${'$'}sourceKey}) + } + RETURN p.id AS id + """.trimIndent() + + val bySourceRevision = """ + MATCH (p:Proposition {contextId: ${'$'}contextId}) + USING INDEX SEEK p:Proposition(contextId, text) + WHERE p.text >= '' AND EXISTS { + MATCH (p)-[r:DERIVED_FROM]->(:Source {key: ${'$'}sourceKey}) + WHERE r.sourceRevision = ${'$'}sourceRevision + } + RETURN p.id AS id + """.trimIndent() + + val revisionlessBySourceKey = """ + MATCH (p:Proposition {contextId: ${'$'}contextId}) + USING INDEX SEEK p:Proposition(contextId, text) + WHERE p.text >= '' AND EXISTS { + MATCH (p)-[r:DERIVED_FROM]->(:Source {key: ${'$'}sourceKey}) + WHERE r.sourceRevision IS NULL + } + RETURN p.id AS id + """.trimIndent() +} + /** * Graph-backed [PropositionRepository] over Drivine / Neo4j. * @@ -77,7 +119,7 @@ class DrivinePropositionRepository( private val logger = LoggerFactory.getLogger(DrivinePropositionRepository::class.java) - /** Runs the dedup find-then-insert as one programmatic transaction (see [save]). */ + /** Owns each standalone dedup attempt or recovery transaction. */ private val txTemplate = TransactionTemplate(transactionManager) /** @@ -104,8 +146,35 @@ class DrivinePropositionRepository( * sibling. If a foreign sibling turns out to be ACTIVE too (only reachable without the * `(contextId, text)` constraint), that's a live duplicate this method won't silently create by * redirecting, but won't collapse either — it logs a WARN naming both ids for the dedup sweep. + * + * An active caller transaction remains authoritative: [save] joins it, and a caller rollback + * rolls back the save. Without one, the programmatic transactions own the attempted insert and, + * after a cross-instance uniqueness rollback, the independent recovery. This preserves the + * repository's historical transaction boundary while still making standalone concurrent + * extraction retry-safe. */ + @Transactional(propagation = Propagation.SUPPORTS) override fun save(proposition: Proposition): Proposition { + if (TransactionSynchronizationManager.isActualTransactionActive()) { + return saveInCallerTransaction(proposition) + } + return saveInOwnedTransaction(proposition) + } + + private fun saveInCallerTransaction(proposition: Proposition): Proposition { + val text = proposition.text + if (text.isBlank()) { + return doPersist(proposition) + } + val contextId = proposition.contextId.value + return synchronized(lockFor(contextId, text)) { + // A database uniqueness race aborts the caller's transaction. Recovery cannot safely + // run until that transaction has rolled back, so preserve atomicity and propagate it. + findOrPersist(proposition, contextId, text) + } + } + + private fun saveInOwnedTransaction(proposition: Proposition): Proposition { val text = proposition.text if (text.isBlank()) { return txTemplate.execute { doPersist(proposition) }!! @@ -119,7 +188,7 @@ class DrivinePropositionRepository( // Cross-instance race: another writer inserted the same (contextId, text) and the DB // (contextId, text) uniqueness constraint rejected ours. The dupe now exists — reuse it. logger.debug("Dedup constraint hit for context {} — reusing existing: '{}'", contextId, text) - txTemplate.execute { findDuplicateId(contextId, text, proposition.id)?.let(::findById) } ?: throw e + txTemplate.execute { recoverDuplicate(proposition, contextId, text) } ?: throw e } } } @@ -138,7 +207,7 @@ class DrivinePropositionRepository( "Dedup: proposition already present as {} in context {} — reusing: '{}'", existingId, contextId, text, ) - existing + mergeDeduplicatedProvenance(existing, proposition.provenanceEntries) } else { // An update lands on its own node even when a same-text sibling exists. If that sibling is // also ACTIVE, this mints a second live copy of the same fact (only reachable without the @@ -157,6 +226,44 @@ class DrivinePropositionRepository( } } + /** + * Re-find the winning proposition after a cross-instance uniqueness race and union the losing + * writer's evidence into it. The raw relationship MERGE makes retries harmless. + */ + private fun recoverDuplicate(proposition: Proposition, contextId: String, text: String): Proposition? { + val winnerId = findDuplicateId(contextId, text, proposition.id) ?: return null + val winner = findById(winnerId) ?: return null + return mergeDeduplicatedProvenance(winner, proposition.provenanceEntries) + } + + /** + * Union evidence from a deduplicated insert into its winner. A genuinely new entry is a + * metadata change and must advance `lastTouched`; an exact replay remains a no-op. + */ + private fun mergeDeduplicatedProvenance( + winner: Proposition, + incomingEntries: List, + ): Proposition { + ensureCompatibleSources(winner.id, incomingEntries) + val knownEntries = winner.provenanceEntries.toHashSet() + val novelEntries = incomingEntries.filterNot(knownEntries::contains) + if (novelEntries.isEmpty()) return winner + val revisedWinner = winner.withProvenanceEntries(novelEntries) + doPersist(revisedWinner) + return findById(winner.id) ?: revisedWinner + } + + /** + * Storage identity equality deliberately uses canonical source keys, so validate the structural + * locator identity before an exact replay can take the no-op path. + */ + private fun ensureCompatibleSources(propositionId: String, entries: List) { + entries.map(PropositionGraphMapper::toDerivedFrom).forEach { edge -> + val entryKey = requireNotNull(edge.entryKey) { "Provenance entryKey must be computed before persistence" } + ensureCompatibleSource(edge.source, provenanceParameters(propositionId, entryKey, edge)) + } + } + /** * Best-effort detection of a Neo4j uniqueness-constraint violation anywhere in the cause chain. * Matches on message substrings, since which form (error code vs. prose) shows up in @@ -178,24 +285,18 @@ class DrivinePropositionRepository( /** * Persist node, mentions, and (append-only) provenance. * - * Two writes with deliberately different cascades — Drivine applies one cascade per `save`: + * Two writes with deliberately different ownership: * - **Node + mentions** via the lean [PropositionView] with `DELETE_ORPHAN`: authoritative, so a * changed mention set is reconciled and stale Mention nodes are cleaned. Provenance is *not* in * this view, so existing `DERIVED_FROM` edges are left intact. - * - **Provenance** via [PropositionWithProvenanceView] with `PRESERVE`: additive — edges are merged, - * never deleted, and idempotent by the shared `:Source` key. So the all-in-one save never drops - * evidence it didn't load (the lean query/findAll paths, the decay sweep). Authoritative - * replacement/removal is the job of [setProvenance] / [clearProvenance]. + * - **Provenance** via raw Cypher keyed by the full evidence tuple: additive, idempotent, and able + * to preserve parallel revisions that relationship-fragment mapping otherwise collapses. + * Authoritative replacement/removal is the job of [setProvenance] / [clearProvenance]. */ private fun doPersist(proposition: Proposition): Proposition { val embedding = embeddingFor(proposition) graphObjectManager.save(PropositionGraphMapper.toView(proposition, embedding), CascadeType.DELETE_ORPHAN) - if (proposition.provenanceEntries.isNotEmpty()) { - graphObjectManager.save( - PropositionGraphMapper.toProvenanceView(proposition, embedding), - CascadeType.PRESERVE, - ) - } + appendProvenance(proposition.id, proposition.provenanceEntries) return proposition } @@ -203,20 +304,150 @@ class DrivinePropositionRepository( proposition.text.takeIf { it.isNotBlank() }?.let { embeddingService.embed(it).toList() } /** - * Authoritative provenance replace (unlike the append-only [save]): save the provenance view with - * `DELETE_ORPHAN`, so `DERIVED_FROM` edges — and any thereby-orphaned `:Source` nodes — not in - * [entries] are removed. [clearProvenance] funnels here with an empty list. + * Authoritative provenance replace (unlike the append-only [save]). Desired evidence is upserted + * first, omitted edges are deleted by their storage identity, and only globally unreferenced + * Source nodes are pruned. [clearProvenance] funnels here with an empty list. */ @Transactional override fun setProvenance(propositionId: String, entries: List): Proposition? { val updated = (findById(propositionId) ?: return null).withProvenance(entries) - graphObjectManager.save( - PropositionGraphMapper.toProvenanceView(updated, embeddingFor(updated)), - CascadeType.DELETE_ORPHAN, - ) + graphObjectManager.save(PropositionGraphMapper.toView(updated, embeddingFor(updated)), CascadeType.DELETE_ORPHAN) + replaceProvenance(propositionId, entries) return updated } + /** + * Append evidence without routing relationship identity through Drivine. `entryKey` is computed + * by the shared graph mapper and is always non-null before it reaches MERGE. + */ + private fun appendProvenance(propositionId: String, entries: List) { + val derived = entries.map(PropositionGraphMapper::toDerivedFrom) + derived.forEach { edge -> + val entryKey = requireNotNull(edge.entryKey) { "Provenance entryKey must be computed before persistence" } + val params = provenanceParameters(propositionId, entryKey, edge) + ensureCompatibleSource(edge.source, params) + if (edge.sourceRevision == null && adoptExactLegacyEdge(params)) return@forEach + persistenceManager.execute( + QuerySpecification.withStatement( + """ + MATCH (p:Proposition {id: ${'$'}propositionId}) + MATCH (s:Source {key: ${'$'}sourceKey}) + MERGE (p)-[r:DERIVED_FROM {entryKey: ${'$'}entryKey}]->(s) + SET r.sourceRevision = ${'$'}sourceRevision, + r.chunkId = ${'$'}chunkId, + r.startOffset = ${'$'}startOffset, + r.endOffset = ${'$'}endOffset, + r.contentHash = ${'$'}contentHash + """.trimIndent() + ).bind(params), + ) + } + } + + @Suppress("UNCHECKED_CAST") + private fun ensureCompatibleSource(source: SourceNode, params: Map) { + val stored = (persistenceManager.query( + QuerySpecification.withStatement( + """ + MERGE (s:Source {key: ${'$'}sourceKey}) + ON CREATE SET s.kind = ${'$'}sourceKind, + s.uri = ${'$'}sourceUri, + s.path = ${'$'}sourcePath, + s.contentHash = ${'$'}sourceContentHash, + s.connectorId = ${'$'}connectorId, + s.externalId = ${'$'}externalId + SET s.display = ${'$'}sourceDisplay + RETURN { + kind: s.kind, + uri: s.uri, + path: s.path, + contentHash: s.contentHash, + connectorId: s.connectorId, + externalId: s.externalId + } AS row + """.trimIndent() + ).bind(params) + ) as List>).single() + val compatible = stored["kind"] == source.kind && + stored["uri"] == source.uri && + stored["path"] == source.path && + stored["contentHash"] == source.contentHash && + stored["connectorId"] == source.connectorId && + stored["externalId"] == source.externalId + require(compatible) { + "Source key collision for '${source.key}': stored source identity differs from incoming locator" + } + } + + /** + * A pre-revision edge may be adopted only by an exactly equal revisionless entry. Revisioned + * evidence deliberately cannot claim legacy state. + */ + private fun adoptExactLegacyEdge(params: Map): Boolean { + val adopted = persistenceManager.maybeGetOne( + QuerySpecification.withStatement( + """ + MATCH (p:Proposition {id: ${'$'}propositionId})-[r:DERIVED_FROM]->(s:Source {key: ${'$'}sourceKey}) + WHERE r.entryKey IS NULL + AND r.sourceRevision IS NULL + AND ((r.chunkId IS NULL AND ${'$'}chunkId IS NULL) OR r.chunkId = ${'$'}chunkId) + AND ((r.startOffset IS NULL AND ${'$'}startOffset IS NULL) OR r.startOffset = ${'$'}startOffset) + AND ((r.endOffset IS NULL AND ${'$'}endOffset IS NULL) OR r.endOffset = ${'$'}endOffset) + AND ((r.contentHash IS NULL AND ${'$'}contentHash IS NULL) OR r.contentHash = ${'$'}contentHash) + WITH r LIMIT 1 + SET r.entryKey = ${'$'}entryKey + RETURN count(r) AS adopted + """.trimIndent() + ).bind(params).transform(Long::class.java) + ) ?: 0L + return adopted > 0 + } + + private fun replaceProvenance(propositionId: String, entries: List) { + appendProvenance(propositionId, entries) + val entryKeys = entries.map(::provenanceStorageEntryKey) + persistenceManager.execute( + QuerySpecification.withStatement( + """ + MATCH (p:Proposition {id: ${'$'}propositionId})-[r:DERIVED_FROM]->() + WHERE r.entryKey IS NULL OR NOT r.entryKey IN ${'$'}entryKeys + DELETE r + """.trimIndent() + ).bind(mapOf("propositionId" to propositionId, "entryKeys" to entryKeys)), + ) + persistenceManager.execute( + QuerySpecification.withStatement( + """ + MATCH (s:Source) + WHERE NOT (s)<-[:DERIVED_FROM]-() + DELETE s + """.trimIndent() + ), + ) + } + + private fun provenanceParameters( + propositionId: String, + entryKey: String, + edge: DerivedFrom, + ): Map = mapOf( + "propositionId" to propositionId, + "entryKey" to entryKey, + "sourceKey" to edge.source.key, + "sourceKind" to edge.source.kind, + "sourceDisplay" to edge.source.display, + "sourceUri" to edge.source.uri, + "sourcePath" to edge.source.path, + "sourceContentHash" to edge.source.contentHash, + "connectorId" to edge.source.connectorId, + "externalId" to edge.source.externalId, + "sourceRevision" to edge.sourceRevision, + "chunkId" to edge.chunkId, + "startOffset" to edge.startOffset, + "endOffset" to edge.endOffset, + "contentHash" to edge.contentHash, + ) + /** * Id of an existing proposition with the same `contextId` and exact `text` (excluding the * candidate's own id), or null if there is no duplicate. Matches on text alone — an identical @@ -248,7 +479,9 @@ class DrivinePropositionRepository( @Transactional(readOnly = true) override fun findById(id: String): Proposition? = - graphObjectManager.load(id)?.let(PropositionGraphMapper::toProposition) + graphObjectManager.load(id) + ?.let(PropositionGraphMapper::toProposition) + ?.let { withRawProvenance(listOf(it)).single() } @Transactional(readOnly = true) override fun findAll(): List = @@ -285,6 +518,43 @@ class DrivinePropositionRepository( where { proposition.contextId eq contextId.value } }.map(PropositionGraphMapper::toProposition) + @Transactional(readOnly = true) + override fun findBySourceKey(contextId: ContextId, sourceKey: String): List = + executeSourceQuery( + SourceProvenanceQueryStatements.bySourceKey, + mapOf("contextId" to contextId.value, "sourceKey" to sourceKey), + ) + + @Transactional(readOnly = true) + override fun findBySourceRevision(contextId: ContextId, ref: SourceRevisionRef): List = + executeSourceQuery( + SourceProvenanceQueryStatements.bySourceRevision, + mapOf( + "contextId" to contextId.value, + "sourceKey" to ref.sourceKey, + "sourceRevision" to ref.sourceRevision, + ), + ) + + @Transactional(readOnly = true) + override fun findRevisionlessBySourceLocator( + contextId: ContextId, + locator: SourceLocator, + ): List = + executeSourceQuery( + SourceProvenanceQueryStatements.revisionlessBySourceKey, + mapOf("contextId" to contextId.value, "sourceKey" to locator.key()), + ) + + @Suppress("UNCHECKED_CAST") + private fun executeSourceQuery(statement: String, params: Map): List { + val ids = persistenceManager.query( + QuerySpecification.withStatement(statement).bind(params) + ) as List + val byId = hydrate(ids) + return withRawProvenance(ids.distinct().mapNotNull { byId[it] }) + } + @Transactional(readOnly = true) override fun findByGrounding(chunkId: String): List = graphObjectManager.loadAll { where { proposition.grounding hasItem chunkId } } @@ -321,10 +591,7 @@ class DrivinePropositionRepository( @Transactional(readOnly = true) override fun findAll(withProvenance: Boolean): List = - if (!withProvenance) findAll() - else graphObjectManager.loadAll { - where { proposition.contextId.isNotNull() } // skip malformed/foreign :Proposition nodes (see findAll) - }.map(PropositionGraphMapper::toProposition) + if (!withProvenance) findAll() else withRawProvenance(findAll()) /** * The materialised `effectiveConfidence` (default k = 2.0, as of the last sweep) only matches a @@ -366,15 +633,69 @@ class DrivinePropositionRepository( return query.limit?.let { list.take(it) } ?: list } - /** Re-load a lean result set's ids through the provenance view (one batch query), preserving order. */ + /** Add raw provenance to a lean result set in one batch query, preserving order. */ private fun enrichWithProvenance(lean: List): List { + return withRawProvenance(lean) + } + + /** + * Raw fallback for full reads. Relationship fragments are mapped by endpoint/type, so raw Cypher + * rows preserve every parallel revision relationship. + */ + @Suppress("UNCHECKED_CAST") + private fun withRawProvenance(lean: List): List { if (lean.isEmpty()) return lean - val ids = lean.map { it.id } - val byId = graphObjectManager.loadAll { where { proposition.id inList ids } } - .associate { it.proposition.id to PropositionGraphMapper.toProposition(it) } - return lean.map { byId[it.id] ?: it } + val rows = persistenceManager.query( + QuerySpecification.withStatement( + """ + MATCH (p:Proposition)-[r:DERIVED_FROM]->(s:Source) + WHERE p.id IN ${'$'}ids + RETURN { + propositionId: p.id, + chunkId: r.chunkId, + startOffset: r.startOffset, + endOffset: r.endOffset, + contentHash: r.contentHash, + sourceRevision: r.sourceRevision, + sourceKey: s.key, + sourceKind: s.kind, + sourceDisplay: s.display, + sourceUri: s.uri, + sourcePath: s.path, + sourceContentHash: s.contentHash, + connectorId: s.connectorId, + externalId: s.externalId + } AS row + """.trimIndent() + ).bind(mapOf("ids" to lean.map { it.id })) + ) as List> + val byId = rows.groupBy { it["propositionId"] as String } + return lean.map { proposition -> + proposition.copy(provenanceEntries = byId[proposition.id].orEmpty().map(::toProvenanceEntry)) + } } + private fun toProvenanceEntry(row: Map): ProvenanceEntry = + PropositionGraphMapper.toProvenanceEntry( + DerivedFrom( + source = SourceNode( + key = row["sourceKey"] as String, + kind = row["sourceKind"] as String, + display = row["sourceDisplay"] as? String, + uri = row["sourceUri"] as? String, + path = row["sourcePath"] as? String, + contentHash = row["sourceContentHash"] as? String, + connectorId = row["connectorId"] as? String, + externalId = row["externalId"] as? String, + ), + chunkId = row["chunkId"] as? String, + startOffset = (row["startOffset"] as? Number)?.toInt(), + endOffset = (row["endOffset"] as? Number)?.toInt(), + contentHash = row["contentHash"] as? String, + sourceRevision = row["sourceRevision"] as? String, + ), + ) + /** Shared `where { }` filter block (PropositionView DSL); reused by query and the filtered-vector path. */ context(builder: WhereBuilder) private fun applyFilters(query: PropositionQuery, includeEffectiveConfidence: Boolean) { diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/PropositionGraphMapper.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/PropositionGraphMapper.kt index 5d88b781..ebe30422 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/PropositionGraphMapper.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/PropositionGraphMapper.kt @@ -168,22 +168,25 @@ object PropositionGraphMapper { // ---- Provenance: ProvenanceEntry <-> (DERIVED_FROM edge + shared Source node) ---- - private fun toDerivedFrom(e: ProvenanceEntry): DerivedFrom = + internal fun toDerivedFrom(e: ProvenanceEntry): DerivedFrom = DerivedFrom( chunkId = e.chunkId, startOffset = e.startOffset, endOffset = e.endOffset, contentHash = e.contentHash, + sourceRevision = e.sourceRevision, + entryKey = provenanceStorageEntryKey(e), source = toSourceNode(e.locator), ) - private fun toProvenanceEntry(df: DerivedFrom): ProvenanceEntry = + internal fun toProvenanceEntry(df: DerivedFrom): ProvenanceEntry = ProvenanceEntry( locator = toLocator(df.source), chunkId = df.chunkId, startOffset = df.startOffset, endOffset = df.endOffset, contentHash = df.contentHash, + sourceRevision = df.sourceRevision, ) private fun toSourceNode(loc: SourceLocator): SourceNode = @@ -228,3 +231,21 @@ object PropositionGraphMapper { ) } } + +/** + * Canonical graph relationship identity for provenance. + * + * Length framing keeps null, empty, and delimiter-containing values distinct without coupling + * graph persistence to the domain's public evidence-key format. + */ +internal fun provenanceStorageEntryKey(entry: ProvenanceEntry): String = + listOf( + entry.locator.key(), + entry.sourceRevision, + entry.chunkId, + entry.startOffset?.toString(), + entry.endOffset?.toString(), + entry.contentHash, + ).joinToString(separator = "") { value -> + if (value == null) "-1:" else "${value.length}:$value" + } diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/model/DerivedFrom.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/model/DerivedFrom.kt index 27de1cc1..2f8538fd 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/model/DerivedFrom.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/model/DerivedFrom.kt @@ -23,10 +23,12 @@ import org.drivine.annotation.RelationshipFragment * the shared node. Together they reconstitute a dice `ProvenanceEntry`. */ @RelationshipFragment -data class DerivedFrom( +data class DerivedFrom @JvmOverloads constructor( val chunkId: String? = null, val startOffset: Int? = null, val endOffset: Int? = null, val contentHash: String? = null, val source: SourceNode, + val sourceRevision: String? = null, + val entryKey: String? = null, ) diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/CollectorTraceRowMapperTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/CollectorTraceRowMapperTest.kt index 947c6d6b..361470d5 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/CollectorTraceRowMapperTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/CollectorTraceRowMapperTest.kt @@ -35,6 +35,7 @@ class CollectorTraceRowMapperTest { foldedGrounding = listOf("g1", "g2"), foldedProvenanceRefs = listOf("prov-1"), foldedSourceIds = listOf("src-1", "src-2"), + foldedProvenanceEvidenceKeys = listOf("evidence-1"), ) val bindMap = CollectorDecisionRowMapper.retiredBindMap("run-1", mockDecision(), retired) @@ -45,6 +46,23 @@ class CollectorTraceRowMapperTest { assertEquals(listOf("g1", "g2"), roundTripped.foldedGrounding) assertEquals(listOf("prov-1"), roundTripped.foldedProvenanceRefs) assertEquals(listOf("src-1", "src-2"), roundTripped.foldedSourceIds) + assertEquals(listOf("evidence-1"), roundTripped.foldedProvenanceEvidenceKeys) + } + + @Test + fun `legacy retired row without evidence keys remains readable`() { + val legacyRow = mapOf( + "propositionId" to "prop-1", + "priorStatus" to "ACTIVE", + "foldedGrounding" to emptyList(), + "foldedProvenanceRefs" to listOf("uri:https://example.com/source"), + "foldedSourceIds" to emptyList(), + ) + + val retired = CollectorDecisionRowMapper.retiredFromRow(legacyRow) + + assertEquals(listOf("uri:https://example.com/source"), retired.foldedProvenanceRefs) + assertEquals(emptyList(), retired.foldedProvenanceEvidenceKeys) } @Test diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineCollectorTraceStoreIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineCollectorTraceStoreIntegrationTest.kt index bdcc8fda..3405165f 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineCollectorTraceStoreIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineCollectorTraceStoreIntegrationTest.kt @@ -20,7 +20,11 @@ import com.embabel.dice.projection.memory.collector.CollectorRunContext import com.embabel.dice.projection.memory.collector.CollectorSurvivorPolicy import com.embabel.dice.projection.memory.collector.MultiSignalCollectorStrategy import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.proposition.PropositionStore import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.UriLocator import com.embabel.dice.spi.CandidatePair import com.embabel.dice.spi.CandidatePairSource import com.embabel.dice.spi.CollectorCandidateEdge @@ -57,6 +61,33 @@ class DrivineCollectorTraceStoreIntegrationTest { @Autowired private lateinit var propositionRepository: DrivinePropositionRepository + private class RecordingPropositionRepository( + private val delegate: PropositionRepository, + ) : PropositionRepository by delegate { + + var saveCalls = 0 + private set + var setProvenanceCalls = 0 + private set + + override fun save(proposition: Proposition): Proposition { + saveCalls++ + return delegate.save(proposition) + } + + override fun setProvenance( + propositionId: String, + entries: List, + ): Proposition? { + setProvenanceCalls++ + return delegate.setProvenance(propositionId, entries) + } + } + + private class BaseStoreView( + delegate: PropositionRepository, + ) : PropositionStore by delegate + @AfterEach fun cleanUp() { CollectorTraceSchema.LABELS.forEach { label -> @@ -70,6 +101,7 @@ class DrivineCollectorTraceStoreIntegrationTest { text: String, status: PropositionStatus = PropositionStatus.ACTIVE, grounding: List = emptyList(), + provenance: List = emptyList(), ) = Proposition( id = id, contextId = ContextId("ctx-undo"), @@ -78,6 +110,7 @@ class DrivineCollectorTraceStoreIntegrationTest { confidence = 0.9, status = status, grounding = grounding, + provenanceEntries = provenance, ) private fun edge(anchorId: String, memberId: String, vetoed: Boolean = false, score: Double = 0.9) = CollectorCandidateEdge( @@ -104,6 +137,7 @@ class DrivineCollectorTraceStoreIntegrationTest { foldedGrounding = listOf("g1", "g2"), foldedProvenanceRefs = listOf("prov-1"), foldedSourceIds = listOf("src-1", "src-2"), + foldedProvenanceEvidenceKeys = listOf("evidence-1"), ), ), ) @@ -139,6 +173,7 @@ class DrivineCollectorTraceStoreIntegrationTest { assertEquals(listOf("g1", "g2"), retired.foldedGrounding) assertEquals(listOf("prov-1"), retired.foldedProvenanceRefs) assertEquals(listOf("src-1", "src-2"), retired.foldedSourceIds) + assertEquals(listOf("evidence-1"), retired.foldedProvenanceEvidenceKeys) } @Test @@ -219,6 +254,68 @@ class DrivineCollectorTraceStoreIntegrationTest { assertNull(traceStore.findRetirement("unknown")) } + @Test + fun `undoSingleCollapse does not write when the retired proposition is missing`() { + val runId = "run-missing-retired" + val survivor = propositionRepository.save(prop("survivor-missing-retired", "Survivor remains")) + traceStore.recordRunContext(runId, survivor.contextId) + traceStore.recordDecision( + runId, + decisionFor( + componentId = "comp-missing-retired", + survivorId = survivor.id, + retiredId = "missing-retired", + ), + ) + val before = propositionRepository.findAll().sortedBy { it.id } + val recordingRepository = RecordingPropositionRepository(propositionRepository) + + val result = undoSingleCollapse( + traceQuery = traceStore, + propositions = recordingRepository, + survivorId = survivor.id, + retiredId = "missing-retired", + ) + + val after = propositionRepository.findAll().sortedBy { it.id } + assertNull(result) + assertEquals(before, after) + assertEquals(0, recordingRepository.saveCalls) + assertEquals(0, recordingRepository.setProvenanceCalls) + } + + @Test + fun `undoSingleCollapse does not write when the survivor proposition is missing`() { + val runId = "run-missing-survivor" + val retired = propositionRepository.save( + prop("retired-missing-survivor", "Retired remains", status = PropositionStatus.STALE), + ) + traceStore.recordRunContext(runId, retired.contextId) + traceStore.recordDecision( + runId, + decisionFor( + componentId = "comp-missing-survivor", + survivorId = "missing-survivor", + retiredId = retired.id, + ), + ) + val before = propositionRepository.findAll().sortedBy { it.id } + val recordingRepository = RecordingPropositionRepository(propositionRepository) + + val result = undoSingleCollapse( + traceQuery = traceStore, + propositions = recordingRepository, + survivorId = "missing-survivor", + retiredId = retired.id, + ) + + val after = propositionRepository.findAll().sortedBy { it.id } + assertNull(result) + assertEquals(before, after) + assertEquals(0, recordingRepository.saveCalls) + assertEquals(0, recordingRepository.setProvenanceCalls) + } + @Test fun `undoSingleCollapse restores one member and subtracts only its exclusive grounding, leaving the sibling member retired`() { val runId = "run-5" @@ -339,4 +436,115 @@ class DrivineCollectorTraceStoreIntegrationTest { // (b) the survivor loses "loser-exclusive" — that one really did come from the loser. assertEquals(setOf("shared"), updatedSurvivor?.grounding?.toSet()) } + + @Test + fun `collector undo removes only the folded revision from persistent provenance`() { + val runId = "run-revision-undo" + val contextId = ContextId("ctx-revision-undo") + val locator = UriLocator("https://example.com/revision-undo") + val revisionOne = ProvenanceEntry(locator = locator, sourceRevision = "r1") + val revisionTwo = ProvenanceEntry(locator = locator, sourceRevision = "r2") + val survivor = propositionRepository.save( + prop( + id = "survivor-revision", + text = "Acme signed the agreement", + provenance = listOf(revisionOne), + ), + ) + val loser = propositionRepository.save( + prop( + id = "loser-revision", + text = "Acme signed an agreement", + provenance = listOf(revisionOne, revisionTwo), + ), + ) + val strategy = MultiSignalCollectorStrategy( + pairSources = listOf( + CandidatePairSource { + candidates, _ -> + listOf(CandidatePair(anchor = candidates[0], member = candidates[1])) + }, + ), + scorers = listOf( + CollectorSignalScorer { _, _ -> CollectorSignalScore(signal = "fixed", score = 1.0) }, + ), + componentsFinder = InMemoryConnectedComponentsFinder(), + traceStore = traceStore, + survivorPolicy = CollectorSurvivorPolicy { members -> members.single { it.id == survivor.id } }, + matchThreshold = 0.5, + ) + + strategy.mark(listOf(survivor, loser), propositionRepository, CollectorRunContext(runId, contextId)) + propositionRepository.save( + propositionRepository.findById(survivor.id)!!.absorbEvidence(loser), + ) + + undoSingleCollapse( + traceQuery = traceStore, + propositions = propositionRepository, + survivorId = survivor.id, + retiredId = loser.id, + ) + + assertEquals( + listOf(revisionOne), + propositionRepository.findById(survivor.id)?.provenanceEntries, + ) + } + + @Test + fun `collector undo removes folded provenance through a base store decorator`() { + val runId = "run-base-store-undo" + val contextId = ContextId("ctx-base-store-undo") + val survivorEvidence = ProvenanceEntry( + locator = UriLocator("https://example.com/base-store-undo/keep"), + ) + val foldedEvidence = ProvenanceEntry( + locator = UriLocator("https://example.com/base-store-undo/remove"), + ) + val survivor = propositionRepository.save( + prop( + id = "survivor-base-store", + text = "Acme signed the agreement", + provenance = listOf(survivorEvidence, foldedEvidence), + ), + ) + propositionRepository.save( + prop( + id = "retired-base-store", + text = "Acme signed an agreement", + status = PropositionStatus.STALE, + provenance = listOf(foldedEvidence), + ), + ) + traceStore.recordRunContext(runId, contextId) + traceStore.recordDecision( + runId, + CollectorDecision( + runId = runId, + componentId = "component-base-store", + survivorId = survivor.id, + action = "duplicate-merge", + retired = listOf( + RetiredProposition( + propositionId = "retired-base-store", + priorStatus = PropositionStatus.ACTIVE, + foldedProvenanceRefs = listOf(foldedEvidence.locator.key()), + ), + ), + ), + ) + + undoSingleCollapse( + traceQuery = traceStore, + propositions = BaseStoreView(propositionRepository), + survivorId = survivor.id, + retiredId = "retired-base-store", + ) + + assertEquals( + listOf(survivorEvidence), + propositionRepository.findById(survivor.id)?.provenanceEntries, + ) + } } diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivinePropositionStoreIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivinePropositionStoreIntegrationTest.kt index 4b89a448..bd681c50 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivinePropositionStoreIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivinePropositionStoreIntegrationTest.kt @@ -27,32 +27,52 @@ import com.embabel.dice.proposition.MentionRole import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionQuery import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.provenance.ConnectorRef import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.SourceLocator +import com.embabel.dice.provenance.SourceRevisionRef import com.embabel.dice.provenance.UriLocator import com.embabel.dice.temporal.TemporalMetadata import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.drivine.connection.DataSourceMap import org.drivine.manager.CascadeType import org.drivine.manager.GraphObjectManager import org.drivine.manager.PersistenceManager import org.drivine.query.QuerySpecification +import org.neo4j.driver.AuthTokens +import org.neo4j.driver.GraphDatabase +import org.neo4j.driver.SessionConfig +import org.neo4j.driver.summary.Plan +import org.springframework.aop.framework.Advised +import org.springframework.aop.framework.ProxyFactory +import org.springframework.aop.support.AopUtils import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.context.TestConfiguration import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.annotation.EnableTransactionManagement +import org.springframework.transaction.interceptor.TransactionInterceptor +import org.springframework.transaction.support.TransactionTemplate import java.time.Duration import java.time.Instant +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit /** * Integration tests for the graph storage stack against a Neo4j testcontainer (provided by Drivine's - * test support). Not `@Transactional`: dedup commits via its own [org.springframework.transaction.support.TransactionTemplate], - * so isolation is by explicit `clearAll()` per test rather than rollback. + * test support). Not `@Transactional`: standalone dedup commits via its own + * [org.springframework.transaction.support.TransactionTemplate], so isolation is by explicit + * `clearAll()` per test rather than an ambient test rollback. */ -@SpringBootTest(classes = [TestApplication::class]) +@SpringBootTest(classes = [TestApplication::class, TransactionProxyTestConfiguration::class]) class DrivinePropositionStoreIntegrationTest { @Autowired @@ -76,6 +96,9 @@ class DrivinePropositionStoreIntegrationTest { @Autowired private lateinit var transactionManager: PlatformTransactionManager + @Autowired + private lateinit var dataSourceMap: DataSourceMap + @AfterEach fun cleanUp() { repository.clearAll() @@ -137,6 +160,170 @@ class DrivinePropositionStoreIntegrationTest { QuerySpecification.withStatement("MATCH (n:$label) RETURN count(n) AS c").transform(Long::class.java), ) + private fun revisionEvidence( + locator: SourceLocator, + revision: String?, + chunkId: String = "chunk", + contentHash: String = "hash", + ): ProvenanceEntry = + ProvenanceEntry( + locator = locator, + chunkId = chunkId, + startOffset = 1, + endOffset = 4, + contentHash = contentHash, + sourceRevision = revision, + ) + + private fun edgeCount(propositionId: String): Long = + persistenceManager.getOne( + QuerySpecification + .withStatement( + "MATCH (:Proposition {id: \$propositionId})-[r:DERIVED_FROM]->() RETURN count(r) AS c", + ) + .bind(mapOf("propositionId" to propositionId)) + .transform(Long::class.java), + ) + + private fun seedLegacyEdge(propositionId: String, entry: ProvenanceEntry) { + val locator = entry.locator as UriLocator + persistenceManager.execute( + QuerySpecification + .withStatement( + """ + MATCH (p:Proposition {id: ${'$'}propositionId}) + MERGE (s:Source {key: ${'$'}sourceKey}) + SET s.kind = 'uri', s.uri = ${'$'}sourceUri, s.display = ${'$'}sourceDisplay + CREATE (p)-[r:DERIVED_FROM]->(s) + SET r.chunkId = ${'$'}chunkId, + r.startOffset = ${'$'}startOffset, + r.endOffset = ${'$'}endOffset, + r.contentHash = ${'$'}contentHash + """.trimIndent(), + ) + .bind( + mapOf( + "propositionId" to propositionId, + "sourceKey" to locator.key(), + "sourceUri" to locator.uri, + "sourceDisplay" to locator.display, + "chunkId" to entry.chunkId, + "startOffset" to entry.startOffset, + "endOffset" to entry.endOffset, + "contentHash" to entry.contentHash, + ), + ), + ) + } + + /** + * A second repository target with the same transaction advisor as the actual Spring bean. + * Separate targets model two application instances (and therefore two lock stripes), while both + * calls still pass through the production annotation-driven transaction interceptor. + */ + private fun newTransactionProxiedRepository( + repositoryPersistenceManager: PersistenceManager = persistenceManager, + ): DrivinePropositionRepository { + val actualBean = repository as Advised + val transactionAdvisor = actualBean.advisors.single { it.advice is TransactionInterceptor } + val proxyFactory = ProxyFactory( + DrivinePropositionRepository( + graphObjectManager, + repositoryPersistenceManager, + embeddingService, + transactionManager, + ), + ) + proxyFactory.setProxyTargetClass(true) + proxyFactory.addAdvisor(transactionAdvisor) + return proxyFactory.proxy as DrivinePropositionRepository + } + + private class DedupLookupBarrierPersistenceManager( + private val delegate: PersistenceManager, + private val barrier: CountDownLatch, + ) : PersistenceManager by delegate { + + override fun maybeGetOne(spec: QuerySpecification): T? { + val result = delegate.maybeGetOne(spec) + if (result == null && spec.parameters.keys == setOf("contextId", "text", "excludeId")) { + barrier.countDown() + assertTrue(barrier.await(10, TimeUnit.SECONDS), "both repositories must observe no duplicate") + } + return result + } + } + + private data class CapturedSourceQuery( + val statement: String, + val parameters: Map, + ) + + private class SourceQueryCapturingPersistenceManager( + private val delegate: PersistenceManager, + ) : PersistenceManager by delegate { + + val captured = mutableListOf() + + override fun query(spec: QuerySpecification): List { + if (spec.parameters.keys.containsAll(setOf("contextId", "sourceKey"))) { + @Suppress("UNCHECKED_CAST") + captured += CapturedSourceQuery( + requireNotNull(spec.statement).text, + spec.parameters.toMap() as Map, + ) + } + return delegate.query(spec) + } + } + + private fun captureAndExplainSourceQuery( + name: String, + expectedStatement: String, + expectedParameters: Map, + invocation: (DrivinePropositionRepository) -> List, + ): String { + val capturingPersistenceManager = SourceQueryCapturingPersistenceManager(persistenceManager) + val capturingRepository = newTransactionProxiedRepository(capturingPersistenceManager) + val results = invocation(capturingRepository) + assertTrue(results.isNotEmpty(), "$name invocation must return seeded evidence") + assertEquals(1, capturingPersistenceManager.captured.size, "$name must execute one source query") + val captured = capturingPersistenceManager.captured.single() + assertEquals(expectedStatement, captured.statement, "$name must execute its production statement") + assertEquals(expectedParameters, captured.parameters, "$name must bind its production parameters") + return explainSourceQuery(name, captured.statement, captured.parameters) + } + + private fun explainSourceQuery( + name: String, + statement: String, + parameters: Map, + ): String { + val dataSource = dataSourceMap.dataSources.getValue("neo") + val uri = "${dataSource.protocol ?: "bolt"}://${dataSource.host}:${dataSource.port ?: 7687}" + val auth = dataSource.userName?.let { AuthTokens.basic(it, dataSource.password.orEmpty()) } + ?: AuthTokens.none() + return GraphDatabase.driver(uri, auth).use { driver -> + val sessionConfig = dataSource.databaseName?.let(SessionConfig::forDatabase) + ?: SessionConfig.defaultConfig() + driver.session(sessionConfig).use { session -> + val plan = session.run("EXPLAIN\n$statement", parameters).consume().plan() + renderPlan(plan).also { println("SOURCE_QUERY_EXPLAIN[$name]\n$it") } + } + } + } + + private fun renderPlan(plan: Plan, depth: Int = 0): String = buildString { + append(" ".repeat(depth)) + append(plan.operatorType()) + if (plan.arguments().isNotEmpty()) { + append(' ') + append(plan.arguments()) + } + appendLine() + plan.children().forEach { append(renderPlan(it, depth + 1)) } + }.trimEnd() + @Test fun `save round-trips all persisted fields`() { val saved = repository.save( @@ -294,6 +481,414 @@ class DrivinePropositionStoreIntegrationTest { assertEquals(1L, sourceCount) } + @Test + fun `colliding connector tuples cannot overwrite a shared source`() { + val originalLocator = ConnectorRef("a:b", "c") + val collidingLocator = ConnectorRef("a", "b:c") + assertEquals(originalLocator.key(), collidingLocator.key()) + val original = repository.save( + prop( + text = "original connector fact", + context = "ctx-a", + provenance = listOf(ProvenanceEntry(originalLocator)), + ), + ) + + assertThrows { + repository.save( + prop( + text = "colliding connector fact", + context = "ctx-b", + provenance = listOf(ProvenanceEntry(collidingLocator)), + ), + ) + } + + assertEquals( + originalLocator, + repository.findById(original.id)?.provenanceEntries?.single()?.locator, + ) + } + + @Test + fun `parallel source revisions persist as distinct edges and hydrate through every full read`() { + val locator = UriLocator("https://example.com/revisioned") + val revisionOne = revisionEvidence(locator, "r1") + val revisionTwo = revisionOne.copy(sourceRevision = "r2") + val saved = repository.save(prop("revisioned fact", provenance = listOf(revisionOne, revisionTwo))) + + val sourceCount = persistenceManager.getOne( + QuerySpecification + .withStatement("MATCH (s:Source {key: \$sourceKey}) RETURN count(s) AS c") + .bind(mapOf("sourceKey" to locator.key())) + .transform(Long::class.java), + ) + val edgeCount = persistenceManager.getOne( + QuerySpecification + .withStatement( + "MATCH (:Proposition {id: \$id})-[r:DERIVED_FROM]->(:Source {key: \$sourceKey}) " + + "RETURN count(r) AS c", + ) + .bind(mapOf("id" to saved.id, "sourceKey" to locator.key())) + .transform(Long::class.java), + ) + + assertEquals(1L, sourceCount, "parallel revisions share one Source node") + assertEquals(2L, edgeCount, "parallel revisions require two DERIVED_FROM relationships") + val expected = setOf(revisionOne, revisionTwo) + assertEquals(expected, repository.findById(saved.id)!!.provenanceEntries.toSet()) + assertEquals( + expected, + repository.query(PropositionQuery.forContextId(ContextId("ctx")), withProvenance = true) + .single { it.id == saved.id }.provenanceEntries.toSet(), + ) + assertEquals( + expected, + repository.findAll(withProvenance = true).single { it.id == saved.id }.provenanceEntries.toSet(), + ) + } + + @Test + fun `repeated and concurrent writes of one revision remain one relationship`() { + val locator = UriLocator("https://example.com/idempotent") + val revisionOne = revisionEvidence(locator, "r1") + val saved = repository.save(prop("idempotent evidence", provenance = listOf(revisionOne))) + repository.save(saved) + + val firstRepository = DrivinePropositionRepository( + graphObjectManager, persistenceManager, embeddingService, transactionManager, + ) + val secondRepository = DrivinePropositionRepository( + graphObjectManager, persistenceManager, embeddingService, transactionManager, + ) + val start = CountDownLatch(1) + val executor = Executors.newFixedThreadPool(2) + try { + val writes = listOf(firstRepository, secondRepository).map { candidate -> + executor.submit { + start.await(10, TimeUnit.SECONDS) + candidate.save(saved) + } + } + start.countDown() + writes.forEach { it.get(30, TimeUnit.SECONDS) } + } finally { + executor.shutdownNow() + } + + assertEquals(1L, edgeCount(saved.id)) + assertEquals(listOf(revisionOne), repository.findById(saved.id)!!.provenanceEntries) + } + + @Test + fun `ordinary exact-text dedup unions incoming source revisions into the winner`() { + val locator = ConnectorRef("ordinary-dedup", "source") + val revisionOne = revisionEvidence(locator, "r1") + val revisionTwo = revisionEvidence(locator, "r2") + val initialRevision = Instant.now().minusSeconds(60) + val winner = repository.save( + prop( + "same extracted fact", + contentRevised = initialRevision, + metadataRevised = initialRevision, + provenance = listOf(revisionOne), + ), + ) + + val deduplicated = repository.save(prop("same extracted fact", provenance = listOf(revisionTwo))) + + assertEquals(winner.id, deduplicated.id) + assertEquals(1, repository.count()) + val revisedWinner = repository.findById(winner.id)!! + assertEquals(setOf(revisionOne, revisionTwo), revisedWinner.provenanceEntries.toSet()) + assertEquals(2L, edgeCount(winner.id)) + assertTrue(revisedWinner.metadataRevised > initialRevision) + assertEquals( + listOf(revisedWinner.id), + repository.query( + PropositionQuery.forContextId(revisedWinner.contextId) + .withRevisedAfter(initialRevision.plusSeconds(1)), + ).map { it.id }, + ) + + val revisedAt = revisedWinner.metadataRevised + repository.save(prop("same extracted fact", provenance = listOf(revisionTwo))) + assertEquals(revisedAt, repository.findById(winner.id)!!.metadataRevised) + } + + @Test + fun `ordinary exact-text dedup rejects a colliding connector source and preserves its winner`() { + val originalLocator = ConnectorRef("a:b", "c") + val collidingLocator = ConnectorRef("a", "b:c") + assertEquals(originalLocator.key(), collidingLocator.key()) + val originalEntry = ProvenanceEntry(originalLocator) + val winner = repository.save(prop("same collision-prone fact", provenance = listOf(originalEntry))) + + assertThrows { + repository.save( + prop( + "same collision-prone fact", + provenance = listOf(ProvenanceEntry(collidingLocator)), + ), + ) + } + + assertEquals(1, repository.count()) + assertEquals( + listOf(originalEntry), + repository.findById(winner.id)?.provenanceEntries, + "a rejected dedup collision must leave the stored winner unchanged", + ) + } + + @Test + fun `caller rollback undoes a completed save`() { + val proposition = prop("transactional save") + TransactionTemplate(transactionManager).executeWithoutResult { status -> + repository.save(proposition) + assertNotNull(repository.findById(proposition.id), "save must be visible inside the caller transaction") + status.setRollbackOnly() + } + + assertNull(repository.findById(proposition.id), "caller rollback must remove the saved proposition") + } + + @Test + fun `Spring proxied cross-instance uniqueness recovery commits evidence and is retry safe`() { + val locator = ConnectorRef("race-dedup", "source") + val revisionOne = revisionEvidence(locator, "r1") + val revisionTwo = revisionEvidence(locator, "r2") + val first = prop("cross-instance fact", provenance = listOf(revisionOne)) + val second = prop("cross-instance fact", provenance = listOf(revisionTwo)) + val barrier = CountDownLatch(2) + val firstRepository = newTransactionProxiedRepository( + DedupLookupBarrierPersistenceManager(persistenceManager, barrier), + ) + val secondRepository = newTransactionProxiedRepository( + DedupLookupBarrierPersistenceManager(persistenceManager, barrier), + ) + assertTrue(AopUtils.isAopProxy(firstRepository), "the primary writer must be the actual Spring proxy") + assertTrue(AopUtils.isAopProxy(secondRepository), "the sibling writer must carry the transaction advisor") + val executor = Executors.newFixedThreadPool(2) + try { + val winners = listOf( + executor.submit { firstRepository.save(first) }, + executor.submit { secondRepository.save(second) }, + ).map { it.get(30, TimeUnit.SECONDS) } + + assertEquals(1, winners.map { it.id }.toSet().size, "both writers must return the database winner") + val winnerId = winners.map { it.id }.distinct().single() + assertEquals(setOf(revisionOne, revisionTwo), repository.findById(winnerId)!!.provenanceEntries.toSet()) + assertEquals(2L, edgeCount(winnerId), "the losing proxy's recovery transaction must commit evidence") + + secondRepository.save(second) + assertEquals(2L, edgeCount(winnerId), "retrying recovered evidence must remain idempotent") + } finally { + executor.shutdownNow() + } + } + + @Test + fun `cross-instance uniqueness recovery rejects a colliding connector source and preserves its winner`() { + val originalLocator = ConnectorRef("a:b", "c") + val collidingLocator = ConnectorRef("a", "b:c") + assertEquals(originalLocator.key(), collidingLocator.key()) + val first = prop( + "cross-instance collision-prone fact", + provenance = listOf(ProvenanceEntry(originalLocator)), + ) + val second = prop( + "cross-instance collision-prone fact", + provenance = listOf(ProvenanceEntry(collidingLocator)), + ) + val barrier = CountDownLatch(2) + val firstRepository = newTransactionProxiedRepository( + DedupLookupBarrierPersistenceManager(persistenceManager, barrier), + ) + val secondRepository = newTransactionProxiedRepository( + DedupLookupBarrierPersistenceManager(persistenceManager, barrier), + ) + val executor = Executors.newFixedThreadPool(2) + try { + val outcomes = listOf( + executor.submit { firstRepository.save(first) }, + executor.submit { secondRepository.save(second) }, + ).map { future -> + runCatching { future.get(30, TimeUnit.SECONDS) } + } + + assertEquals(1, outcomes.count { it.isSuccess }, "exactly one structurally valid writer must win") + assertEquals(1, outcomes.count { it.isFailure }, "the colliding writer's recovery must fail") + val collision = outcomes.single { it.isFailure }.exceptionOrNull()!! + assertTrue( + generateSequence(collision) { it.cause }.any { + it is IllegalArgumentException && it.message?.contains("Source key collision") == true + }, + "the losing recovery must report the structural Source key collision", + ) + + val winner = outcomes.single { it.isSuccess }.getOrThrow() + val storedWinner = repository.findById(winner.id)!! + assertEquals(1, repository.count()) + assertEquals( + winner.provenanceEntries, + storedWinner.provenanceEntries, + "a rejected recovery collision must leave the database winner unchanged", + ) + assertEquals(1L, edgeCount(winner.id)) + } finally { + executor.shutdownNow() + } + } + + @Test + fun `legacy unkeyed revisionless evidence is adopted exactly while revisioned writes stay separate`() { + val locator = UriLocator("https://example.com/legacy") + val revisionless = revisionEvidence(locator, null) + val revisionOne = revisionEvidence(locator, "r1") + val adoptTarget = repository.save(prop("adopt legacy")) + seedLegacyEdge(adoptTarget.id, revisionless) + + assertEquals(listOf(revisionless), repository.findById(adoptTarget.id)!!.provenanceEntries) + repository.save(adoptTarget.withProvenanceEntries(listOf(revisionless))) + assertEquals(1L, edgeCount(adoptTarget.id), "the exact revisionless legacy edge is adopted in place") + val adoptedKeys = persistenceManager.getOne( + QuerySpecification + .withStatement( + "MATCH (:Proposition {id: \$id})-[r:DERIVED_FROM]->() " + + "RETURN count(r.entryKey) AS c", + ) + .bind(mapOf("id" to adoptTarget.id)) + .transform(Long::class.java), + ) + assertEquals(1L, adoptedKeys) + + val revisionedTarget = repository.save(prop("keep legacy separate")) + seedLegacyEdge(revisionedTarget.id, revisionless) + repository.save(revisionedTarget.withProvenanceEntries(listOf(revisionOne))) + + assertEquals(2L, edgeCount(revisionedTarget.id), "a revisioned write must not adopt an unkeyed edge") + assertEquals( + setOf(revisionless, revisionOne), + repository.findById(revisionedTarget.id)!!.provenanceEntries.toSet(), + ) + + val nonExactTarget = repository.save(prop("keep non-exact legacy separate")) + val differentRevisionless = revisionless.copy(chunkId = "different-chunk") + seedLegacyEdge(nonExactTarget.id, revisionless) + repository.save(nonExactTarget.withProvenanceEntries(listOf(differentRevisionless))) + + assertEquals(2L, edgeCount(nonExactTarget.id), "a non-exact revisionless write must not adopt the legacy edge") + assertEquals( + setOf(revisionless, differentRevisionless), + repository.findById(nonExactTarget.id)!!.provenanceEntries.toSet(), + ) + } + + @Test + fun `authoritative provenance replace removes omitted edges and only globally orphaned sources`() { + val sharedLocator = UriLocator("https://example.com/shared-replace") + val exclusiveLocator = UriLocator("https://example.com/exclusive-replace") + val keep = revisionEvidence(sharedLocator, "r1", chunkId = "keep") + val omittedParallel = revisionEvidence(sharedLocator, "r2", chunkId = "omit-parallel") + val omittedExclusive = revisionEvidence(exclusiveLocator, "r1", chunkId = "omit-exclusive") + val subject = repository.save( + prop("replace subject", provenance = listOf(keep, omittedParallel, omittedExclusive)), + ) + val other = repository.save( + prop("shared source remains", provenance = listOf(revisionEvidence(sharedLocator, "other"))), + ) + + repository.setProvenance(subject.id, listOf(keep)) + + assertEquals(listOf(keep), repository.findById(subject.id)!!.provenanceEntries) + assertEquals(1L, edgeCount(subject.id)) + assertEquals(1L, edgeCount(other.id), "replace must not disturb another proposition's edge") + val sourceKeys = persistenceManager.query( + QuerySpecification.withStatement("MATCH (s:Source) RETURN s.key AS key"), + ).toSet() + assertEquals(setOf(sharedLocator.key()), sourceKeys, "only the globally orphaned source is pruned") + } + + @Test + fun `source queries push down by context and distinguish exact and revisionless evidence`() { + val locator = UriLocator("https://example.com/query-source") + val revisionless = repository.save( + prop("revisionless ctx-a", context = "ctx-a", provenance = listOf(revisionEvidence(locator, null))), + ) + val revisionOne = repository.save( + prop("r1 ctx-a", context = "ctx-a", provenance = listOf(revisionEvidence(locator, "r1"))), + ) + val revisionTwo = repository.save( + prop("r2 ctx-a", context = "ctx-a", provenance = listOf(revisionEvidence(locator, "r2"))), + ) + repository.save( + prop("r1 ctx-b", context = "ctx-b", provenance = listOf(revisionEvidence(locator, "r1"))), + ) + val context = ContextId("ctx-a") + + assertEquals( + setOf(revisionless.id, revisionOne.id, revisionTwo.id), + repository.findBySourceKey(context, locator.key()).map { it.id }.toSet(), + ) + assertEquals( + listOf(revisionOne.id), + repository.findBySourceRevision(context, SourceRevisionRef(locator.key(), "r1")).map { it.id }, + ) + assertEquals( + listOf(revisionless.id), + repository.findRevisionlessBySourceLocator(context, locator).map { it.id }, + ) + } + + @Test + fun `public source queries execute their parameterized tenant-first Neo4j plans`() { + val locator = UriLocator("https://example.com/explain-source") + repository.save( + prop( + "explain revisionless", + context = "ctx-explain", + provenance = listOf(revisionEvidence(locator, null)), + ), + ) + repository.save( + prop( + "explain r1", + context = "ctx-explain", + provenance = listOf(revisionEvidence(locator, "r1")), + ), + ) + val commonParameters = mapOf( + "contextId" to "ctx-explain", + "sourceKey" to locator.key(), + ) + val plans = mapOf( + "source-key" to captureAndExplainSourceQuery( + "source-key", + SourceProvenanceQueryStatements.bySourceKey, + commonParameters, + ) { it.findBySourceKey(ContextId("ctx-explain"), locator.key()) }, + "source-revision" to captureAndExplainSourceQuery( + "source-revision", + SourceProvenanceQueryStatements.bySourceRevision, + commonParameters + ("sourceRevision" to "r1"), + ) { it.findBySourceRevision(ContextId("ctx-explain"), SourceRevisionRef(locator.key(), "r1")) }, + "revisionless-source" to captureAndExplainSourceQuery( + "revisionless-source", + SourceProvenanceQueryStatements.revisionlessBySourceKey, + commonParameters, + ) { it.findRevisionlessBySourceLocator(ContextId("ctx-explain"), locator) }, + ) + + plans.forEach { (name, plan) -> + assertTrue(plan.isNotBlank(), "$name EXPLAIN must return a plan") + assertTrue( + plan.lineSequence().any { it.contains("IndexSeek") && it.contains("contextId") }, + "$name should seek the tenant contextId index before graph expansion:\n$plan", + ) + } + } + @Test fun `chunk history records, dedups, and bookmarks`() { val contextId = ContextId("chunk-ctx") @@ -1000,3 +1595,7 @@ class DrivinePropositionStoreIntegrationTest { assertEquals(old, repository.findById(saved.id)!!.lastAccessed) } } + +@TestConfiguration(proxyBeanMethods = false) +@EnableTransactionManagement(proxyTargetClass = true) +open class TransactionProxyTestConfiguration diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/PropositionGraphMapperRevisionTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/PropositionGraphMapperRevisionTest.kt new file mode 100644 index 00000000..cb76e809 --- /dev/null +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/PropositionGraphMapperRevisionTest.kt @@ -0,0 +1,129 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.UriLocator +import com.embabel.dice.storage.model.DerivedFrom +import com.embabel.dice.storage.model.SourceNode +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import org.junit.jupiter.api.Test +import java.time.Instant +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull + +class PropositionGraphMapperRevisionTest { + + @Test + fun `DerivedFrom retains its legacy Java constructor descriptor`() { + DerivedFrom::class.java.getConstructor( + String::class.java, + Integer::class.java, + Integer::class.java, + String::class.java, + SourceNode::class.java, + ) + } + + @Test + fun `revisionless and revisioned evidence round trips without duplicating source identity`() { + val locator = UriLocator("https://example.com/source") + val entries = listOf( + evidence(locator = locator, sourceRevision = null), + evidence(locator = locator, sourceRevision = "r1"), + evidence(locator = locator, sourceRevision = "r2"), + ) + + val view = PropositionGraphMapper.toProvenanceView(proposition(entries)) + + assertEquals(listOf(null, "r1", "r2"), view.provenance.map { it.sourceRevision }) + assertEquals(setOf(locator.key()), view.provenance.map { it.source.key }.toSet()) + assertFalse(DerivedFrom::class.java.declaredFields.any { it.name == "sourceKey" }) + assertEquals(entries, PropositionGraphMapper.toProposition(view).provenanceEntries) + } + + @Test + fun `entry key is stable and includes every evidence component`() { + val base = evidence(sourceRevision = "r1") + val stableKey = entryKey(base) + + assertEquals(stableKey, entryKey(base.copy())) + listOf( + base.copy(locator = UriLocator("https://example.com/other")), + base.copy(sourceRevision = "r2"), + base.copy(chunkId = "other-chunk"), + base.copy(startOffset = 2), + base.copy(endOffset = 4), + base.copy(contentHash = "other-hash"), + ).forEach { changed -> + assertNotEquals(stableKey, entryKey(changed)) + } + } + + @Test + fun `legacy null relationship identity maps losslessly and storage key stays out of domain json`() { + val original = evidence(sourceRevision = null) + val mapped = PropositionGraphMapper.toProvenanceView(proposition(listOf(original))) + val legacy = mapped.copy( + provenance = mapped.provenance.map { it.copy(entryKey = null) }, + ) + + assertNull(legacy.provenance.single().entryKey) + val roundTripped = PropositionGraphMapper.toProposition(legacy).provenanceEntries.single() + assertEquals(original, roundTripped) + assertFalse(jacksonObjectMapper().writeValueAsString(roundTripped).contains("entryKey")) + } + + private fun entryKey(entry: ProvenanceEntry): String = + assertNotNull( + PropositionGraphMapper.toProvenanceView(proposition(listOf(entry))) + .provenance.single().entryKey, + ) + + private fun proposition(entries: List): Proposition = + Proposition.create( + id = "proposition-1", + contextIdValue = "context-1", + text = "Mapped evidence", + mentions = emptyList(), + confidence = 1.0, + decay = 0.0, + reasoning = null, + grounding = emptyList(), + created = Instant.EPOCH, + revised = Instant.EPOCH, + status = PropositionStatus.entries.first(), + provenanceEntries = entries, + ) + + private fun evidence( + locator: UriLocator = UriLocator("https://example.com/source"), + sourceRevision: String? = "r1", + ): ProvenanceEntry = + ProvenanceEntry( + locator = locator, + chunkId = "chunk-1", + startOffset = 1, + endOffset = 3, + contentHash = "content-hash", + sourceRevision = sourceRevision, + ) +} diff --git a/dice/src/main/kotlin/com/embabel/dice/common/ConversationAnalysisRequestEvent.kt b/dice/src/main/kotlin/com/embabel/dice/common/ConversationAnalysisRequestEvent.kt index e2affcf4..a0fd7592 100644 --- a/dice/src/main/kotlin/com/embabel/dice/common/ConversationAnalysisRequestEvent.kt +++ b/dice/src/main/kotlin/com/embabel/dice/common/ConversationAnalysisRequestEvent.kt @@ -20,6 +20,8 @@ import com.embabel.chat.Conversation import com.embabel.chat.Message import com.embabel.dice.incremental.ConversationSource import com.embabel.dice.incremental.IncrementalSource +import com.embabel.dice.provenance.SourceLocator +import com.embabel.dice.provenance.SourceRevisionRef /** * Event published after a conversation exchange to trigger async proposition extraction. @@ -31,6 +33,25 @@ class ConversationAnalysisRequestEvent( @JvmField val conversation: Conversation, ) : SourceAnalysisRequestEvent(source, user) { + private var eventSourceLocator: SourceLocator? = null + + private var eventSourceRevision: SourceRevisionRef? = null + + constructor( + source: Any, + user: NamedEntity, + conversation: Conversation, + sourceLocator: SourceLocator, + sourceRevision: SourceRevisionRef? = null, + ) : this(source, user, conversation) { + eventSourceLocator = sourceLocator + eventSourceRevision = sourceRevision + } + override fun incrementalSource(): IncrementalSource = ConversationSource(conversation) + + override fun sourceLocator(): SourceLocator? = eventSourceLocator + + override fun sourceRevision(): SourceRevisionRef? = eventSourceRevision } diff --git a/dice/src/main/kotlin/com/embabel/dice/common/SourceAnalysisContext.kt b/dice/src/main/kotlin/com/embabel/dice/common/SourceAnalysisContext.kt index 5389b27c..0d4546d5 100644 --- a/dice/src/main/kotlin/com/embabel/dice/common/SourceAnalysisContext.kt +++ b/dice/src/main/kotlin/com/embabel/dice/common/SourceAnalysisContext.kt @@ -18,6 +18,7 @@ package com.embabel.dice.common import com.embabel.agent.core.ContextId import com.embabel.agent.core.DataDictionary import com.embabel.dice.provenance.SourceLocator +import com.embabel.dice.provenance.SourceRevisionRef import com.embabel.dice.proposition.extraction.ExtractionPerspective /** @@ -35,6 +36,8 @@ import com.embabel.dice.proposition.extraction.ExtractionPerspective * @param sourceLocator optional pointer to where this run's material lives. When set, the pipeline * stamps it onto every extracted proposition's provenance, so a caller that knows the real source * (a file, a URI, a connector record) gets richer grounding than the content-hash fallback. + * @param sourceRevision optional validated revision of [sourceLocator]. The locator is required and + * its source key must match before analysis can begin. * @param mintNewEntities whether a mention the resolver could NOT match to an existing entity may * be persisted as a NEW entity node. Default FALSE: unresolved mentions stay unresolved (the * proposition is still persisted; its mention simply carries no resolvedId), so extraction never @@ -60,8 +63,20 @@ data class SourceAnalysisContext @JvmOverloads constructor( * values win over extractor-supplied properties of the same key. */ val mintedEntityProperties: Map = emptyMap(), + val sourceRevision: SourceRevisionRef? = null, ) { + init { + sourceRevision?.let { revision -> + val locator = requireNotNull(sourceLocator) { + "sourceLocator is required when sourceRevision is set" + } + require(revision.sourceKey == locator.key()) { + "sourceRevision source key must match sourceLocator source key" + } + } + } + companion object { /** * Start building a SourceAnalysisContext with the given context ID. @@ -127,6 +142,12 @@ data class SourceAnalysisContext @JvmOverloads constructor( fun withSourceLocator(sourceLocator: SourceLocator): SourceAnalysisContext = copy(sourceLocator = sourceLocator) + /** + * Returns a copy carrying a validated revision of this context's source. + */ + fun withSourceRevision(sourceRevision: SourceRevisionRef): SourceAnalysisContext = + copy(sourceRevision = sourceRevision) + /** * Returns a copy allowing (or forbidding) this analysis to persist NEW entities * for mentions the resolver could not match. See [mintNewEntities]. diff --git a/dice/src/main/kotlin/com/embabel/dice/common/SourceAnalysisRequestEvent.kt b/dice/src/main/kotlin/com/embabel/dice/common/SourceAnalysisRequestEvent.kt index b4a3aac6..644f04e1 100644 --- a/dice/src/main/kotlin/com/embabel/dice/common/SourceAnalysisRequestEvent.kt +++ b/dice/src/main/kotlin/com/embabel/dice/common/SourceAnalysisRequestEvent.kt @@ -18,6 +18,8 @@ package com.embabel.dice.common import com.embabel.agent.rag.model.NamedEntity import com.embabel.chat.Message import com.embabel.dice.incremental.IncrementalSource +import com.embabel.dice.provenance.SourceLocator +import com.embabel.dice.provenance.SourceRevisionRef import org.springframework.context.ApplicationEvent /** @@ -31,4 +33,8 @@ abstract class SourceAnalysisRequestEvent( ) : ApplicationEvent(source) { abstract fun incrementalSource(): IncrementalSource + + open fun sourceLocator(): SourceLocator? = null + + open fun sourceRevision(): SourceRevisionRef? = null } diff --git a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt index b395090a..5de9b4d9 100644 --- a/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt +++ b/dice/src/main/kotlin/com/embabel/dice/pipeline/PropositionPipeline.kt @@ -341,6 +341,7 @@ class PropositionPipeline private constructor( locator = context.sourceLocator ?: ContentAddressedLocator(contentHash), chunkId = chunk.id, contentHash = contentHash, + sourceRevision = context.sourceRevision?.sourceRevision, ) return propositions.map { it.withProvenanceEntries(listOf(entry)) } } diff --git a/dice/src/main/kotlin/com/embabel/dice/projection/memory/collector/MultiSignalCollectorStrategy.kt b/dice/src/main/kotlin/com/embabel/dice/projection/memory/collector/MultiSignalCollectorStrategy.kt index be531210..6d9be433 100644 --- a/dice/src/main/kotlin/com/embabel/dice/projection/memory/collector/MultiSignalCollectorStrategy.kt +++ b/dice/src/main/kotlin/com/embabel/dice/projection/memory/collector/MultiSignalCollectorStrategy.kt @@ -19,6 +19,7 @@ import com.embabel.agent.core.ContextId import com.embabel.dice.projection.memory.RunAwareCollectorStrategy import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.provenance.ProvenanceEvidenceKey import com.embabel.dice.spi.CandidatePair import com.embabel.dice.spi.CandidatePairSource import com.embabel.dice.spi.CollectorCandidateEdge @@ -306,14 +307,24 @@ class MultiSignalCollectorStrategy( // pre-merge (common for near-duplicates from the same source) must not be recorded as // "folded", or undo would later strip evidence the survivor held independently. val survivorGrounding = survivor.grounding.toSet() - val survivorProvenanceRefs = survivor.provenanceEntries.map { it.locator.key() }.toSet() + val survivorProvenanceRefs = survivor.provenanceEntries + .map { it.locator.key() } + .toSet() + val survivorProvenanceEvidenceKeys = survivor.provenanceEntries + .map(ProvenanceEvidenceKey::encode) + .toSet() val survivorSourceIds = survivor.sourceIds.toSet() val retired = losers.map { loser -> RetiredProposition( propositionId = loser.id, priorStatus = loser.status, foldedGrounding = loser.grounding - survivorGrounding, - foldedProvenanceRefs = loser.provenanceEntries.map { it.locator.key() } - survivorProvenanceRefs, + foldedProvenanceRefs = loser.provenanceEntries + .map { it.locator.key() } + .distinct() - survivorProvenanceRefs, + foldedProvenanceEvidenceKeys = loser.provenanceEntries + .map(ProvenanceEvidenceKey::encode) + .distinct() - survivorProvenanceEvidenceKeys, foldedSourceIds = loser.sourceIds - survivorSourceIds, ) } diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/Proposition.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/Proposition.kt index ec4a6926..66df0bc7 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/Proposition.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/Proposition.kt @@ -20,6 +20,7 @@ import com.embabel.agent.core.ContextId import com.embabel.agent.rag.model.Retrievable import com.embabel.common.core.types.ZeroToOne import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.ProvenanceEvidenceKey import com.embabel.dice.temporal.TemporalMetadata import org.jetbrains.annotations.ApiStatus import java.time.Instant @@ -321,9 +322,9 @@ data class Proposition( /** * Create a copy with exactly [groundingToRemove], [provenanceRefsToRemove] and * [sourceIdsToRemove] taken out of this proposition's evidence — the inverse of - * [absorbEvidence] for one loser's contribution. Provenance entries are matched by - * [com.embabel.dice.provenance.SourceLocator.key], the same identity [absorbEvidence]'s - * caller uses to record a loser's folded refs (see `RetiredProposition.foldedProvenanceRefs`). + * [absorbEvidence] for one loser's contribution. Provenance refs use the shared evidence-key + * contract so current refs identify one full entry while legacy locator refs match only + * revisionless evidence. * * Callers subtracting one collapsed member's evidence from a survivor that absorbed several * members must first drop any ref another still-retired member also contributed, or this @@ -339,7 +340,9 @@ data class Proposition( val provenanceRefSet = provenanceRefsToRemove.toSet() return copy( grounding = grounding - groundingToRemove.toSet(), - provenanceEntries = provenanceEntries.filterNot { it.locator.key() in provenanceRefSet }, + provenanceEntries = provenanceEntries.filterNot { entry -> + provenanceRefSet.any { ProvenanceEvidenceKey.matches(entry, it) } + }, sourceIds = sourceIds - sourceIdsToRemove.toSet(), metadataRevised = Instant.now(), ) diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionRepository.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionRepository.kt index f33f0a37..c7d29ef2 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionRepository.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionRepository.kt @@ -15,12 +15,15 @@ */ package com.embabel.dice.proposition +import com.embabel.agent.core.ContextId import com.embabel.agent.rag.model.Retrievable import com.embabel.agent.rag.service.CoreSearchOperations import com.embabel.common.core.types.SimilarityResult import com.embabel.common.core.types.TextSimilaritySearchRequest import com.embabel.common.util.loggerFor import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.SourceLocator +import com.embabel.dice.provenance.SourceRevisionRef /** * The full proposition repository: combines the base persistence port with every opt-in @@ -73,6 +76,75 @@ interface PropositionRepository : */ fun findAll(withProvenance: Boolean): List = findAll() + // ======================================================================== + // Source provenance queries + // + // Defaults stay context-scoped and inspect loaded provenance in memory. Backends whose normal + // context reads omit or abbreviate provenance must override the typed methods and execute the + // predicates against their authoritative provenance representation. + // ======================================================================== + + /** + * Find propositions in [contextId] with evidence from any revision of [sourceKey]. + * + * Both revisioned and revisionless evidence matches when its locator key equals [sourceKey]. + * Backends with lean provenance reads must override this typed method. + */ + fun findBySourceKey(contextId: ContextId, sourceKey: String): List = + findByContextId(contextId).filter { proposition -> + proposition.provenanceEntries.any { it.locator.key() == sourceKey } + } + + /** + * Java-friendly bridge to [findBySourceKey]. + */ + fun findBySourceKey(contextIdValue: String, sourceKey: String): List = + findBySourceKey(ContextId(contextIdValue), sourceKey) + + /** + * Find propositions in [contextId] with evidence from exactly [ref]'s source key and revision. + * + * Backends with lean provenance reads must override this typed method. + */ + fun findBySourceRevision(contextId: ContextId, ref: SourceRevisionRef): List = + findByContextId(contextId).filter { proposition -> + proposition.provenanceEntries.any { + it.locator.key() == ref.sourceKey && it.sourceRevision == ref.sourceRevision + } + } + + /** + * Java-friendly bridge to [findBySourceRevision]. + */ + fun findBySourceRevision(contextIdValue: String, ref: SourceRevisionRef): List = + findBySourceRevision(ContextId(contextIdValue), ref) + + /** + * Find propositions in [contextId] with revisionless evidence whose locator key equals + * [locator]'s key. + * + * Evidence carrying any revision does not match. Backends with lean provenance reads must + * override this typed method. + */ + fun findRevisionlessBySourceLocator( + contextId: ContextId, + locator: SourceLocator, + ): List = + findByContextId(contextId).filter { proposition -> + proposition.provenanceEntries.any { + it.locator.key() == locator.key() && it.sourceRevision == null + } + } + + /** + * Java-friendly bridge to [findRevisionlessBySourceLocator]. + */ + fun findRevisionlessBySourceLocator( + contextIdValue: String, + locator: SourceLocator, + ): List = + findRevisionlessBySourceLocator(ContextId(contextIdValue), locator) + // ======================================================================== // Administrative operations - bulk re-embed and coarse deletion // @@ -141,31 +213,31 @@ interface PropositionRepository : /** * The provenance entries of a proposition, or an empty list if it has none or does not exist. */ - fun provenanceOf(propositionId: String): List = - findById(propositionId)?.provenanceEntries ?: emptyList() + override fun provenanceOf(propositionId: String): List = + super.provenanceOf(propositionId) /** * Append provenance to a proposition (deduplicated); never removes existing entries. * * @return the updated proposition, or null if no proposition with that id exists. */ - fun addProvenance(propositionId: String, entries: List): Proposition? = - findById(propositionId)?.let { save(it.withProvenanceEntries(entries)) } + override fun addProvenance(propositionId: String, entries: List): Proposition? = + super.addProvenance(propositionId, entries) /** * Authoritatively set a proposition's provenance to exactly [entries], removing any not listed. * * @return the updated proposition, or null if no proposition with that id exists. */ - fun setProvenance(propositionId: String, entries: List): Proposition? = - findById(propositionId)?.let { save(it.withProvenance(entries)) } + override fun setProvenance(propositionId: String, entries: List): Proposition? = + super.setProvenance(propositionId, entries) /** * Remove all provenance from a proposition. * * @return the updated proposition, or null if no proposition with that id exists. */ - fun clearProvenance(propositionId: String): Proposition? = + override fun clearProvenance(propositionId: String): Proposition? = setProvenance(propositionId, emptyList()) // RAG VectorSearch bridge — only Proposition is supported diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionStore.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionStore.kt index 3ed4c503..aaacf183 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionStore.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/PropositionStore.kt @@ -18,6 +18,7 @@ package com.embabel.dice.proposition import com.embabel.agent.core.ContextId import com.embabel.agent.rag.service.RetrievableIdentifier import com.embabel.dice.common.DiceMetadataKeys +import com.embabel.dice.provenance.ProvenanceEntry import org.slf4j.LoggerFactory import java.time.Instant @@ -100,6 +101,10 @@ interface PropositionStore { /** * Save a proposition. If a proposition with the same ID exists, it will be replaced. + * + * Transaction-aware implementations participate in an active caller transaction, so a caller + * rollback also rolls back a completed save. Implementations may own a transaction when no + * caller transaction exists. */ fun save(proposition: Proposition): Proposition @@ -175,6 +180,44 @@ interface PropositionStore { */ fun count(): Int + // ======================================================================== + // Provenance management — authoritative evidence replacement + // ======================================================================== + + /** + * The provenance entries of a proposition, or an empty list if it has none or does not exist. + */ + fun provenanceOf(propositionId: String): List = + findById(propositionId)?.provenanceEntries ?: emptyList() + + /** + * Append provenance to a proposition (deduplicated); never removes existing entries. + * + * @return the updated proposition, or null if no proposition with that id exists. + */ + fun addProvenance(propositionId: String, entries: List): Proposition? = + findById(propositionId)?.let { save(it.withProvenanceEntries(entries)) } + + /** + * Authoritatively set a proposition's provenance to exactly [entries], removing any not listed. + * + * Backends whose normal [save] path preserves unloaded provenance must override this operation. + * Making the capability part of the base store contract lets evidence-sensitive operations such + * as collector undo depend on it explicitly instead of probing for a richer repository type. + * + * @return the updated proposition, or null if no proposition with that id exists. + */ + fun setProvenance(propositionId: String, entries: List): Proposition? = + findById(propositionId)?.let { save(it.withProvenance(entries)) } + + /** + * Remove all provenance from a proposition. + * + * @return the updated proposition, or null if no proposition with that id exists. + */ + fun clearProvenance(propositionId: String): Proposition? = + setProvenance(propositionId, emptyList()) + /** * Refresh [Proposition.lastAccessed] to now for [ids] — the read-side reinforcement that lets a * DECAYING proposition's decay anchor (see [Proposition.effectiveConfidenceAt]) track actual use diff --git a/dice/src/main/kotlin/com/embabel/dice/proposition/extraction/IncrementalPropositionExtraction.kt b/dice/src/main/kotlin/com/embabel/dice/proposition/extraction/IncrementalPropositionExtraction.kt index b57833cb..366e2dc9 100644 --- a/dice/src/main/kotlin/com/embabel/dice/proposition/extraction/IncrementalPropositionExtraction.kt +++ b/dice/src/main/kotlin/com/embabel/dice/proposition/extraction/IncrementalPropositionExtraction.kt @@ -34,6 +34,8 @@ import com.embabel.dice.pipeline.ChunkPropositionResult import com.embabel.dice.pipeline.PropositionPipeline import com.embabel.dice.projection.graph.GraphProjectionService import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.provenance.SourceLocator +import com.embabel.dice.provenance.SourceRevisionRef import org.slf4j.LoggerFactory import org.springframework.context.event.EventListener import org.springframework.scheduling.annotation.Async @@ -162,7 +164,48 @@ open class IncrementalPropositionExtraction @JvmOverloads constructor( * Extract propositions from a file via Tika and persist them. * Requires `embabel-agent-rag-tika` on the classpath. */ - open fun rememberFile(inputStream: InputStream, filename: String, user: NamedEntity) { + open fun rememberFile(inputStream: InputStream, filename: String, user: NamedEntity) = + withRememberedFileText(inputStream, filename) { text -> + rememberText( + text = text, + sourceId = "remember:$filename", + user = user, + ) + } + + /** + * Extract propositions from a file and ground them in the caller's typed source. + * + * A non-null [sourceRevision] is a host assertion that the locator's revision covers + * the full extracted file aggregate. DICE cannot infer aggregate revision coverage. + */ + @JvmOverloads + open fun rememberFileFromSource( + inputStream: InputStream, + filename: String, + user: NamedEntity, + sourceLocator: SourceLocator, + sourceRevision: SourceRevisionRef? = null, + ): Unit { + require(sourceRevision == null || sourceRevision.sourceKey == sourceLocator.key()) { + "sourceRevision source key must match sourceLocator source key" + } + withRememberedFileText(inputStream, filename) { text -> + rememberTextFromSource( + text = text, + sourceId = "remember:$filename", + user = user, + sourceLocator = sourceLocator, + sourceRevision = sourceRevision, + ) + } + } + + private fun withRememberedFileText( + inputStream: InputStream, + filename: String, + remember: (String) -> Unit, + ) { try { val reader = com.embabel.agent.rag.ingestion.TikaHierarchicalContentReader() val document = reader.parseContent(inputStream, "remember://$filename") @@ -173,7 +216,7 @@ open class IncrementalPropositionExtraction @JvmOverloads constructor( return } - rememberText(text, "remember:$filename", user) + remember(text) } catch (e: Exception) { logger.warn("Failed to learn file: {}", filename, e) } @@ -204,8 +247,64 @@ open class IncrementalPropositionExtraction @JvmOverloads constructor( additionalGrounding: List = emptyList(), perspective: ExtractionPerspective? = null, mintNewEntities: Boolean? = null, + ) = + rememberTextInternal( + text = text, + sourceId = sourceId, + user = user, + additionalGrounding = additionalGrounding, + perspective = perspective, + mintNewEntities = mintNewEntities, + ) + + /** + * Extract propositions from raw text and ground them in the caller's typed source. + * + * [sourceId] remains the exact caller-supplied chunk/grounding identifier. A non-null + * [sourceRevision] is a host assertion that [sourceLocator]'s revision covers the full + * text aggregate; DICE cannot derive revision coverage from untyped identifiers or + * [additionalGrounding]. + */ + @JvmOverloads + open fun rememberTextFromSource( + text: String, + sourceId: String, + user: NamedEntity, + sourceLocator: SourceLocator, + sourceRevision: SourceRevisionRef? = null, + additionalGrounding: List = emptyList(), + perspective: ExtractionPerspective? = null, + mintNewEntities: Boolean? = null, + ): Unit = + rememberTextInternal( + text = text, + sourceId = sourceId, + user = user, + sourceLocator = sourceLocator, + sourceRevision = sourceRevision, + additionalGrounding = additionalGrounding, + perspective = perspective, + mintNewEntities = mintNewEntities, + ) + + private fun rememberTextInternal( + text: String, + sourceId: String, + user: NamedEntity, + sourceLocator: SourceLocator? = null, + sourceRevision: SourceRevisionRef? = null, + additionalGrounding: List = emptyList(), + perspective: ExtractionPerspective? = null, + mintNewEntities: Boolean? = null, ) { - val context = buildContext(user, sourceId, perspective, mintNewEntities) + val context = buildContext( + user = user, + sourceId = sourceId, + perspective = perspective, + mintNewEntities = mintNewEntities, + sourceLocator = sourceLocator, + sourceRevision = sourceRevision, + ) val result = propositionPipeline.processOnce( text, sourceId, context, additionalGrounding = additionalGrounding, ) @@ -252,7 +351,12 @@ open class IncrementalPropositionExtraction @JvmOverloads constructor( return } - val context = buildContext(event.user, source.id) + val context = buildContext( + user = event.user, + sourceId = source.id, + sourceLocator = event.sourceLocator(), + sourceRevision = event.sourceRevision(), + ) logger.info( "Context relations count: {}, injected relations count: {}", context.relations.size(), relations.size(), @@ -283,6 +387,8 @@ open class IncrementalPropositionExtraction @JvmOverloads constructor( sourceId: String = "", perspective: ExtractionPerspective? = null, mintNewEntities: Boolean? = null, + sourceLocator: SourceLocator? = null, + sourceRevision: SourceRevisionRef? = null, ): SourceAnalysisContext { val aliases = try { currentUserAliasesProvider(user) @@ -342,6 +448,12 @@ open class IncrementalPropositionExtraction @JvmOverloads constructor( ctx = ctx.withMintedEntityProperties(stamped) } } + if (sourceLocator != null) { + ctx = ctx.withSourceLocator(sourceLocator) + } + if (sourceRevision != null) { + ctx = ctx.withSourceRevision(sourceRevision) + } return ctx } diff --git a/dice/src/main/kotlin/com/embabel/dice/provenance/ProvenanceEntry.kt b/dice/src/main/kotlin/com/embabel/dice/provenance/ProvenanceEntry.kt index 5d20dc92..6fc713aa 100644 --- a/dice/src/main/kotlin/com/embabel/dice/provenance/ProvenanceEntry.kt +++ b/dice/src/main/kotlin/com/embabel/dice/provenance/ProvenanceEntry.kt @@ -33,6 +33,7 @@ package com.embabel.dice.provenance * @property endOffset Optional exclusive end character offset within the source/chunk * @property contentHash Optional hash of the source content. Comparing it against the * source later can reveal that the source has since changed. + * @property sourceRevision Optional provider-defined opaque revision of the source */ data class ProvenanceEntry @JvmOverloads constructor( val locator: SourceLocator, @@ -40,6 +41,7 @@ data class ProvenanceEntry @JvmOverloads constructor( val startOffset: Int? = null, val endOffset: Int? = null, val contentHash: String? = null, + val sourceRevision: String? = null, ) { init { @@ -48,5 +50,8 @@ data class ProvenanceEntry @JvmOverloads constructor( require( startOffset == null || endOffset == null || endOffset >= startOffset ) { "endOffset must be >= startOffset" } + require(sourceRevision == null || sourceRevision.isNotBlank()) { + "sourceRevision must not be blank" + } } } diff --git a/dice/src/main/kotlin/com/embabel/dice/provenance/ProvenanceEvidenceKey.kt b/dice/src/main/kotlin/com/embabel/dice/provenance/ProvenanceEvidenceKey.kt new file mode 100644 index 00000000..130d509e --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/provenance/ProvenanceEvidenceKey.kt @@ -0,0 +1,121 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.provenance + +/** + * Owns the stable evidence identity shared by provenance producers and consumers. + * + * Values are length-framed in a fixed field order. A length of `-1` represents null, + * keeping absence distinct from every string value without escaping delimiters. + */ +internal object ProvenanceEvidenceKey { + + private const val MAGIC_PREFIX = "dice-provenance:" + private const val VERSION_PREFIX = "${MAGIC_PREFIX}v1:" + + fun encode(entry: ProvenanceEntry): String = + buildString(VERSION_PREFIX.length + 64) { + append(VERSION_PREFIX) + appendFrame(entry.locator.key()) + appendFrame(entry.sourceRevision) + appendFrame(entry.chunkId) + appendFrame(entry.startOffset?.toString()) + appendFrame(entry.endOffset?.toString()) + appendFrame(entry.contentHash) + } + + fun matches(entry: ProvenanceEntry, encoded: String): Boolean { + if (!encoded.startsWith(MAGIC_PREFIX)) { + return entry.sourceRevision == null && encoded == entry.locator.key() + } + if (!encoded.startsWith(VERSION_PREFIX)) { + return false + } + + val matcher = FrameMatcher(encoded, VERSION_PREFIX.length) + return matcher.matches(entry.locator.key()) && + matcher.matches(entry.sourceRevision) && + matcher.matches(entry.chunkId) && + matcher.matches(entry.startOffset?.toString()) && + matcher.matches(entry.endOffset?.toString()) && + matcher.matches(entry.contentHash) && + matcher.isExhausted() + } + + private fun StringBuilder.appendFrame(value: String?) { + if (value == null) { + append("-1:") + } else { + append(value.length) + append(':') + append(value) + } + } + + private class FrameMatcher( + private val encoded: String, + private var offset: Int, + ) { + + fun matches(value: String?): Boolean { + val separator = encoded.indexOf(':', offset) + if (separator < 0) { + return false + } + val length = parseLength(separator) ?: return false + offset = separator + 1 + if (length == -1) { + return value == null + } + if (value == null || length != value.length || length > encoded.length - offset) { + return false + } + if (!encoded.regionMatches(offset, value, 0, length)) { + return false + } + offset += length + return true + } + + fun isExhausted(): Boolean = offset == encoded.length + + private fun parseLength(separator: Int): Int? { + if (separator == offset) { + return null + } + if (encoded[offset] == '-') { + return if (separator == offset + 2 && encoded[offset + 1] == '1') -1 else null + } + if (separator > offset + 1 && encoded[offset] == '0') { + return null + } + + var length = 0 + for (index in offset until separator) { + val character = encoded[index] + if (character !in '0'..'9') { + return null + } + val digit = character - '0' + if (length > (Int.MAX_VALUE - digit) / 10) { + return null + } + length = length * 10 + digit + } + return length + } + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/provenance/SourceRevisionRef.kt b/dice/src/main/kotlin/com/embabel/dice/provenance/SourceRevisionRef.kt new file mode 100644 index 00000000..45d455e2 --- /dev/null +++ b/dice/src/main/kotlin/com/embabel/dice/provenance/SourceRevisionRef.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.provenance + +/** + * Identifies an opaque revision of a source. + * + * @property sourceKey Canonical source identity produced by [SourceLocator.key] + * @property sourceRevision Provider-defined opaque revision value + */ +data class SourceRevisionRef( + val sourceKey: String, + val sourceRevision: String, +) { + + init { + require(sourceKey.isNotBlank()) { "sourceKey must not be blank" } + require(sourceRevision.isNotBlank()) { "sourceRevision must not be blank" } + } +} diff --git a/dice/src/main/kotlin/com/embabel/dice/query/discovery/DiscoveryDtos.kt b/dice/src/main/kotlin/com/embabel/dice/query/discovery/DiscoveryDtos.kt index ff5d88d0..1cab1afc 100644 --- a/dice/src/main/kotlin/com/embabel/dice/query/discovery/DiscoveryDtos.kt +++ b/dice/src/main/kotlin/com/embabel/dice/query/discovery/DiscoveryDtos.kt @@ -20,9 +20,11 @@ import com.embabel.dice.projection.lineage.ProjectionRecord import com.embabel.dice.projection.memory.CollectorRunResult import com.embabel.dice.proposition.EntityMention import com.embabel.dice.proposition.Proposition +import com.embabel.dice.provenance.ProvenanceEntry import com.embabel.dice.query.graph.GraphNeighborhood import com.embabel.dice.query.graph.GraphPath import com.embabel.dice.query.graph.PropositionLineage +import com.fasterxml.jackson.annotation.JsonInclude /** * Outward-facing discovery DTOs — the trust boundary between domain internals and external callers. @@ -132,6 +134,31 @@ data class NeighborhoodDto( } } +/** + * Primitive-only provenance attached to a discovery explanation. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +data class DiscoveryProvenanceDto( + val locator: String, + val sourceRevision: String? = null, + val chunkId: String? = null, + val startOffset: Int? = null, + val endOffset: Int? = null, + val contentHash: String? = null, +) { + companion object { + @JvmStatic + fun from(entry: ProvenanceEntry): DiscoveryProvenanceDto = DiscoveryProvenanceDto( + locator = entry.locator.key(), + sourceRevision = entry.sourceRevision, + chunkId = entry.chunkId, + startOffset = entry.startOffset, + endOffset = entry.endOffset, + contentHash = entry.contentHash, + ) + } +} + /** * A leak-free lineage summary — the "why" behind a stored fact. * @@ -141,14 +168,16 @@ data class NeighborhoodDto( * @property reinforceCount how many times the proposition has been reinforced * @property groundingChunkIds the grounding chunk ids as opaque strings * @property sourceSummaries the source proposition statements this one was abstracted from + * @property provenance the proposition's ordered source evidence */ -data class LineageDto( +data class LineageDto @JvmOverloads constructor( val propositionId: String, val text: String, val status: String, val reinforceCount: Int, val groundingChunkIds: List, val sourceSummaries: List, + val provenance: List = emptyList(), ) { companion object { @JvmStatic @@ -159,6 +188,7 @@ data class LineageDto( reinforceCount = lineage.reinforceCount, groundingChunkIds = lineage.groundingChunkIds, sourceSummaries = lineage.sources.map { it.text }, + provenance = lineage.provenanceEntries.map { DiscoveryProvenanceDto.from(it) }, ) } } diff --git a/dice/src/main/kotlin/com/embabel/dice/spi/CollectorSignals.kt b/dice/src/main/kotlin/com/embabel/dice/spi/CollectorSignals.kt index 424e81d5..8d30158a 100644 --- a/dice/src/main/kotlin/com/embabel/dice/spi/CollectorSignals.kt +++ b/dice/src/main/kotlin/com/embabel/dice/spi/CollectorSignals.kt @@ -19,6 +19,7 @@ import com.embabel.agent.core.ContextId import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionStatus import com.embabel.dice.proposition.PropositionStore +import com.fasterxml.jackson.annotation.JsonIgnore /** * One proposed pair of propositions worth scoring. @@ -99,13 +100,20 @@ data class CollectorDecision( /** * One proposition that was folded into a survivor, and what a merging sweep would carry over * from it (grounding, provenance and source ids) so the fold can be undone. + * + * @property foldedProvenanceRefs Stable locator keys retained for compatibility with trace + * consumers that display or compare source references. + * @property foldedProvenanceEvidenceKeys Opaque full-evidence identities used for precise undo. + * Empty on traces written before revision-aware evidence identity was introduced. */ -data class RetiredProposition( +data class RetiredProposition @JvmOverloads constructor( val propositionId: String, val priorStatus: PropositionStatus, val foldedGrounding: List = emptyList(), val foldedProvenanceRefs: List = emptyList(), val foldedSourceIds: List = emptyList(), + @get:JsonIgnore + val foldedProvenanceEvidenceKeys: List = emptyList(), ) /** @@ -191,29 +199,41 @@ fun undoSingleCollapse( "Proposition $retiredId was retired into survivor ${decision.survivorId}, not $survivorId" } val retirement = decision.retired.firstOrNull { it.propositionId == retiredId } ?: return null + val survivor = propositions.findById(survivorId) ?: return null + val retiredProposition = propositions.findById(retiredId) ?: return null // Other members of this same collapse: whatever they also folded must stay on the survivor // even though we're subtracting retirement's copy of it. val others = decision.retired.filter { it.propositionId != retiredId } val stillNeededGrounding = others.flatMap { it.foldedGrounding }.toSet() - val stillNeededProvenanceRefs = others.flatMap { it.foldedProvenanceRefs }.toSet() + val stillNeededProvenanceRefs = others.flatMap { it.provenanceEvidenceKeysForUndo() }.toSet() val stillNeededSourceIds = others.flatMap { it.foldedSourceIds }.toSet() - val survivor = propositions.findById(survivorId) ?: return null - val updatedSurvivor = propositions.save( - survivor.withoutFoldedEvidence( - groundingToRemove = retirement.foldedGrounding.filterNot { it in stillNeededGrounding }, - provenanceRefsToRemove = retirement.foldedProvenanceRefs.filterNot { it in stillNeededProvenanceRefs }, - sourceIdsToRemove = retirement.foldedSourceIds.filterNot { it in stillNeededSourceIds }, - ), + val survivorWithoutFoldedEvidence = survivor.withoutFoldedEvidence( + groundingToRemove = retirement.foldedGrounding.filterNot { it in stillNeededGrounding }, + provenanceRefsToRemove = retirement.provenanceEvidenceKeysForUndo() + .filterNot { it in stillNeededProvenanceRefs }, + sourceIdsToRemove = retirement.foldedSourceIds.filterNot { it in stillNeededSourceIds }, ) + val savedSurvivor = propositions.save(survivorWithoutFoldedEvidence) + val updatedSurvivor = propositions.setProvenance( + survivorWithoutFoldedEvidence.id, + survivorWithoutFoldedEvidence.provenanceEntries, + ) ?: savedSurvivor - val retiredProposition = propositions.findById(retiredId) ?: return null val restored = propositions.save(retiredProposition.withStatus(retirement.priorStatus)) return CollapseUndoResult(survivor = updatedSurvivor, restored = restored) } +/** + * New traces carry exact evidence identities separately from their stable, human-readable source + * references. Falling back to the old field keeps both locator-only legacy traces and traces from + * the short-lived versioned-ref format readable. + */ +private fun RetiredProposition.provenanceEvidenceKeysForUndo(): List = + foldedProvenanceEvidenceKeys.ifEmpty { foldedProvenanceRefs } + /** * Groups proposition ids into connected components from scored, non-vetoed edges. */ diff --git a/dice/src/main/kotlin/com/embabel/dice/web/rest/MemoryDtos.kt b/dice/src/main/kotlin/com/embabel/dice/web/rest/MemoryDtos.kt index c0434240..873d764c 100644 --- a/dice/src/main/kotlin/com/embabel/dice/web/rest/MemoryDtos.kt +++ b/dice/src/main/kotlin/com/embabel/dice/web/rest/MemoryDtos.kt @@ -37,12 +37,21 @@ import java.time.Instant * @param schemaName Optional schema name for extraction. Uses default if not specified. * @param options Extraction options */ -data class ExtractRequest( +data class ExtractRequest @JvmOverloads constructor( val text: String, val sourceId: String? = null, val knownEntities: List = emptyList(), val schemaName: String? = null, val options: ExtractOptions = ExtractOptions(), + val sourceLocator: SourceLocatorInputDto? = null, + val sourceRevision: String? = null, +) + +data class SourceLocatorInputDto( + val kind: String, + val value: String, + val connectorId: String? = null, + val display: String? = null, ) data class ExtractOptions( @@ -232,15 +241,17 @@ data class EntityMentionDto( * @property startOffset character offset where the supporting span begins, when known * @property endOffset character offset where the supporting span ends, when known * @property contentHash hash of the source content, when known + * @property sourceRevision opaque revision token for this source, when known */ @JsonInclude(JsonInclude.Include.NON_NULL) -data class ProvenanceEntryDto( +data class ProvenanceEntryDto @JvmOverloads constructor( val locator: String, val display: String?, val chunkId: String?, val startOffset: Int?, val endOffset: Int?, val contentHash: String?, + val sourceRevision: String? = null, ) { companion object { fun from(entry: ProvenanceEntry): ProvenanceEntryDto = ProvenanceEntryDto( @@ -250,6 +261,7 @@ data class ProvenanceEntryDto( startOffset = entry.startOffset, endOffset = entry.endOffset, contentHash = entry.contentHash, + sourceRevision = entry.sourceRevision, ) } } diff --git a/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt b/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt index a129e153..bce32ed7 100644 --- a/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt +++ b/dice/src/main/kotlin/com/embabel/dice/web/rest/PropositionPipelineController.kt @@ -29,10 +29,18 @@ import com.embabel.dice.common.KnownEntity import com.embabel.dice.common.NewEntity import com.embabel.dice.common.SchemaRegistry import com.embabel.dice.common.SourceAnalysisContext +import com.embabel.dice.common.support.Sha256ContentHasher import com.embabel.dice.pipeline.ChunkPropositionResult import com.embabel.dice.pipeline.PropositionPipeline import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.proposition.revision.RevisionResult +import com.embabel.dice.provenance.ConnectorRef +import com.embabel.dice.provenance.ContentAddressedLocator +import com.embabel.dice.provenance.FileLocator +import com.embabel.dice.provenance.SourceLocator +import com.embabel.dice.provenance.SourceRevisionRef +import com.embabel.dice.provenance.UriLocator +import com.fasterxml.jackson.core.JsonProcessingException import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue @@ -91,12 +99,28 @@ class PropositionPipelineController( return ResponseEntity.badRequest().build() } - val chunk = Chunk.create( - text = request.text, - parentId = request.sourceId ?: "api-request", + val sourceProvenance = try { + resolveSourceProvenance(request.sourceLocator, request.sourceRevision) + } catch (e: IllegalArgumentException) { + logger.warn("Rejecting extract request for context {}: {}", contextId, e.message) + return ResponseEntity.badRequest().build() + } + val context = buildContext( + contextId = contextId, + knownEntityDtos = request.knownEntities, + schemaName = request.schemaName, + sourceLocator = sourceProvenance.first, + sourceRevision = sourceProvenance.second, ) - val context = buildContext(contextId, request.knownEntities, request.schemaName) + val chunk = revisionStableChunk( + chunk = Chunk.create( + text = request.text, + parentId = request.sourceId ?: "api-request", + ), + sourceRevision = sourceProvenance.second, + ordinal = 0, + ) val result = propositionPipeline.processChunk(chunk, context) // Persist what revision says to keep — both the freshly extracted propositions and any @@ -122,10 +146,22 @@ class PropositionPipelineController( @RequestPart("sourceId", required = false) sourceId: String?, @RequestPart("knownEntities", required = false) knownEntitiesJson: String?, @RequestPart("schemaName", required = false) schemaName: String?, + @RequestPart("sourceLocator", required = false) sourceLocatorJson: String?, + @RequestPart("sourceRevision", required = false) sourceRevision: String?, ): ResponseEntity { val filename = file.originalFilename ?: "uploaded-file" logger.info("Extracting propositions from file '{}' for context: {}", filename, contextId) + val sourceProvenance = try { + resolveSourceProvenance(parseSourceLocator(sourceLocatorJson), sourceRevision) + } catch (e: JsonProcessingException) { + logger.warn("Rejecting file extract request for context {}: invalid sourceLocator JSON", contextId) + return ResponseEntity.badRequest().build() + } catch (e: IllegalArgumentException) { + logger.warn("Rejecting file extract request for context {}: {}", contextId, e.message) + return ResponseEntity.badRequest().build() + } + // Parse file content using Tika val document = file.inputStream.use { inputStream -> contentReader.parseContent(inputStream, sourceId ?: filename) @@ -134,7 +170,11 @@ class PropositionPipelineController( logger.info("Parsed document '{}' with {} sections", document.title, document.leaves().count()) // Chunk the document - val chunks = contentChunker.chunk(document).toList() + val chunks = contentChunker.chunk(document) + .mapIndexed { ordinal, chunk -> + revisionStableChunk(chunk, sourceProvenance.second, ordinal) + } + .toList() logger.info("Created {} chunks from document", chunks.size) if (chunks.isEmpty()) { @@ -155,7 +195,13 @@ class PropositionPipelineController( // the whole upload), shares entity identity across chunks, and is the only path that honors // the configured extraction execution strategy (Serial/Parallel/Batched). processChunk(), by // contrast, propagates failures and runs one chunk in isolation. - val context = buildContext(contextId, parseKnownEntities(knownEntitiesJson), schemaName) + val context = buildContext( + contextId = contextId, + knownEntityDtos = parseKnownEntities(knownEntitiesJson), + schemaName = schemaName, + sourceLocator = sourceProvenance.first, + sourceRevision = sourceProvenance.second, + ) val processResult = propositionPipeline.process(chunks, context) val chunkResults = processResult.chunkResults @@ -238,10 +284,50 @@ class PropositionPipelineController( private fun parseKnownEntities(json: String?): List = if (json.isNullOrBlank()) emptyList() else objectMapper.readValue(json) + private fun parseSourceLocator(json: String?): SourceLocatorInputDto? = + if (json.isNullOrBlank()) null else objectMapper.readValue(json) + + private fun resolveSourceProvenance( + sourceLocatorInput: SourceLocatorInputDto?, + revision: String?, + ): Pair { + require(sourceLocatorInput != null || revision == null) { + "sourceRevision requires sourceLocator" + } + val sourceLocator = sourceLocatorInput?.toSourceLocator() + val sourceRevision = revision?.let { + SourceRevisionRef(sourceLocator!!.key(), it) + } + return sourceLocator to sourceRevision + } + + private fun revisionStableChunk( + chunk: Chunk, + sourceRevision: SourceRevisionRef?, + ordinal: Int, + ): Chunk { + if (sourceRevision == null) return chunk + val identityMaterial = listOf( + sourceRevision.sourceKey, + sourceRevision.sourceRevision, + ordinal.toString(), + chunk.text, + ).joinToString(separator = "") { value -> "${value.length}:$value" } + return Chunk.create( + id = "source-revision:${Sha256ContentHasher.hash(identityMaterial)}", + text = chunk.text, + urtext = chunk.urtext, + parentId = chunk.parentId, + metadata = chunk.metadata, + ) + } + private fun buildContext( contextId: String, knownEntityDtos: List, schemaName: String? = null, + sourceLocator: SourceLocator? = null, + sourceRevision: SourceRevisionRef? = null, ): SourceAnalysisContext { val knownEntities = knownEntityDtos.map { dto -> val entity = SimpleNamedEntityData( @@ -256,12 +342,20 @@ class PropositionPipelineController( val schema = schemaRegistry.getOrDefault(schemaName) - return SourceAnalysisContext( + var context = SourceAnalysisContext( schema = schema, entityResolver = entityResolver, contextId = ContextId(contextId), knownEntities = knownEntities, ) + + if (sourceLocator != null) { + context = context.withSourceLocator(sourceLocator) + } + if (sourceRevision != null) { + context = context.withSourceRevision(sourceRevision) + } + return context } private fun buildExtractResponse( @@ -330,4 +424,34 @@ class PropositionPipelineController( revision = revisionSummary, ) } + + private fun SourceLocatorInputDto.toSourceLocator(): SourceLocator { + require(value.isNotBlank()) { "sourceLocator.value must not be blank" } + return when (kind) { + "uri" -> { + require(connectorId == null) { "uri sourceLocator must not set connectorId" } + UriLocator(value, display) + } + + "file" -> { + require(connectorId == null) { "file sourceLocator must not set connectorId" } + FileLocator(value, display) + } + + "content" -> { + require(connectorId == null) { "content sourceLocator must not set connectorId" } + ContentAddressedLocator(value, display) + } + + "connector" -> { + require(!connectorId.isNullOrBlank()) { "connector sourceLocator requires connectorId" } + require(':' !in connectorId) { + "connector sourceLocator connectorId must not contain ':'" + } + ConnectorRef(connectorId, value, display) + } + + else -> throw IllegalArgumentException("Unsupported sourceLocator.kind: $kind") + } + } } diff --git a/dice/src/test/java/com/embabel/dice/SourceRevisionJavaInteropTest.java b/dice/src/test/java/com/embabel/dice/SourceRevisionJavaInteropTest.java new file mode 100644 index 00000000..cbf96c4a --- /dev/null +++ b/dice/src/test/java/com/embabel/dice/SourceRevisionJavaInteropTest.java @@ -0,0 +1,268 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice; + +import com.embabel.agent.rag.model.NamedEntity; +import com.embabel.chat.Message; +import com.embabel.dice.common.SourceAnalysisRequestEvent; +import com.embabel.dice.incremental.IncrementalSource; +import com.embabel.dice.proposition.extraction.ExtractionPerspective; +import com.embabel.dice.proposition.extraction.IncrementalPropositionExtraction; +import com.embabel.dice.provenance.ContentAddressedLocator; +import com.embabel.dice.provenance.ProvenanceEntry; +import com.embabel.dice.provenance.SourceLocator; +import com.embabel.dice.provenance.SourceRevisionRef; +import com.embabel.dice.query.discovery.LineageDto; +import com.embabel.dice.spi.RetiredProposition; +import com.embabel.dice.web.rest.ExtractOptions; +import com.embabel.dice.web.rest.ExtractRequest; +import com.embabel.dice.web.rest.ProvenanceEntryDto; +import org.junit.jupiter.api.Test; + +import java.io.InputStream; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class SourceRevisionJavaInteropTest { + + @Test + void retainsEveryLegacyJavaConcreteConstructorDescriptor() throws Exception { + Class[][] provenanceParameters = { + {SourceLocator.class}, + {SourceLocator.class, String.class}, + {SourceLocator.class, String.class, Integer.class}, + {SourceLocator.class, String.class, Integer.class, Integer.class}, + {SourceLocator.class, String.class, Integer.class, Integer.class, String.class}, + }; + for (Class[] parameters : provenanceParameters) { + ProvenanceEntry.class.getConstructor(parameters); + } + ProvenanceEntry.class.getConstructor( + SourceLocator.class, + String.class, + Integer.class, + Integer.class, + String.class, + String.class + ); + + ExtractRequest.class.getConstructor( + String.class, + String.class, + List.class, + String.class, + ExtractOptions.class + ); + ProvenanceEntryDto.class.getConstructor( + String.class, + String.class, + String.class, + Integer.class, + Integer.class, + String.class + ); + LineageDto.class.getConstructor( + String.class, + String.class, + String.class, + int.class, + List.class, + List.class + ); + RetiredProposition.class.getConstructor( + String.class, + com.embabel.dice.proposition.PropositionStatus.class, + List.class, + List.class, + List.class + ); + } + + @Test + void retainsEveryLegacyRememberDescriptorAndAddsDistinctSourceDescriptors() throws Exception { + Class[][] legacyTextParameters = { + {String.class, String.class, NamedEntity.class}, + {String.class, String.class, NamedEntity.class, List.class}, + {String.class, String.class, NamedEntity.class, List.class, ExtractionPerspective.class}, + {String.class, String.class, NamedEntity.class, List.class, ExtractionPerspective.class, + Boolean.class}, + }; + for (Class[] parameters : legacyTextParameters) { + IncrementalPropositionExtraction.class.getMethod("rememberText", parameters); + } + + Class[][] sourceAwareTextParameters = { + {String.class, String.class, NamedEntity.class, SourceLocator.class}, + {String.class, String.class, NamedEntity.class, SourceLocator.class, SourceRevisionRef.class}, + {String.class, String.class, NamedEntity.class, SourceLocator.class, SourceRevisionRef.class, + List.class}, + {String.class, String.class, NamedEntity.class, SourceLocator.class, SourceRevisionRef.class, + List.class, ExtractionPerspective.class}, + {String.class, String.class, NamedEntity.class, SourceLocator.class, SourceRevisionRef.class, + List.class, ExtractionPerspective.class, Boolean.class}, + }; + for (Class[] parameters : sourceAwareTextParameters) { + IncrementalPropositionExtraction.class.getMethod("rememberTextFromSource", parameters); + } + assertThrows( + NoSuchMethodException.class, + () -> IncrementalPropositionExtraction.class.getMethod( + "rememberText", + String.class, + String.class, + NamedEntity.class, + SourceLocator.class + ) + ); + + IncrementalPropositionExtraction.class.getMethod( + "rememberFile", + InputStream.class, + String.class, + NamedEntity.class + ); + IncrementalPropositionExtraction.class.getMethod( + "rememberFileFromSource", + InputStream.class, + String.class, + NamedEntity.class, + SourceLocator.class + ); + IncrementalPropositionExtraction.class.getMethod( + "rememberFileFromSource", + InputStream.class, + String.class, + NamedEntity.class, + SourceLocator.class, + SourceRevisionRef.class + ); + assertThrows( + NoSuchMethodException.class, + () -> IncrementalPropositionExtraction.class.getMethod( + "rememberFile", + InputStream.class, + String.class, + NamedEntity.class, + SourceLocator.class + ) + ); + } + + @Test + void legacyAndRevisionAwareJavaEventSubclassesUseTheBaseConstructor() throws Exception { + assertArrayEquals( + new Class[]{Object.class, NamedEntity.class}, + SourceAnalysisRequestEvent.class + .getDeclaredConstructor(Object.class, NamedEntity.class) + .getParameterTypes() + ); + + NamedEntity user = org.mockito.Mockito.mock(NamedEntity.class); + LegacyJavaEvent legacy = new LegacyJavaEvent(this, user); + assertSame(user, legacy.user); + assertNull(legacy.sourceLocator()); + assertNull(legacy.sourceRevision()); + + SourceLocator locator = new ContentAddressedLocator("java-event-source"); + SourceRevisionRef revision = new SourceRevisionRef(locator.key(), "opaque-r1"); + RevisionAwareJavaEvent revisionAware = + new RevisionAwareJavaEvent(this, user, locator, revision); + assertSame(locator, revisionAware.sourceLocator()); + assertSame(revision, revisionAware.sourceRevision()); + assertEquals("opaque-r1", revisionAware.sourceRevision().getSourceRevision()); + } + + /** + * Compiling this body proves the legacy and additive Java source entry points remain callable. + * It is intentionally never executed because extraction has observable side effects. + */ + @SuppressWarnings({"unused", "DataFlowIssue"}) + private static void compileJavaSourceCalls( + IncrementalPropositionExtraction extraction, + InputStream input, + NamedEntity user, + SourceLocator locator, + SourceRevisionRef revision + ) { + extraction.rememberFile(input, "legacy.txt", user); + extraction.rememberFileFromSource(input, "source.txt", user, locator); + extraction.rememberFileFromSource(input, "revisioned.txt", user, locator, revision); + extraction.rememberText("legacy", "legacy-id", user); + extraction.rememberText("legacy", "legacy-id", user, List.of()); + extraction.rememberText("legacy", "legacy-id", user, List.of(), null); + extraction.rememberText("legacy", "legacy-id", user, List.of(), null, null); + extraction.rememberTextFromSource("revisioned", "revisioned-id", user, locator); + extraction.rememberTextFromSource("revisioned", "revisioned-id", user, locator, revision); + extraction.rememberTextFromSource( + "revisioned", "revisioned-id", user, locator, revision, List.of() + ); + extraction.rememberTextFromSource( + "revisioned", "revisioned-id", user, locator, revision, List.of(), null + ); + extraction.rememberTextFromSource( + "revisioned", "revisioned-id", user, locator, revision, List.of(), null, null + ); + } + + private static final class LegacyJavaEvent extends SourceAnalysisRequestEvent { + + private LegacyJavaEvent(Object source, NamedEntity user) { + super(source, user); + } + + @Override + public IncrementalSource incrementalSource() { + throw new UnsupportedOperationException("Not needed by this compatibility test"); + } + } + + private static final class RevisionAwareJavaEvent extends SourceAnalysisRequestEvent { + + private final SourceLocator locator; + private final SourceRevisionRef revision; + + private RevisionAwareJavaEvent( + Object source, + NamedEntity user, + SourceLocator locator, + SourceRevisionRef revision + ) { + super(source, user); + this.locator = locator; + this.revision = revision; + } + + @Override + public IncrementalSource incrementalSource() { + throw new UnsupportedOperationException("Not needed by this compatibility test"); + } + + @Override + public SourceLocator sourceLocator() { + return locator; + } + + @Override + public SourceRevisionRef sourceRevision() { + return revision; + } + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/SourceRevisionBinaryCompatibilityTest.kt b/dice/src/test/kotlin/com/embabel/dice/SourceRevisionBinaryCompatibilityTest.kt new file mode 100644 index 00000000..5be07b50 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/SourceRevisionBinaryCompatibilityTest.kt @@ -0,0 +1,193 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.springframework.asm.ClassReader +import org.springframework.asm.ClassVisitor +import org.springframework.asm.ClassWriter +import org.springframework.asm.MethodVisitor +import org.springframework.asm.Opcodes +import java.io.ByteArrayOutputStream +import java.io.PrintStream +import java.net.URLClassLoader +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.security.MessageDigest +import java.util.jar.JarFile + +class SourceRevisionBinaryCompatibilityTest { + + @Test + fun `pinned legacy client links only the approved compatibility boundary`() { + val source = requireResource(FIXTURE_SOURCE) + val clientJar = requireResource(FIXTURE_JAR) + + assertEquals(FIXTURE_SOURCE_SHA256, sha256(Files.readAllBytes(source))) + assertEquals(FIXTURE_JAR_SHA256, sha256(Files.readAllBytes(clientJar))) + assertFixtureJar(clientJar) + + val linkedOutput = runClient(clientJar, javaClass.classLoader) + assertEquals(EXPECTED_CANDIDATE_OUTPUT, linkedOutput) + println("legacy-client candidate output:") + linkedOutput.forEach(::println) + + val withoutApprovedConstructor = WithoutLegacyProvenanceConstructor(javaClass.classLoader) + val negativeOutput = runClient(clientJar, withoutApprovedConstructor) + assertEquals(1, withoutApprovedConstructor.removedConstructors) + assertTrue( + "ProvenanceEntry.constructor.full=NoSuchMethodError" in negativeOutput, + "Removing the approved constructor must break the same legacy call site: $negativeOutput", + ) + println("legacy-client negative-control output:") + negativeOutput.forEach(::println) + } + + private fun assertFixtureJar(clientJar: Path) { + JarFile(clientJar.toFile()).use { jar -> + assertEquals( + setOf( + "META-INF/", + "META-INF/MANIFEST.MF", + "com/embabel/dice/compat/", + "com/embabel/dice/compat/Source_revision_legacy_clientKt.class", + ), + jar.entries().asSequence().map { it.name }.toSet(), + ) + assertEquals(PINNED_BASE_SHA, jar.manifest.mainAttributes.getValue("Dice-Base-SHA")) + assertEquals( + FIXTURE_SOURCE_SHA256, + jar.manifest.mainAttributes.getValue("Fixture-Source-SHA256"), + ) + } + } + + private fun runClient(clientJar: Path, parent: ClassLoader): List { + val bytes = ByteArrayOutputStream() + val originalOut = System.out + try { + PrintStream(bytes, true, StandardCharsets.UTF_8).use { captured -> + System.setOut(captured) + URLClassLoader(arrayOf(clientJar.toUri().toURL()), parent).use { loader -> + loader + .loadClass(CLIENT_MAIN_CLASS) + .getMethod("main") + .invoke(null) + } + } + } finally { + System.setOut(originalOut) + } + return bytes + .toString(StandardCharsets.UTF_8) + .lineSequence() + .filter(String::isNotBlank) + .toList() + } + + private fun requireResource(name: String): Path = + Path.of(requireNotNull(javaClass.classLoader.getResource(name)) { "Missing resource $name" }.toURI()) + + private fun sha256(bytes: ByteArray): String = + MessageDigest + .getInstance("SHA-256") + .digest(bytes) + .joinToString("") { "%02x".format(it) } + + private class WithoutLegacyProvenanceConstructor( + parent: ClassLoader, + ) : ClassLoader(parent) { + + var removedConstructors: Int = 0 + private set + + override fun loadClass(name: String, resolve: Boolean): Class<*> { + if (name != PROVENANCE_ENTRY_CLASS) { + return super.loadClass(name, resolve) + } + synchronized(getClassLoadingLock(name)) { + var loaded = findLoadedClass(name) + if (loaded == null) { + loaded = defineWithoutLegacyConstructor(name) + } + if (resolve) { + resolveClass(loaded) + } + return loaded + } + } + + private fun defineWithoutLegacyConstructor(name: String): Class<*> { + val resource = name.replace('.', '/') + ".class" + val candidateBytes = + requireNotNull(parent.getResourceAsStream(resource)) { + "Candidate bytecode is missing $resource" + }.use { it.readAllBytes() } + val reader = ClassReader(candidateBytes) + val writer = ClassWriter(reader, 0) + reader.accept( + object : ClassVisitor(Opcodes.ASM9, writer) { + override fun visitMethod( + access: Int, + name: String, + descriptor: String, + signature: String?, + exceptions: Array?, + ): MethodVisitor? { + if (name == "" && descriptor == LEGACY_PROVENANCE_CONSTRUCTOR) { + removedConstructors += 1 + return null + } + return super.visitMethod(access, name, descriptor, signature, exceptions) + } + }, + 0, + ) + val transformed = writer.toByteArray() + return defineClass(name, transformed, 0, transformed.size) + } + } + + private companion object { + const val PINNED_BASE_SHA = "c769d9c479d3e90c5c23c88343c79bd31e70a78f" + const val FIXTURE_SOURCE_SHA256 = + "8a2ce95b81d59303d5361db85e60074249652f73f4d6890c89499d940a29e524" + const val FIXTURE_JAR_SHA256 = + "505cb11891a051f69cefe70e3a0862e29e66e78df864cae2aac2a9c2eddf08a2" + const val FIXTURE_SOURCE = "compat/source-revision-legacy-client.kt" + const val FIXTURE_JAR = "compat/source-revision-legacy-client.jar" + const val CLIENT_MAIN_CLASS = "com.embabel.dice.compat.Source_revision_legacy_clientKt" + const val PROVENANCE_ENTRY_CLASS = "com.embabel.dice.provenance.ProvenanceEntry" + const val LEGACY_PROVENANCE_CONSTRUCTOR = + "(Lcom/embabel/dice/provenance/SourceLocator;Ljava/lang/String;" + + "Ljava/lang/Integer;Ljava/lang/Integer;Ljava/lang/String;)V" + + val EXPECTED_CANDIDATE_OUTPUT = + listOf( + "ProvenanceEntry.constructor.full=LINKED", + "ProvenanceEntry.constructor.nullable=LINKED", + "ProvenanceEntry.copy.direct=NoSuchMethodError", + "ProvenanceEntry.copy.default=NoSuchMethodError", + "SourceAnalysisContext.constructor.full=LINKED", + "SourceAnalysisContext.constructor.alternate=LINKED", + "SourceAnalysisContext.copy.direct=NoSuchMethodError", + "SourceAnalysisContext.copy.default=NoSuchMethodError", + ) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/SourceRevisionCompatibilityTest.kt b/dice/src/test/kotlin/com/embabel/dice/SourceRevisionCompatibilityTest.kt new file mode 100644 index 00000000..80a1fead --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/SourceRevisionCompatibilityTest.kt @@ -0,0 +1,146 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice + +import com.embabel.agent.core.ContextId +import com.embabel.agent.core.DataDictionary +import com.embabel.dice.common.SourceAnalysisContext +import com.embabel.dice.common.resolver.AlwaysCreateEntityResolver +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.provenance.ContentAddressedLocator +import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.SourceRevisionRef +import com.embabel.dice.spi.RetiredProposition +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.KotlinModule +import com.fasterxml.jackson.module.kotlin.readValue +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class SourceRevisionCompatibilityTest { + + private val mapper = ObjectMapper() + .registerModule(KotlinModule.Builder().build()) + .registerModule(JavaTimeModule()) + + @Test + fun `legacy Kotlin source constructors and copy calls recompile against the candidate`() { + val locator = ContentAddressedLocator("legacy-kotlin-source") + val provenance = ProvenanceEntry(locator).copy(contentHash = "updated") + assertEquals("updated", provenance.contentHash) + assertNull(provenance.sourceRevision) + + val context = SourceAnalysisContext( + schema = DataDictionary.fromClasses("compatibility"), + entityResolver = AlwaysCreateEntityResolver, + contextId = ContextId("compatibility"), + ).copy(promptVariables = mapOf("legacy" to true)) + assertEquals(true, context.promptVariables["legacy"]) + assertNull(context.sourceRevision) + } + + @Test + fun `new Kotlin source constructors and copy calls carry an opaque revision`() { + val locator = ContentAddressedLocator("revisioned-kotlin-source") + val revision = SourceRevisionRef(locator.key(), "opaque::r/1") + val provenance = ProvenanceEntry( + locator = locator, + sourceRevision = revision.sourceRevision, + ).copy(contentHash = "updated") + assertEquals("opaque::r/1", provenance.sourceRevision) + + val context = SourceAnalysisContext( + schema = DataDictionary.fromClasses("compatibility"), + entityResolver = AlwaysCreateEntityResolver, + contextId = ContextId("compatibility"), + sourceLocator = locator, + sourceRevision = revision, + ).copy(promptVariables = mapOf("revisioned" to true)) + assertSame(revision, context.sourceRevision) + } + + @Test + fun `revisionless stored JSON remains readable and revisioned JSON round trips`() { + val revisionless = mapper.readValue(fixture("proposition-revisionless.json")) + assertNull(revisionless.provenanceEntries.single().sourceRevision) + assertEquals( + revisionless, + mapper.readValue(mapper.writeValueAsString(revisionless)), + ) + + val revisioned = mapper.readValue(fixture("proposition-revisioned.json")) + assertEquals(listOf("r1", "null"), revisioned.provenanceEntries.map { it.sourceRevision }) + assertEquals( + revisioned, + mapper.readValue(mapper.writeValueAsString(revisioned)), + ) + } + + @Test + fun `collector trace JSON keeps readable refs without exposing storage identities`() { + val json = mapper.writeValueAsString( + RetiredProposition( + propositionId = "retired", + priorStatus = PropositionStatus.ACTIVE, + foldedProvenanceRefs = listOf("uri:https://example.com/source"), + foldedProvenanceEvidenceKeys = listOf("dice-provenance:v1:opaque"), + ), + ) + + assertTrue(json.contains("uri:https://example.com/source")) + assertFalse(json.contains("foldedProvenanceEvidenceKeys")) + assertFalse(json.contains("dice-provenance:v1:opaque")) + } + + @Test + fun `old Kotlin synthetic constructor and copy descriptors are outside the approved boundary`() { + val marker = Class.forName("kotlin.jvm.internal.DefaultConstructorMarker") + + assertThrows(NoSuchMethodException::class.java) { + ProvenanceEntry::class.java.getDeclaredConstructor( + com.embabel.dice.provenance.SourceLocator::class.java, + String::class.java, + Int::class.javaObjectType, + Int::class.javaObjectType, + String::class.java, + Int::class.javaPrimitiveType, + marker, + ) + } + assertThrows(NoSuchMethodException::class.java) { + ProvenanceEntry::class.java.getDeclaredMethod( + "copy", + com.embabel.dice.provenance.SourceLocator::class.java, + String::class.java, + Int::class.javaObjectType, + Int::class.javaObjectType, + String::class.java, + ) + } + } + + private fun fixture(name: String): String = + requireNotNull(javaClass.getResource("/provenance/$name")) { + "Missing provenance fixture: $name" + }.readText() +} diff --git a/dice/src/test/kotlin/com/embabel/dice/common/SourceAnalysisRequestEventRevisionTest.kt b/dice/src/test/kotlin/com/embabel/dice/common/SourceAnalysisRequestEventRevisionTest.kt new file mode 100644 index 00000000..c549ed46 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/common/SourceAnalysisRequestEventRevisionTest.kt @@ -0,0 +1,110 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.common + +import com.embabel.agent.core.ContextId +import com.embabel.agent.core.DataDictionary +import com.embabel.agent.rag.model.NamedEntity +import com.embabel.chat.Conversation +import com.embabel.chat.Message +import com.embabel.dice.incremental.IncrementalSource +import com.embabel.dice.provenance.ContentAddressedLocator +import com.embabel.dice.provenance.SourceRevisionRef +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.mockito.Mockito.mock + +class SourceAnalysisRequestEventRevisionTest { + + @Test + fun `legacy subclass and concrete constructor retain null provenance defaults`() { + val user = mock(NamedEntity::class.java) + + val legacy = LegacyEvent(this, user) + assertNull(legacy.sourceLocator()) + assertNull(legacy.sourceRevision()) + + val conversation = ConversationAnalysisRequestEvent( + source = this, + user = user, + conversation = mock(Conversation::class.java), + ) + assertNull(conversation.sourceLocator()) + assertNull(conversation.sourceRevision()) + } + + @Test + fun `provenance aware conversation event carries exact locator and revision`() { + val locator = ContentAddressedLocator("source-content") + val revision = SourceRevisionRef(locator.key(), "revision-3") + val event = ConversationAnalysisRequestEvent( + source = this, + user = mock(NamedEntity::class.java), + conversation = mock(Conversation::class.java), + sourceLocator = locator, + sourceRevision = revision, + ) + + assertSame(locator, event.sourceLocator()) + assertSame(revision, event.sourceRevision()) + } + + @Test + fun `event provenance key mismatch is rejected through context creation`() { + val locator = ContentAddressedLocator("source-content") + val event = ConversationAnalysisRequestEvent( + source = this, + user = mock(NamedEntity::class.java), + conversation = mock(Conversation::class.java), + sourceLocator = locator, + sourceRevision = SourceRevisionRef("different-key", "revision-3"), + ) + + assertThrows(IllegalArgumentException::class.java) { + SourceAnalysisContext( + schema = mock(DataDictionary::class.java), + entityResolver = mock(EntityResolver::class.java), + contextId = ContextId("event-revision-test"), + sourceLocator = event.sourceLocator(), + sourceRevision = event.sourceRevision(), + ) + } + } + + @Test + fun `base constructor descriptor remains source and user only`() { + val constructor = SourceAnalysisRequestEvent::class.java.declaredConstructors.single() + + assertEquals(2, constructor.parameterCount) + assertArrayEquals( + arrayOf(Any::class.java, NamedEntity::class.java), + constructor.parameterTypes, + ) + } + + private class LegacyEvent( + source: Any, + user: NamedEntity, + ) : SourceAnalysisRequestEvent(source, user) { + + override fun incrementalSource(): IncrementalSource = + throw UnsupportedOperationException("not needed by this compatibility test") + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/pipeline/SourceRevisionPipelineTest.kt b/dice/src/test/kotlin/com/embabel/dice/pipeline/SourceRevisionPipelineTest.kt new file mode 100644 index 00000000..c5997cd3 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/pipeline/SourceRevisionPipelineTest.kt @@ -0,0 +1,140 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.pipeline + +import com.embabel.agent.core.ContextId +import com.embabel.agent.core.DataDictionary +import com.embabel.agent.rag.model.Chunk +import com.embabel.dice.common.EntityResolver +import com.embabel.dice.common.KnownEntity +import com.embabel.dice.common.Relations +import com.embabel.dice.common.SourceAnalysisContext +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionExtractor +import com.embabel.dice.proposition.extraction.ExtractionPerspective +import com.embabel.dice.provenance.ContentAddressedLocator +import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.SourceRevisionRef +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.mockito.ArgumentMatchers.anyList +import org.mockito.Mockito.doAnswer +import org.mockito.Mockito.mock +import org.mockito.Mockito.verifyNoInteractions + +class SourceRevisionPipelineTest { + + @Test + fun `matching source revision is stamped onto provenance`() { + val locator = ContentAddressedLocator("source-content") + val context = context(sourceLocator = locator) + .withSourceRevision(SourceRevisionRef(locator.key(), "revision-7")) + val entry = stampProvenance(context) + + assertSame(locator, entry.locator) + assertEquals("revision-7", entry.sourceRevision) + } + + @Test + fun `missing locator and mismatched key fail before extractor use`() { + val extractor = mock(PropositionExtractor::class.java) + val missingLocator = SourceRevisionRef("source-key", "revision-1") + + assertThrows(IllegalArgumentException::class.java) { + context().withSourceRevision(missingLocator) + } + + val locator = ContentAddressedLocator("source-content") + assertThrows(IllegalArgumentException::class.java) { + context(sourceLocator = locator) + .withSourceRevision(SourceRevisionRef("different-key", "revision-1")) + } + verifyNoInteractions(extractor) + } + + @Test + fun `context refinements retain locator revision and other dimensions`() { + val locator = ContentAddressedLocator("source-content") + val revision = SourceRevisionRef(locator.key(), "revision-2") + val knownEntity = mock(KnownEntity::class.java) + val perspective = mock(ExtractionPerspective::class.java) + val refined = context(sourceLocator = locator) + .withSourceRevision(revision) + .withKnownEntities(knownEntity) + .withRelations(Relations.empty()) + .withPromptVariables(mapOf("audience" to "test")) + .withPerspective(perspective) + .withMintNewEntities(true) + .withMintedEntityProperties(mapOf("tenant" to "one")) + + assertSame(locator, refined.sourceLocator) + assertSame(revision, refined.sourceRevision) + assertEquals(listOf(knownEntity), refined.knownEntities) + assertEquals(mapOf("audience" to "test"), refined.promptVariables) + assertSame(perspective, refined.perspective) + assertEquals(true, refined.mintNewEntities) + assertEquals(mapOf("tenant" to "one"), refined.mintedEntityProperties) + } + + @Test + fun `null source revision preserves content addressed fallback`() { + val entry = stampProvenance(context()) + + assertInstanceOf(ContentAddressedLocator::class.java, entry.locator) + assertNull(entry.sourceRevision) + } + + private fun context(sourceLocator: ContentAddressedLocator? = null): SourceAnalysisContext = + SourceAnalysisContext( + schema = mock(DataDictionary::class.java), + entityResolver = mock(EntityResolver::class.java), + contextId = ContextId("source-revision-test"), + sourceLocator = sourceLocator, + ) + + @Suppress("UNCHECKED_CAST") + private fun stampProvenance(context: SourceAnalysisContext): ProvenanceEntry { + val extractor = mock(PropositionExtractor::class.java) + val pipeline = PropositionPipeline.withExtractor(extractor) + val proposition = mock(Proposition::class.java) + lateinit var captured: List + doAnswer { invocation -> + captured = invocation.getArgument(0) + proposition + }.`when`(proposition).withProvenanceEntries(anyList()) + + val method = PropositionPipeline::class.java.getDeclaredMethod( + "stampProvenance", + List::class.java, + Chunk::class.java, + SourceAnalysisContext::class.java, + ) + method.isAccessible = true + val stamped = method.invoke( + pipeline, + listOf(proposition), + Chunk.create(text = "source text", parentId = "source", id = "chunk"), + context, + ) as List + + assertEquals(listOf(proposition), stamped) + return captured.single() + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/projection/memory/collector/MultiSignalCollectorStrategyTest.kt b/dice/src/test/kotlin/com/embabel/dice/projection/memory/collector/MultiSignalCollectorStrategyTest.kt index b6736030..38e357fb 100644 --- a/dice/src/test/kotlin/com/embabel/dice/projection/memory/collector/MultiSignalCollectorStrategyTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/projection/memory/collector/MultiSignalCollectorStrategyTest.kt @@ -22,6 +22,9 @@ import com.embabel.dice.proposition.EntityMention import com.embabel.dice.proposition.MentionRole import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.store.InMemoryPropositionRepository +import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.ProvenanceEvidenceKey +import com.embabel.dice.provenance.UriLocator import com.embabel.dice.spi.CollectorCandidateEdge import com.embabel.dice.spi.CollectorComponent import com.embabel.dice.spi.CollectorDecision @@ -61,6 +64,7 @@ class MultiSignalCollectorStrategyTest { confidence: Double = 0.8, reinforceCount: Int = 0, mentions: List = emptyList(), + provenanceEntries: List = emptyList(), ): Proposition = Proposition( contextId = contextId, @@ -69,6 +73,7 @@ class MultiSignalCollectorStrategyTest { confidence = confidence, decay = 0.0, reinforceCount = reinforceCount, + provenanceEntries = provenanceEntries, ) private fun mention(resolvedId: String): EntityMention = @@ -189,6 +194,49 @@ class MultiSignalCollectorStrategyTest { assertEquals(weak.sourceIds, retired.foldedSourceIds) } + @Test + fun `records only distinct full evidence added by a fold`() { + setEmbedding("strong statement", floatArrayOf(1f, 0f, 0f)) + setEmbedding("weak statement", floatArrayOf(0.99f, 0.1f, 0f)) + val locator = UriLocator("https://example.com/source") + val revisionOne = ProvenanceEntry(locator = locator, sourceRevision = "r1") + val revisionTwo = ProvenanceEntry(locator = locator, sourceRevision = "r2") + val strong = repo.save( + proposition( + "strong statement", + confidence = 0.9, + provenanceEntries = listOf(revisionOne), + ) + ) + val weak = repo.save( + proposition( + "weak statement", + confidence = 0.3, + provenanceEntries = listOf(revisionOne, revisionOne, revisionTwo, revisionTwo), + ) + ) + val traceStore = InMemoryCollectorTraceStore() + + vectorOnlyStrategy(traceStore).mark( + listOf(strong, weak), + repo, + CollectorRunContext("revision-trace-run", contextId), + ) + + val retired = traceStore.decisionsFor("revision-trace-run").single().retired.single() + assertTrue( + ProvenanceEvidenceKey.encode(revisionOne) != ProvenanceEvidenceKey.encode(revisionTwo) + ) + assertEquals( + emptyList(), + retired.foldedProvenanceRefs, + ) + assertEquals( + listOf(ProvenanceEvidenceKey.encode(revisionTwo)), + retired.foldedProvenanceEvidenceKeys, + ) + } + @Test fun `does not merge a and c through a shared neighbor b when a and c contradict`() { // Cosine similarity isn't transitive: a-b and b-c are both proposed and score above diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/PropositionFoldEvidenceTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/PropositionFoldEvidenceTest.kt new file mode 100644 index 00000000..60b88065 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/PropositionFoldEvidenceTest.kt @@ -0,0 +1,125 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition + +import com.embabel.agent.core.ContextId +import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.ProvenanceEvidenceKey +import com.embabel.dice.provenance.UriLocator +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class PropositionFoldEvidenceTest { + + private val locator = UriLocator("obsidian://vault/note") + + @Test + fun `undo removes only folded revision while preserving survivor evidence and collisions`() { + val survivorR1 = evidence( + revision = "r1", + chunkId = "chunk|shared", + contentHash = "hash", + ) + val foldedR2 = evidence( + revision = "r2", + chunkId = "chunk|shared", + contentHash = "hash", + ) + val collision = evidence( + revision = "r2", + chunkId = "chunk", + contentHash = "shared|hash", + ) + val survivor = proposition( + grounding = listOf("survivor-grounding"), + sourceIds = listOf("survivor-source"), + provenanceEntries = listOf(survivorR1, collision), + ) + val loser = proposition( + grounding = listOf("folded-grounding"), + sourceIds = listOf("folded-source"), + provenanceEntries = listOf(foldedR2), + ) + + val restored = survivor.absorbEvidence(loser).withoutFoldedEvidence( + groundingToRemove = loser.grounding, + provenanceRefsToRemove = listOf(ProvenanceEvidenceKey.encode(foldedR2)), + sourceIdsToRemove = loser.sourceIds, + ) + + assertEquals(listOf(survivorR1, collision), restored.provenanceEntries) + assertEquals(listOf("survivor-grounding"), restored.grounding) + assertEquals(listOf("survivor-source"), restored.sourceIds) + } + + @Test + fun `legacy locator trace removes only revisionless evidence`() { + val revisionless = evidence(revision = null) + val revisioned = evidence(revision = "r1") + val proposition = proposition(provenanceEntries = listOf(revisionless, revisioned)) + + val restored = proposition.withoutFoldedEvidence( + groundingToRemove = emptyList(), + provenanceRefsToRemove = listOf(locator.key()), + sourceIdsToRemove = emptyList(), + ) + + assertEquals(listOf(revisioned), restored.provenanceEntries) + } + + @Test + fun `malformed versioned trace fails closed without deleting evidence`() { + val evidence = evidence(revision = "r2") + val malformedVersionedKey = ProvenanceEvidenceKey.encode(evidence).dropLast(1) + val proposition = proposition(provenanceEntries = listOf(evidence)) + + val restored = proposition.withoutFoldedEvidence( + groundingToRemove = emptyList(), + provenanceRefsToRemove = listOf(malformedVersionedKey), + sourceIdsToRemove = emptyList(), + ) + + assertEquals(listOf(evidence), restored.provenanceEntries) + } + + private fun evidence( + revision: String?, + chunkId: String? = "chunk", + contentHash: String? = "hash", + ) = ProvenanceEntry( + locator = locator, + sourceRevision = revision, + chunkId = chunkId, + startOffset = 1, + endOffset = 2, + contentHash = contentHash, + ) + + private fun proposition( + grounding: List = emptyList(), + sourceIds: List = emptyList(), + provenanceEntries: List = emptyList(), + ) = Proposition( + id = "proposition", + contextId = ContextId("test"), + text = "Test proposition", + mentions = emptyList(), + confidence = 0.8, + grounding = grounding, + sourceIds = sourceIds, + provenanceEntries = provenanceEntries, + ) +} diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/PropositionRepositoryDelegationTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/PropositionRepositoryDelegationTest.kt index 21cff4c4..8bfff7bd 100644 --- a/dice/src/test/kotlin/com/embabel/dice/proposition/PropositionRepositoryDelegationTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/PropositionRepositoryDelegationTest.kt @@ -20,6 +20,9 @@ import com.embabel.agent.rag.service.RetrievableIdentifier import com.embabel.common.core.types.SimilarityResult import com.embabel.common.core.types.TextSimilaritySearchRequest import com.embabel.dice.common.DiceEventListener +import com.embabel.dice.provenance.SourceLocator +import com.embabel.dice.provenance.SourceRevisionRef +import com.embabel.dice.provenance.UriLocator import org.junit.jupiter.api.Assertions.assertDoesNotThrow import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -48,7 +51,7 @@ class PropositionRepositoryDelegationTest { * findByContextIdValue to their default implementations, which is exactly the * recursion site under test. */ - private inner class MinimalRepository : PropositionRepository { + private open inner class MinimalRepository : PropositionRepository { private val store = mutableMapOf() fun seed(p: Proposition) { store[p.id] = p } @@ -65,6 +68,41 @@ class PropositionRepositoryDelegationTest { override fun count(): Int = store.size } + private inner class TypedSourceOverrideRepository : MinimalRepository() { + val sourceKeyResult = listOf(proposition(contextId, "source-key")) + val sourceRevisionResult = listOf(proposition(contextId, "source-revision")) + val revisionlessResult = listOf(proposition(contextId, "revisionless")) + + var sourceKeyCalls = 0 + var sourceRevisionCalls = 0 + var revisionlessCalls = 0 + val receivedContexts = mutableListOf() + + override fun findBySourceKey(contextId: ContextId, sourceKey: String): List { + sourceKeyCalls++ + receivedContexts += contextId + return sourceKeyResult + } + + override fun findBySourceRevision( + contextId: ContextId, + ref: SourceRevisionRef, + ): List { + sourceRevisionCalls++ + receivedContexts += contextId + return sourceRevisionResult + } + + override fun findRevisionlessBySourceLocator( + contextId: ContextId, + locator: SourceLocator, + ): List { + revisionlessCalls++ + receivedContexts += contextId + return revisionlessResult + } + } + @Test fun `findByContextId does not StackOverflow through the decorator`() { val delegate = MinimalRepository().apply { @@ -97,4 +135,29 @@ class PropositionRepositoryDelegationTest { assertEquals(expected.toSet(), result.toSet()) assertEquals(1, result.size) } + + @Test + fun `string source bridges dispatch once to typed overrides`() { + val repository = TypedSourceOverrideRepository() + val locator = UriLocator("https://example.com/source") + val ref = SourceRevisionRef(sourceKey = locator.key(), sourceRevision = "rev-1") + + assertEquals( + repository.sourceKeyResult, + repository.findBySourceKey(contextId.value, locator.key()), + ) + assertEquals( + repository.sourceRevisionResult, + repository.findBySourceRevision(contextId.value, ref), + ) + assertEquals( + repository.revisionlessResult, + repository.findRevisionlessBySourceLocator(contextId.value, locator), + ) + + assertEquals(1, repository.sourceKeyCalls) + assertEquals(1, repository.sourceRevisionCalls) + assertEquals(1, repository.revisionlessCalls) + assertEquals(listOf(contextId, contextId, contextId), repository.receivedContexts) + } } diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/extraction/IncrementalPropositionExtractionTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/extraction/IncrementalPropositionExtractionTest.kt new file mode 100644 index 00000000..f418474d --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/extraction/IncrementalPropositionExtractionTest.kt @@ -0,0 +1,499 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.proposition.extraction + +import com.embabel.agent.core.DataDictionary +import com.embabel.agent.rag.model.NamedEntity +import com.embabel.agent.rag.service.NamedEntityDataRepository +import com.embabel.chat.Message +import com.embabel.dice.common.EntityResolver +import com.embabel.dice.common.Relations +import com.embabel.dice.common.SourceAnalysisContext +import com.embabel.dice.common.SourceAnalysisRequestEvent +import com.embabel.dice.incremental.ChunkHistoryStore +import com.embabel.dice.incremental.IncrementalSource +import com.embabel.dice.pipeline.ChunkPropositionResult +import com.embabel.dice.pipeline.PropositionPipeline +import com.embabel.dice.projection.graph.GraphProjectionService +import com.embabel.dice.proposition.PropositionRepository +import com.embabel.dice.provenance.SourceLocator +import com.embabel.dice.provenance.SourceRevisionRef +import com.embabel.dice.provenance.UriLocator +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.anyOrNull +import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.doNothing +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.spy +import org.mockito.kotlin.verify +import org.mockito.kotlin.verifyNoInteractions +import org.mockito.kotlin.whenever +import java.io.ByteArrayInputStream +import java.io.InputStream +import java.util.concurrent.atomic.AtomicInteger + +class IncrementalPropositionExtractionTest { + + @Test + fun `legacy and source aware JVM descriptors are exact`() { + val rememberTextParameters = IncrementalPropositionExtraction::class.java.declaredMethods + .filter { it.name == "rememberText" && !it.isSynthetic } + .map { it.parameterTypes.toList() } + .toSet() + val rememberFileParameters = IncrementalPropositionExtraction::class.java.declaredMethods + .filter { it.name == "rememberFile" && !it.isSynthetic } + .map { it.parameterTypes.toList() } + .toSet() + val rememberTextFromSourceParameters = IncrementalPropositionExtraction::class.java.declaredMethods + .filter { it.name == "rememberTextFromSource" && !it.isSynthetic } + .map { it.parameterTypes.toList() } + .toSet() + val rememberFileFromSourceParameters = IncrementalPropositionExtraction::class.java.declaredMethods + .filter { it.name == "rememberFileFromSource" && !it.isSynthetic } + .map { it.parameterTypes.toList() } + .toSet() + + val legacyTextPrefix = listOf( + String::class.java, + String::class.java, + NamedEntity::class.java, + ) + assertEquals( + setOf( + legacyTextPrefix, + legacyTextPrefix + List::class.java, + legacyTextPrefix + listOf(List::class.java, ExtractionPerspective::class.java), + legacyTextPrefix + listOf( + List::class.java, + ExtractionPerspective::class.java, + Boolean::class.javaObjectType, + ), + ), + rememberTextParameters, + ) + + val sourceTextPrefix = legacyTextPrefix + SourceLocator::class.java + assertEquals( + setOf( + sourceTextPrefix, + sourceTextPrefix + SourceRevisionRef::class.java, + sourceTextPrefix + listOf(SourceRevisionRef::class.java, List::class.java), + sourceTextPrefix + listOf( + SourceRevisionRef::class.java, + List::class.java, + ExtractionPerspective::class.java, + ), + sourceTextPrefix + listOf( + SourceRevisionRef::class.java, + List::class.java, + ExtractionPerspective::class.java, + Boolean::class.javaObjectType, + ), + ), + rememberTextFromSourceParameters, + ) + + val legacyFile = listOf( + InputStream::class.java, + String::class.java, + NamedEntity::class.java, + ) + assertEquals(setOf(legacyFile), rememberFileParameters) + assertEquals( + setOf( + legacyFile + SourceLocator::class.java, + legacyFile + listOf(SourceLocator::class.java, SourceRevisionRef::class.java), + ), + rememberFileFromSourceParameters, + ) + } + + @Test + fun `new text entry point retains exact typed and untyped inputs`() { + val pipeline = pipelineReturningNoResult() + val extraction = extraction(pipeline) + val user = user() + val locator = UriLocator("https://example.com/source") + val revision = SourceRevisionRef(locator.key(), "r7") + val perspective = mock() + val grounding = listOf("record:one", "record:two") + + extraction.rememberTextFromSource( + text = "source text", + sourceId = "caller:source:r7", + user = user, + sourceLocator = locator, + sourceRevision = revision, + additionalGrounding = grounding, + perspective = perspective, + mintNewEntities = true, + ) + + val context = capturedContext(pipeline, "source text", "caller:source:r7", grounding) + assertSame(locator, context.sourceLocator) + assertSame(revision, context.sourceRevision) + assertSame(perspective, context.perspective) + assertEquals(true, context.mintNewEntities) + } + + @Test + fun `legacy text inputs never synthesize typed provenance`() { + val pipeline = pipelineReturningNoResult() + val extraction = extraction(pipeline) + val grounding = listOf("revision:r9", "source:https://example.com/untyped") + + extraction.rememberText( + text = "legacy text", + sourceId = "untyped:r9", + user = user(), + additionalGrounding = grounding, + ) + + val context = capturedContext(pipeline, "legacy text", "untyped:r9", grounding) + assertNull(context.sourceLocator) + assertNull(context.sourceRevision) + } + + @Test + fun `mismatched typed provenance fails before pipeline invocation`() { + val pipeline = pipelineReturningNoResult() + val extraction = extraction(pipeline) + val locator = UriLocator("https://example.com/source") + + assertThrows(IllegalArgumentException::class.java) { + extraction.rememberTextFromSource( + text = "source text", + sourceId = "exact-source", + user = user(), + sourceLocator = locator, + sourceRevision = SourceRevisionRef("different-key", "r1"), + ) + } + verifyNoInteractions(pipeline) + } + + @Test + fun `mismatched file provenance fails before parsing or pipeline invocation`() { + val pipeline = pipelineReturningNoResult() + val extraction = extraction(pipeline) + val locator = UriLocator("file:///notes/example.txt") + + assertThrows(IllegalArgumentException::class.java) { + extraction.rememberFileFromSource( + inputStream = ByteArrayInputStream("file source text".toByteArray()), + filename = "example.txt", + user = user(), + sourceLocator = locator, + sourceRevision = SourceRevisionRef("different-key", "r1"), + ) + } + verifyNoInteractions(pipeline) + } + + @Test + fun `legacy file retains remember source id without typed provenance`() { + val pipeline = pipelineReturningNoResult() + val extraction = extraction(pipeline) + + extraction.rememberFile( + inputStream = ByteArrayInputStream("legacy file text".toByteArray()), + filename = "legacy.txt", + user = user(), + ) + + val contextCaptor = argumentCaptor() + verify(pipeline).processOnce( + any(), + eq("remember:legacy.txt"), + contextCaptor.capture(), + anyOrNull(), + any(), + eq(emptyList()), + ) + assertNull(contextCaptor.firstValue.sourceLocator) + assertNull(contextCaptor.firstValue.sourceRevision) + } + + @Test + fun `source aware file retains remember source id and typed provenance`() { + val pipeline = pipelineReturningNoResult() + val extraction = extraction(pipeline) + val locator = UriLocator("file:///notes/example.txt") + val revision = SourceRevisionRef(locator.key(), "file-r2") + + extraction.rememberFileFromSource( + inputStream = ByteArrayInputStream("file source text".toByteArray()), + filename = "example.txt", + user = user(), + sourceLocator = locator, + sourceRevision = revision, + ) + + val contextCaptor = argumentCaptor() + verify(pipeline).processOnce( + any(), + eq("remember:example.txt"), + contextCaptor.capture(), + anyOrNull(), + any(), + eq(emptyList()), + ) + assertSame(locator, contextCaptor.firstValue.sourceLocator) + assertSame(revision, contextCaptor.firstValue.sourceRevision) + } + + @Test + fun `legacy generic Mockito shaped call sites remain uniquely resolvable`() { + val extraction = mock() + val user = user() + val four = GenericMatcherValues("four", "source-4", user, emptyList()) + val five = GenericMatcherValues("five", "source-5", user, emptyList(), null) + val six = GenericMatcherValues("six", "source-6", user, emptyList(), null, null) + + extraction.rememberText(four.any(), four.any(), four.any(), four.any()) + extraction.rememberText(five.any(), five.any(), five.any(), five.any(), five.any()) + extraction.rememberText(six.any(), six.any(), six.any(), six.any(), six.any(), six.any()) + + verify(extraction).rememberText("four", "source-4", user, emptyList(), null, null) + verify(extraction).rememberText("five", "source-5", user, emptyList(), null, null) + verify(extraction).rememberText("six", "source-6", user, emptyList(), null, null) + } + + @Test + fun `Kotlin callable references and named calls distinguish legacy and source entry points`() { + val extraction = mock() + val user = user() + val locator = UriLocator("https://example.com/callable") + val revision = SourceRevisionRef(locator.key(), "callable-r1") + val legacyText: + (String, String, NamedEntity, List, ExtractionPerspective?, Boolean?) -> Unit = + extraction::rememberText + val sourceText: + ( + String, + String, + NamedEntity, + SourceLocator, + SourceRevisionRef?, + List, + ExtractionPerspective?, + Boolean?, + ) -> Unit = extraction::rememberTextFromSource + val legacyFile: (InputStream, String, NamedEntity) -> Unit = extraction::rememberFile + val sourceFile: + (InputStream, String, NamedEntity, SourceLocator, SourceRevisionRef?) -> Unit = + extraction::rememberFileFromSource + + legacyText("callable legacy", "callable-legacy", user, emptyList(), null, null) + sourceText("callable source", "callable-source", user, locator, revision, emptyList(), null, null) + legacyFile(ByteArrayInputStream(byteArrayOf()), "callable-legacy.txt", user) + sourceFile(ByteArrayInputStream(byteArrayOf()), "callable-source.txt", user, locator, revision) + extraction.rememberText(text = "named legacy", sourceId = "named-legacy", user = user) + extraction.rememberTextFromSource( + text = "named source", + sourceId = "named-source", + user = user, + sourceLocator = locator, + ) + extraction.rememberFile( + inputStream = ByteArrayInputStream(byteArrayOf()), + filename = "named-legacy.txt", + user = user, + ) + extraction.rememberFileFromSource( + inputStream = ByteArrayInputStream(byteArrayOf()), + filename = "named-source.txt", + user = user, + sourceLocator = locator, + ) + + verify(extraction).rememberText("callable legacy", "callable-legacy", user, emptyList(), null, null) + verify(extraction).rememberTextFromSource( + "callable source", + "callable-source", + user, + locator, + revision, + emptyList(), + null, + null, + ) + verify(extraction).rememberText("named legacy", "named-legacy", user) + verify(extraction).rememberTextFromSource("named source", "named-source", user, locator) + } + + @Test + fun `legacy and source file entry points dispatch through their open text entry points`() { + val pipeline = pipelineReturningNoResult() + val extraction = spy(extraction(pipeline)) + val user = user() + val locator = UriLocator("file:///notes/dispatch.txt") + val revision = SourceRevisionRef(locator.key(), "dispatch-r1") + doNothing().whenever(extraction).rememberText( + any(), + any(), + any(), + any(), + anyOrNull(), + anyOrNull(), + ) + doNothing().whenever(extraction).rememberTextFromSource( + any(), + any(), + any(), + any(), + anyOrNull(), + any(), + anyOrNull(), + anyOrNull(), + ) + + extraction.rememberFile( + ByteArrayInputStream("legacy dispatch".toByteArray()), + "legacy-dispatch.txt", + user, + ) + extraction.rememberFileFromSource( + ByteArrayInputStream("source dispatch".toByteArray()), + "source-dispatch.txt", + user, + locator, + revision, + ) + + verify(extraction).rememberText("legacy dispatch", "remember:legacy-dispatch.txt", user) + verify(extraction).rememberTextFromSource( + "source dispatch", + "remember:source-dispatch.txt", + user, + locator, + revision, + emptyList(), + null, + null, + ) + verifyNoInteractions(pipeline) + } + + @Test + fun `event provenance reaches the context observed by the pipeline`() { + val pipeline = pipelineReturningNoResult() + val extraction = extraction(pipeline) + val source = mock>() + whenever(source.id).thenReturn("event-source") + whenever(source.size).thenReturn(1) + val locator = UriLocator("https://example.com/event") + val revision = SourceRevisionRef(locator.key(), "event-r1") + val locatorCalls = AtomicInteger() + val revisionCalls = AtomicInteger() + val event = object : SourceAnalysisRequestEvent(this, user()) { + override fun incrementalSource(): IncrementalSource = source + + override fun sourceLocator(): SourceLocator = + locator.also { locatorCalls.incrementAndGet() } + + override fun sourceRevision(): SourceRevisionRef = + revision.also { revisionCalls.incrementAndGet() } + } + + extraction.extractPropositions(event) + + assertEquals(1, locatorCalls.get()) + assertEquals(1, revisionCalls.get()) + val contextCaptor = argumentCaptor() + verify(pipeline).processChunk(any(), contextCaptor.capture()) + assertSame(locator, contextCaptor.firstValue.sourceLocator) + assertSame(revision, contextCaptor.firstValue.sourceRevision) + } + + private fun pipelineReturningNoResult(): PropositionPipeline = + mock().also { pipeline -> + whenever( + pipeline.processOnce( + any(), + any(), + any(), + anyOrNull(), + any(), + any(), + ), + ).thenReturn(null) + whenever(pipeline.processChunk(any(), any())).thenReturn( + ChunkPropositionResult.Failed("event-source", "test result"), + ) + } + + private fun extraction(pipeline: PropositionPipeline): IncrementalPropositionExtraction { + val properties = mock() + whenever(properties.windowSize).thenReturn(1) + whenever(properties.overlapSize).thenReturn(1) + whenever(properties.triggerInterval).thenReturn(1) + return IncrementalPropositionExtraction( + propositionPipeline = pipeline, + chunkHistoryStore = mock(), + dataDictionary = mock(), + relations = Relations.empty(), + propositionRepository = mock(), + entityRepository = mock(), + entityResolver = mock(), + graphProjectionService = mock(), + properties = properties, + ) + } + + private fun user(): NamedEntity = + mock().also { user -> + whenever(user.id).thenReturn("user-1") + whenever(user.name).thenReturn("Test User") + } + + private fun capturedContext( + pipeline: PropositionPipeline, + text: String, + sourceId: String, + grounding: List, + ): SourceAnalysisContext { + val contextCaptor = argumentCaptor() + verify(pipeline).processOnce( + eq(text), + eq(sourceId), + contextCaptor.capture(), + anyOrNull(), + any(), + eq(grounding), + ) + return contextCaptor.firstValue + } + + /** + * Matches Mockito's no-argument generic matcher signature while returning real values, + * so the test executes Kotlin default-argument bridges without entering Mockito matcher state. + */ + private class GenericMatcherValues(vararg values: Any?) { + private val values = values.toList() + private var index = 0 + + @Suppress("UNCHECKED_CAST") + fun any(): T = values[index++] as T + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/store/InMemoryPropositionProvenanceTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/store/InMemoryPropositionProvenanceTest.kt index 73014c26..57311e9f 100644 --- a/dice/src/test/kotlin/com/embabel/dice/proposition/store/InMemoryPropositionProvenanceTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/store/InMemoryPropositionProvenanceTest.kt @@ -18,7 +18,9 @@ package com.embabel.dice.proposition.store import com.embabel.agent.core.ContextId import com.embabel.common.ai.model.EmbeddingService import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.SourceRevisionRef import com.embabel.dice.provenance.UriLocator import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull @@ -28,6 +30,124 @@ import org.mockito.kotlin.any import org.mockito.kotlin.mock import org.mockito.kotlin.whenever +internal class PortableSourceQueryFixture { + + val contextId = ContextId("portable-source-query") + private val foreignContextId = ContextId("portable-source-query-foreign") + val locator = UriLocator("https://example.com/shared-source") + val revisionOneRef = SourceRevisionRef(locator.key(), "r1") + val revisionTwoRef = SourceRevisionRef(locator.key(), "r2") + + val revisionless = proposition( + contextId = contextId, + text = "revisionless", + entries = listOf(entry()), + ) + val revisionOne = proposition( + contextId = contextId, + text = "r1", + entries = listOf(entry("r1")), + ) + val revisionTwo = proposition( + contextId = contextId, + text = "r2", + entries = listOf(entry("r2")), + ) + val duplicateRevisionOne = proposition( + contextId = contextId, + text = "duplicate r1 evidence", + entries = listOf( + entry("r1", "duplicate-r1-a"), + entry("r1", "duplicate-r1-b"), + ), + ) + val foreignContext = proposition( + contextId = foreignContextId, + text = "foreign context", + entries = listOf( + entry(), + entry("r1", "foreign-r1-a"), + entry("r1", "foreign-r1-b"), + entry("r2"), + ), + ) + + val propositions = listOf( + revisionless, + revisionOne, + revisionTwo, + duplicateRevisionOne, + foreignContext, + ) + + private fun entry( + sourceRevision: String? = null, + chunkId: String? = null, + ): ProvenanceEntry = + ProvenanceEntry( + locator = locator, + sourceRevision = sourceRevision, + chunkId = chunkId, + ) + + private fun proposition( + contextId: ContextId, + text: String, + entries: List, + ): Proposition = + Proposition( + contextId = contextId, + text = text, + mentions = emptyList(), + confidence = 0.9, + provenanceEntries = entries, + ) +} + +internal fun assertPortableSourceQueries( + repository: PropositionRepository, + fixture: PortableSourceQueryFixture, +) { + val allSourceVersions = repository.findBySourceKey( + fixture.contextId, + fixture.locator.key(), + ) + val exactRevisionOne = repository.findBySourceRevision( + fixture.contextId, + fixture.revisionOneRef, + ) + val exactRevisionTwo = repository.findBySourceRevision( + fixture.contextId, + fixture.revisionTwoRef, + ) + val revisionless = repository.findRevisionlessBySourceLocator( + fixture.contextId, + fixture.locator, + ) + + assertEquals( + setOf( + fixture.revisionless.id, + fixture.revisionOne.id, + fixture.revisionTwo.id, + fixture.duplicateRevisionOne.id, + ), + allSourceVersions.map { it.id }.toSet(), + ) + assertEquals( + setOf(fixture.revisionOne.id, fixture.duplicateRevisionOne.id), + exactRevisionOne.map { it.id }.toSet(), + ) + assertEquals(setOf(fixture.revisionTwo.id), exactRevisionTwo.map { it.id }.toSet()) + assertEquals(setOf(fixture.revisionless.id), revisionless.map { it.id }.toSet()) + + listOf(allSourceVersions, exactRevisionOne, exactRevisionTwo, revisionless).forEach { result -> + assertEquals(result.size, result.map { it.id }.toSet().size) + assertEquals(setOf(fixture.contextId), result.map { it.contextId }.toSet()) + assertEquals(false, result.any { it.id == fixture.foreignContext.id }) + } +} + /** * The provenance-management defaults on [com.embabel.dice.proposition.PropositionRepository] over the * in-memory backend: append, authoritative replace, read, and the absent-proposition contract. @@ -89,4 +209,12 @@ class InMemoryPropositionProvenanceTest { assertNull(repo.setProvenance("missing", listOf(uri("https://example.com/a")))) assertEquals(emptyList(), repo.provenanceOf("missing")) } + + @Test + fun `portable source queries distinguish revisions without duplicates or context leaks`() { + val fixture = PortableSourceQueryFixture() + repo.saveAll(fixture.propositions) + + assertPortableSourceQueries(repo, fixture) + } } diff --git a/dice/src/test/kotlin/com/embabel/dice/proposition/store/JsonFilePropositionRepositoryTest.kt b/dice/src/test/kotlin/com/embabel/dice/proposition/store/JsonFilePropositionRepositoryTest.kt index 8a169532..c155432b 100644 --- a/dice/src/test/kotlin/com/embabel/dice/proposition/store/JsonFilePropositionRepositoryTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/proposition/store/JsonFilePropositionRepositoryTest.kt @@ -18,10 +18,14 @@ package com.embabel.dice.proposition.store import com.embabel.agent.core.ContextId import com.embabel.common.core.types.TextSimilaritySearchRequest import com.embabel.dice.proposition.Proposition +import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.SourceRevisionRef +import com.embabel.dice.provenance.UriLocator import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files import java.nio.file.Path class JsonFilePropositionRepositoryTest { @@ -29,12 +33,17 @@ class JsonFilePropositionRepositoryTest { private val contextA = ContextId("user-a") private val contextB = ContextId("user-b") - private fun proposition(contextId: ContextId, text: String): Proposition = + private fun proposition( + contextId: ContextId, + text: String, + provenanceEntries: List = emptyList(), + ): Proposition = Proposition( contextId = contextId, text = text, mentions = emptyList(), confidence = 0.9, + provenanceEntries = provenanceEntries, ) @Test @@ -76,4 +85,59 @@ class JsonFilePropositionRepositoryTest { assertTrue(results.isEmpty()) } + + @Test + fun `portable source queries retain their meanings across a JSON reload`( + @TempDir tempDir: Path, + ) { + val storeFile = tempDir.resolve("propositions.json") + val fixture = PortableSourceQueryFixture() + val repository = JsonFilePropositionRepository(storeFile) + repository.saveAll(fixture.propositions) + + assertPortableSourceQueries(repository, fixture) + assertPortableSourceQueries(JsonFilePropositionRepository(storeFile), fixture) + } + + @Test + fun `legacy JSON without source revision participates only as revisionless evidence`( + @TempDir tempDir: Path, + ) { + val storeFile = tempDir.resolve("propositions.json") + val locator = UriLocator("https://example.com/legacy-source") + val local = proposition( + contextA, + "legacy local", + listOf(ProvenanceEntry(locator = locator, sourceRevision = "legacy-r1")), + ) + val foreign = proposition( + contextB, + "legacy foreign", + listOf(ProvenanceEntry(locator = locator, sourceRevision = "legacy-r1")), + ) + JsonFilePropositionRepository(storeFile).saveAll(listOf(local, foreign)) + + val currentJson = Files.readString(storeFile) + val oldJson = currentJson.replace( + Regex(",\\s*\"sourceRevision\"\\s*:\\s*(?:null|\"(?:\\\\.|[^\"\\\\])*\")"), + "", + ) + assertTrue(oldJson != currentJson) + assertTrue(!oldJson.contains("\"sourceRevision\"")) + Files.writeString(storeFile, oldJson) + + val reloaded = JsonFilePropositionRepository(storeFile) + val allSourceVersions = reloaded.findBySourceKey(contextA, locator.key()) + val exactRevision = reloaded.findBySourceRevision( + contextA, + SourceRevisionRef(locator.key(), "legacy-r1"), + ) + val revisionless = reloaded.findRevisionlessBySourceLocator(contextA, locator) + + assertEquals(listOf(local.id), allSourceVersions.map { it.id }) + assertEquals(emptyList(), exactRevision) + assertEquals(listOf(local.id), revisionless.map { it.id }) + assertEquals(false, allSourceVersions.any { it.id == foreign.id }) + assertEquals(false, revisionless.any { it.id == foreign.id }) + } } diff --git a/dice/src/test/kotlin/com/embabel/dice/provenance/ProvenanceEvidenceKeyTest.kt b/dice/src/test/kotlin/com/embabel/dice/provenance/ProvenanceEvidenceKeyTest.kt new file mode 100644 index 00000000..37ee1a95 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/provenance/ProvenanceEvidenceKeyTest.kt @@ -0,0 +1,122 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.provenance + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class ProvenanceEvidenceKeyTest { + + private val locator = UriLocator("https://example.com/source") + + @Test + fun `full evidence identity participates in encoding`() { + val entry = ProvenanceEntry( + locator = locator, + sourceRevision = "r1", + chunkId = "chunk", + startOffset = 3, + endOffset = 17, + contentHash = "sha256:abc", + ) + val variants = listOf( + entry.copy(locator = UriLocator("https://example.com/other")), + entry.copy(sourceRevision = "r2"), + entry.copy(chunkId = "other"), + entry.copy(startOffset = 4), + entry.copy(endOffset = 18), + entry.copy(contentHash = "sha256:def"), + ) + val encoded = ProvenanceEvidenceKey.encode(entry) + + assertThat(ProvenanceEvidenceKey.matches(entry, encoded)).isTrue() + assertThat(variants.map(ProvenanceEvidenceKey::encode) + encoded) + .doesNotHaveDuplicates() + variants.forEach { + assertThat(ProvenanceEvidenceKey.matches(it, encoded)).isFalse() + } + } + + @Test + fun `null and literal null remain distinct without delimiter or unicode collisions`() { + val revisionless = ProvenanceEntry( + locator = UriLocator("https://example.com/a:|💾"), + chunkId = "null:|雪", + contentHash = "", + ) + val literalNull = revisionless.copy(sourceRevision = "null") + val revisionlessKey = ProvenanceEvidenceKey.encode(revisionless) + val literalNullKey = ProvenanceEvidenceKey.encode(literalNull) + + assertThat(revisionlessKey).isNotEqualTo(literalNullKey) + assertThat(ProvenanceEvidenceKey.matches(revisionless, revisionlessKey)).isTrue() + assertThat(ProvenanceEvidenceKey.matches(literalNull, literalNullKey)).isTrue() + assertThat(ProvenanceEvidenceKey.matches(revisionless, literalNullKey)).isFalse() + assertThat(ProvenanceEvidenceKey.matches(literalNull, revisionlessKey)).isFalse() + } + + @Test + fun `revision values and legacy raw keys match conservatively`() { + val revisionless = ProvenanceEntry(locator = locator) + val revisionOne = revisionless.copy(sourceRevision = "r1") + val revisionTwo = revisionless.copy(sourceRevision = "r2") + + assertThat(ProvenanceEvidenceKey.matches(revisionless, locator.key())).isTrue() + assertThat(ProvenanceEvidenceKey.matches(revisionOne, locator.key())).isFalse() + assertThat(ProvenanceEvidenceKey.matches(revisionTwo, locator.key())).isFalse() + assertThat( + ProvenanceEvidenceKey.matches( + revisionOne, + ProvenanceEvidenceKey.encode(revisionTwo), + ) + ).isFalse() + } + + @Test + fun `truncated malformed and unknown version keys fail closed`() { + val entry = ProvenanceEntry( + locator = locator, + sourceRevision = "r1", + chunkId = "chunk", + ) + val encoded = ProvenanceEvidenceKey.encode(entry) + val malformed = listOf( + "dice-provenance:v1:", + "dice-provenance:v1::", + "dice-provenance:v1:-2:", + "dice-provenance:v1:01:x", + "dice-provenance:v1:１:x", + "dice-provenance:v1:999999999999999999999:", + "$encoded!", + ) + + for (end in "dice-provenance:v1:".length until encoded.length) { + assertThat(ProvenanceEvidenceKey.matches(entry, encoded.substring(0, end))).isFalse() + } + malformed.forEach { + assertThat(ProvenanceEvidenceKey.matches(entry, it)).isFalse() + } + assertThat( + ProvenanceEvidenceKey.matches( + entry.copy( + locator = UriLocator("dice-provenance:v2:legacy-looking"), + sourceRevision = null, + ), + "dice-provenance:v2:legacy-looking", + ) + ).isFalse() + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/provenance/ProvenanceJsonCompatibilityTest.kt b/dice/src/test/kotlin/com/embabel/dice/provenance/ProvenanceJsonCompatibilityTest.kt new file mode 100644 index 00000000..85b654c9 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/provenance/ProvenanceJsonCompatibilityTest.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.provenance + +import com.embabel.dice.proposition.Proposition +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.KotlinModule +import com.fasterxml.jackson.module.kotlin.readValue +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class ProvenanceJsonCompatibilityTest { + + private val defaultMapper = ObjectMapper() + .registerModule(KotlinModule.Builder().build()) + .registerModule(JavaTimeModule()) + + private val bundleReader = ObjectMapper() + .registerModule(KotlinModule.Builder().build()) + .registerModule(JavaTimeModule()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + + @Test + fun `revisionless fixture loads a null source revision`() { + val proposition = defaultMapper.readValue(fixture("proposition-revisionless.json")) + val evidence = proposition.provenanceEntries.single() + + assertNull(evidence.sourceRevision) + assertEquals("chunk-legacy", evidence.chunkId) + assertEquals("uri:https://example.com/legacy", evidence.locator.key()) + assertEquals( + proposition, + defaultMapper.readValue(defaultMapper.writeValueAsString(proposition)), + ) + } + + @Test + fun `revision scalars round trip losslessly and literal null remains a value`() { + val fixture = fixture("proposition-revisioned.json") + val proposition = defaultMapper.readValue(fixture) + + assertTrue(fixture.contains("\"sourceRevision\": \"null\"")) + assertEquals(listOf("r1", "null"), proposition.provenanceEntries.map { it.sourceRevision }) + + val roundTripped = defaultMapper.readValue( + defaultMapper.writeValueAsString(proposition), + ) + assertEquals(proposition, roundTripped) + assertEquals("null", roundTripped.provenanceEntries.last().sourceRevision) + } + + @Test + fun `only the explicit bundle reader tolerates a future unknown field`() { + val tree = defaultMapper.readTree(fixture("proposition-revisioned.json")) as ObjectNode + (tree.withArray("provenanceEntries")[0] as ObjectNode) + .put("futureEvidenceField", "future-value") + val withFutureField = tree.toString() + + assertTrue(defaultMapper.isEnabled(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)) + assertThrows(UnrecognizedPropertyException::class.java) { + defaultMapper.readValue(withFutureField) + } + + val imported = bundleReader.readValue(withFutureField) + assertEquals(listOf("r1", "null"), imported.provenanceEntries.map { it.sourceRevision }) + } + + private fun fixture(name: String): String = + requireNotNull(javaClass.getResource("/provenance/$name")) { + "Missing provenance fixture: $name" + }.readText() +} diff --git a/dice/src/test/kotlin/com/embabel/dice/provenance/SourceRevisionContractTest.kt b/dice/src/test/kotlin/com/embabel/dice/provenance/SourceRevisionContractTest.kt new file mode 100644 index 00000000..31ef0355 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/provenance/SourceRevisionContractTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.provenance + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatIllegalArgumentException +import org.junit.jupiter.api.Test + +class SourceRevisionContractTest { + + private val objectMapper = jacksonObjectMapper() + private val locator = UriLocator("https://example.com/source") + + @Test + fun `source revision ref preserves opaque values`() { + val ref = SourceRevisionRef( + sourceKey = locator.key(), + sourceRevision = "null", + ) + + assertThat(ref.sourceKey).isEqualTo(locator.key()) + assertThat(ref.sourceRevision).isEqualTo("null") + assertThat(objectMapper.readValue(objectMapper.writeValueAsString(ref))) + .isEqualTo(ref) + } + + @Test + fun `source revision ref rejects blank components`() { + assertThatIllegalArgumentException() + .isThrownBy { SourceRevisionRef(" ", "r1") } + assertThatIllegalArgumentException() + .isThrownBy { SourceRevisionRef(locator.key(), "\t") } + } + + @Test + fun `provenance revision participates in equality and deduplication`() { + val revisionless = ProvenanceEntry(locator = locator) + val sameRevisionless = ProvenanceEntry(locator = locator) + val revisionOne = ProvenanceEntry(locator = locator, sourceRevision = "r1") + val duplicateRevisionOne = ProvenanceEntry(locator = locator, sourceRevision = "r1") + val revisionTwo = ProvenanceEntry(locator = locator, sourceRevision = "r2") + + assertThat(revisionless).isEqualTo(sameRevisionless) + assertThat(revisionOne).isNotEqualTo(revisionTwo) + assertThat(listOf(revisionOne, duplicateRevisionOne, revisionTwo).distinct()) + .containsExactly(revisionOne, revisionTwo) + } + + @Test + fun `provenance rejects a blank present revision`() { + assertThatIllegalArgumentException() + .isThrownBy { ProvenanceEntry(locator = locator, sourceRevision = " ") } + } + + @Test + fun `old and new provenance json preserve revision absence and value`() { + val revisionless = ProvenanceEntry(locator = locator) + val oldJson = objectMapper.valueToTree( + revisionless + ).apply { + remove("sourceRevision") + } + val fromOldJson = objectMapper.treeToValue(oldJson, ProvenanceEntry::class.java) + + val revised = ProvenanceEntry(locator = locator, sourceRevision = "null") + val fromNewJson = objectMapper.readValue( + objectMapper.writeValueAsString(revised) + ) + + assertThat(fromOldJson).isEqualTo(revisionless) + assertThat(fromOldJson.sourceRevision).isNull() + assertThat(fromNewJson).isEqualTo(revised) + assertThat(fromNewJson.sourceRevision).isEqualTo("null") + assertThat(fromNewJson).isNotEqualTo(fromOldJson) + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/query/discovery/DiscoveryDtoLeakTest.kt b/dice/src/test/kotlin/com/embabel/dice/query/discovery/DiscoveryDtoLeakTest.kt index 28dfe2da..ddea27b9 100644 --- a/dice/src/test/kotlin/com/embabel/dice/query/discovery/DiscoveryDtoLeakTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/query/discovery/DiscoveryDtoLeakTest.kt @@ -15,6 +15,11 @@ */ package com.embabel.dice.query.discovery +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.UriLocator +import com.embabel.dice.query.graph.PropositionLineage import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -39,6 +44,7 @@ class DiscoveryDtoLeakTest { EntityMentionSummaryDto::class, PathDto::class, NeighborhoodDto::class, + DiscoveryProvenanceDto::class, LineageDto::class, ProjectionHealthDto::class, TargetHealthDto::class, @@ -58,6 +64,7 @@ class DiscoveryDtoLeakTest { // Any raw proposition-package type (e.g. the PropositionStatus enum) must be projected to a // primitive in a DTO, never exposed directly. DTOs surface enum names as Strings. "com.embabel.dice.proposition", + "com.embabel.dice.provenance", ) private val forbiddenExactFqns = listOf( @@ -92,6 +99,53 @@ class DiscoveryDtoLeakTest { assertEquals(5, RetrievalMode.entries.size, "RetrievalMode must expose exactly five modes") } + @Test + fun `discovery provenance has the exact primitive shape`() { + assertEquals( + mapOf( + "locator" to "kotlin.String", + "sourceRevision" to "kotlin.String?", + "chunkId" to "kotlin.String?", + "startOffset" to "kotlin.Int?", + "endOffset" to "kotlin.Int?", + "contentHash" to "kotlin.String?", + ), + DiscoveryProvenanceDto::class.memberProperties.associate { + it.name to it.returnType.toString() + }, + ) + } + + @Test + fun `lineage mapper uses authoritative ordered evidence instead of nested proposition evidence`() { + val locator = UriLocator("https://example.com/authoritative") + val nestedProposition = Proposition( + id = "lean-proposition", + contextId = ContextId("ctx-discovery"), + text = "Lean proposition", + mentions = emptyList(), + confidence = 0.9, + provenanceEntries = emptyList(), + ) + val lineage = PropositionLineage( + proposition = nestedProposition, + groundingChunkIds = emptyList(), + sources = emptyList(), + reinforceCount = 0, + status = nestedProposition.status, + temporal = null, + provenanceEntries = listOf( + ProvenanceEntry(locator = locator, sourceRevision = "r1"), + ProvenanceEntry(locator = locator, sourceRevision = "r2"), + ), + ) + + assertEquals( + listOf("r1", "r2"), + LineageDto.from(lineage).provenance.map { it.sourceRevision }, + ) + } + private fun walk( klass: KClass<*>, visited: MutableSet>, diff --git a/dice/src/test/kotlin/com/embabel/dice/spi/CollectorUndoCapabilityTest.kt b/dice/src/test/kotlin/com/embabel/dice/spi/CollectorUndoCapabilityTest.kt new file mode 100644 index 00000000..478ef959 --- /dev/null +++ b/dice/src/test/kotlin/com/embabel/dice/spi/CollectorUndoCapabilityTest.kt @@ -0,0 +1,108 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.spi + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import com.embabel.dice.proposition.PropositionStore +import com.embabel.dice.proposition.store.InMemoryPropositionRepository +import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.UriLocator +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class CollectorUndoCapabilityTest { + + @Test + fun `undo uses authoritative provenance replacement through a base store decorator`() { + val store = AppendPreservingStore() + val trace = InMemoryCollectorTraceStore() + val contextId = ContextId("ctx-base-store-undo") + val keep = ProvenanceEntry(UriLocator("https://example.com/keep")) + val folded = ProvenanceEntry(UriLocator("https://example.com/folded")) + val survivor = store.save(proposition("survivor", contextId, listOf(keep, folded))) + val retired = store.save( + proposition( + id = "retired", + contextId = contextId, + provenance = listOf(folded), + status = PropositionStatus.STALE, + ), + ) + val runId = "run-base-store-undo" + trace.recordRunContext(runId, contextId) + trace.recordDecision( + runId, + CollectorDecision( + runId = runId, + componentId = "component-base-store-undo", + survivorId = survivor.id, + action = "duplicate-merge", + retired = listOf( + RetiredProposition( + propositionId = retired.id, + priorStatus = PropositionStatus.ACTIVE, + foldedProvenanceRefs = listOf(folded.locator.key()), + ), + ), + ), + ) + + val result = undoSingleCollapse(trace, store, survivor.id, retired.id) + + assertEquals(listOf(keep), result?.survivor?.provenanceEntries) + assertEquals(listOf(keep), store.findById(survivor.id)?.provenanceEntries) + assertEquals(PropositionStatus.ACTIVE, result?.restored?.status) + } + + private fun proposition( + id: String, + contextId: ContextId, + provenance: List, + status: PropositionStatus = PropositionStatus.ACTIVE, + ) = Proposition( + id = id, + contextId = contextId, + text = "$id proposition", + mentions = emptyList(), + confidence = 0.9, + provenanceEntries = provenance, + status = status, + ) + + /** Models a persistent backend whose ordinary save path never removes unloaded evidence. */ + private class AppendPreservingStore( + private val delegate: InMemoryPropositionRepository = InMemoryPropositionRepository(), + ) : PropositionStore by delegate { + + override fun save(proposition: Proposition): Proposition { + val existing = delegate.findById(proposition.id) ?: return delegate.save(proposition) + return delegate.save( + proposition.copy( + provenanceEntries = (existing.provenanceEntries + proposition.provenanceEntries).distinct(), + ), + ) + } + + override fun setProvenance( + propositionId: String, + entries: List, + ): Proposition? = delegate.findById(propositionId)?.let { existing -> + delegate.save(existing.withProvenance(entries)) + } + } +} diff --git a/dice/src/test/kotlin/com/embabel/dice/web/rest/DiscoveryControllerTest.kt b/dice/src/test/kotlin/com/embabel/dice/web/rest/DiscoveryControllerTest.kt index b44ab35a..7cb1634f 100644 --- a/dice/src/test/kotlin/com/embabel/dice/web/rest/DiscoveryControllerTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/web/rest/DiscoveryControllerTest.kt @@ -26,6 +26,8 @@ import com.embabel.dice.proposition.MentionRole import com.embabel.dice.proposition.Proposition import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.proposition.store.InMemoryPropositionRepository +import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.UriLocator import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.kotlin.KotlinModule @@ -141,6 +143,70 @@ class DiscoveryControllerTest { .andExpect(status().isNotFound) } + @Test + fun `GET why includes empty provenance`() { + repository.save(proposition(id = "empty-provenance")) + + mockMvc.perform(get("/api/v1/contexts/$contextId/discovery/why/empty-provenance")) + .andExpect(status().isOk) + .andExpect(jsonPath("$.provenance").isArray) + .andExpect(jsonPath("$.provenance.length()").value(0)) + } + + @Test + fun `GET why includes revisionless provenance and omits null scalars`() { + val locator = UriLocator("https://example.com/revisionless") + repository.save( + proposition( + id = "revisionless", + provenance = listOf( + ProvenanceEntry( + locator = locator, + chunkId = "chunk-0", + startOffset = 2, + endOffset = 8, + contentHash = "hash-0", + ), + ), + ), + ) + + mockMvc.perform(get("/api/v1/contexts/$contextId/discovery/why/revisionless")) + .andExpect(status().isOk) + .andExpect(jsonPath("$.provenance[0].locator").value(locator.key())) + .andExpect(jsonPath("$.provenance[0].chunkId").value("chunk-0")) + .andExpect(jsonPath("$.provenance[0].startOffset").value(2)) + .andExpect(jsonPath("$.provenance[0].endOffset").value(8)) + .andExpect(jsonPath("$.provenance[0].contentHash").value("hash-0")) + .andExpect(jsonPath("$.provenance[0].sourceRevision").doesNotExist()) + } + + @Test + fun `GET why includes r1 and r2 provenance in order`() { + val locator = UriLocator("https://example.com/revisioned") + repository.save( + proposition( + id = "revisioned", + provenance = listOf( + ProvenanceEntry(locator = locator, sourceRevision = "r1"), + ProvenanceEntry(locator = locator, sourceRevision = "r2"), + ), + ), + ) + + mockMvc.perform(get("/api/v1/contexts/$contextId/discovery/why/revisioned")) + .andExpect(status().isOk) + .andExpect(jsonPath("$.provenance.length()").value(2)) + .andExpect(jsonPath("$.provenance[0].locator").value(locator.key())) + .andExpect(jsonPath("$.provenance[0].sourceRevision").value("r1")) + .andExpect(jsonPath("$.provenance[0].chunkId").doesNotExist()) + .andExpect(jsonPath("$.provenance[0].startOffset").doesNotExist()) + .andExpect(jsonPath("$.provenance[0].endOffset").doesNotExist()) + .andExpect(jsonPath("$.provenance[0].contentHash").doesNotExist()) + .andExpect(jsonPath("$.provenance[1].locator").value(locator.key())) + .andExpect(jsonPath("$.provenance[1].sourceRevision").value("r2")) + } + @Test fun `GET projection-health returns a per-target summary`() { mockMvc.perform(get("/api/v1/contexts/$contextId/discovery/projection-health")) @@ -254,4 +320,17 @@ class DiscoveryControllerTest { } if (fqn in forbiddenExact) offenders.add("$owner -> $fqn (domain/graph/store type)") } + + private fun proposition( + id: String, + provenance: List = emptyList(), + ): Proposition = + Proposition( + id = id, + contextId = ContextId(contextId), + text = "Fact $id", + mentions = emptyList(), + confidence = 0.9, + provenanceEntries = provenance, + ) } diff --git a/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt b/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt index 1df89616..cf586dff 100644 --- a/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt +++ b/dice/src/test/kotlin/com/embabel/dice/web/rest/PropositionPipelineControllerTest.kt @@ -20,6 +20,7 @@ import com.embabel.agent.core.DataDictionary import com.embabel.dice.common.EntityResolver import com.embabel.dice.common.NewEntity import com.embabel.dice.common.Resolutions +import com.embabel.dice.common.SourceAnalysisContext import com.embabel.dice.common.SuggestedEntity import com.embabel.dice.common.resolver.AlwaysCreateEntityResolver import com.embabel.dice.common.support.InMemorySchemaRegistry @@ -32,6 +33,8 @@ import com.embabel.dice.pipeline.PropositionPipeline import com.embabel.dice.pipeline.PropositionResults import com.embabel.dice.proposition.* import com.embabel.dice.proposition.revision.RevisionResult +import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.UriLocator import com.fasterxml.jackson.annotation.JsonClassDescription import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule @@ -39,9 +42,12 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule import io.mockk.every import io.mockk.mockk import io.mockk.verify +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.http.MediaType +import org.springframework.http.converter.StringHttpMessageConverter import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter import org.springframework.mock.web.MockMultipartFile import org.springframework.test.web.servlet.MockMvc @@ -92,7 +98,10 @@ class PropositionPipelineControllerTest { ) mockMvc = MockMvcBuilders.standaloneSetup(controller) - .setMessageConverters(MappingJackson2HttpMessageConverter(objectMapper)) + .setMessageConverters( + StringHttpMessageConverter(), + MappingJackson2HttpMessageConverter(objectMapper), + ) .build() } @@ -161,6 +170,249 @@ class PropositionPipelineControllerTest { .andExpect(jsonPath("$.entities.created").isArray) } + @Test + fun `POST extract carries every locator kind with an opaque revision and returns scalar revision`() { + val contextId = "test-context" + val sourceRevision = "rev:opaque|雪" + val proposition = Proposition( + contextId = ContextId(contextId), + text = "Revision-aware fact", + mentions = emptyList(), + confidence = 0.9, + provenanceEntries = listOf( + ProvenanceEntry( + locator = UriLocator("https://example.com/source"), + sourceRevision = sourceRevision, + ) + ), + ) + val result = ChunkPropositionResult.Success( + chunkId = "chunk-revision", + suggestedPropositions = SuggestedPropositions( + chunkId = "chunk-revision", + propositions = emptyList(), + ), + entityResolutions = Resolutions( + chunkIds = setOf("chunk-revision"), + resolutions = emptyList(), + ), + propositions = listOf(proposition), + revisionResults = emptyList(), + ) + every { + propositionPipeline.processChunk(any(), match { it.sourceRevision != null }) + } returns result + + val locatorCases = listOf( + """{"kind":"uri","value":"https://example.com/source"}""" to + "uri:https://example.com/source", + """{"kind":"file","value":"/vault/note.md","display":"Note"}""" to + "file:/vault/note.md", + """{"kind":"content","value":"sha256:abc"}""" to + "content:sha256:abc", + """{"kind":"connector","value":"message-42","connectorId":"gmail"}""" to + "connector:gmail:message-42", + ) + + locatorCases.forEach { (sourceLocator, expectedKey) -> + mockMvc.perform( + post("/api/v1/contexts/$contextId/extract") + .contentType(MediaType.APPLICATION_JSON) + .content( + """ + { + "text": "Revision-aware fact", + "sourceId": "legacy-parent", + "sourceLocator": $sourceLocator, + "sourceRevision": "$sourceRevision" + } + """.trimIndent() + ) + ) + .andExpect(status().isOk) + .andExpect(jsonPath("$.propositions[0].provenance[0].locator") + .value("uri:https://example.com/source")) + .andExpect(jsonPath("$.propositions[0].provenance[0].sourceRevision") + .value(sourceRevision)) + .andExpect(jsonPath("$.propositions[0].provenance[0].sourceKey").doesNotExist()) + + verify(exactly = 1) { + propositionPipeline.processChunk( + match { it.parentId == "legacy-parent" }, + match { + it.sourceLocator?.key() == expectedKey && + it.sourceRevision?.sourceKey == expectedKey && + it.sourceRevision?.sourceRevision == sourceRevision + }, + ) + } + } + } + + @Test + fun `POST extract replay persists one grounded provenance row per source revision`() { + val observedChunkIds = mutableListOf() + every { + propositionPipeline.processChunk(any(), any()) + } answers { + val chunk = firstArg() + val context = secondArg() + val provenanceEntry = ProvenanceEntry( + locator = requireNotNull(context.sourceLocator), + chunkId = chunk.id, + sourceRevision = context.sourceRevision?.sourceRevision, + ) + observedChunkIds += chunk.id + ChunkPropositionResult.Success( + chunkId = chunk.id, + suggestedPropositions = SuggestedPropositions( + chunkId = chunk.id, + propositions = emptyList(), + ), + entityResolutions = Resolutions( + chunkIds = setOf(chunk.id), + resolutions = emptyList(), + ), + propositions = listOf( + Proposition( + id = "fact-${chunk.id}", + contextId = context.contextId, + text = "Stable fact", + mentions = emptyList(), + confidence = 0.9, + grounding = listOf(chunk.id), + provenanceEntries = listOf(provenanceEntry), + ), + ), + revisionResults = emptyList(), + ) + } + + fun extract(revision: String) { + mockMvc.perform( + post("/api/v1/contexts/test-context/extract") + .contentType(MediaType.APPLICATION_JSON) + .content( + """ + { + "text": "Stable fact", + "sourceId": "logical-source", + "sourceLocator": {"kind":"uri","value":"https://example.com/source"}, + "sourceRevision": "$revision" + } + """.trimIndent() + ) + ).andExpect(status().isOk) + } + + extract("r1") + extract("r1") + extract("r2") + + assertEquals(observedChunkIds[0], observedChunkIds[1]) + assertNotEquals(observedChunkIds[0], observedChunkIds[2]) + + assertEquals(2, propositionRepository.count()) + val persistedR1 = requireNotNull( + propositionRepository.findById("fact-${observedChunkIds[0]}"), + ) + val persistedR2 = requireNotNull( + propositionRepository.findById("fact-${observedChunkIds[2]}"), + ) + assertEquals(1, persistedR1.grounding.size) + assertEquals(1, persistedR1.provenanceEntries.size) + assertEquals("r1", persistedR1.provenanceEntries.single().sourceRevision) + assertEquals(1, persistedR2.grounding.size) + assertEquals(1, persistedR2.provenanceEntries.size) + assertEquals("r2", persistedR2.provenanceEntries.single().sourceRevision) + } + + @Test + fun `POST extract rejects invalid locator unions and revision without locator before pipeline`() { + val invalidRequests = listOf( + """{"text":"fact","sourceRevision":"r1"}""", + """{"text":"fact","sourceLocator":{"kind":"unknown","value":"x"}}""", + """{"text":"fact","sourceLocator":{"kind":"uri","value":"https://example.com","connectorId":"gmail"}}""", + """{"text":"fact","sourceLocator":{"kind":"connector","value":"message-42"}}""", + """{"text":"fact","sourceLocator":{"kind":"connector","value":"c","connectorId":"a:b"}}""", + ) + + invalidRequests.forEach { requestBody -> + mockMvc.perform( + post("/api/v1/contexts/test-context/extract") + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody) + ).andExpect(status().isBadRequest) + } + + verify(exactly = 0) { propositionPipeline.processChunk(any(), any()) } + } + + @Test + fun `POST extract keeps old requests revisionless and omits response revision`() { + val proposition = Proposition( + contextId = ContextId("test-context"), + text = "Revisionless fact", + mentions = emptyList(), + confidence = 0.9, + provenanceEntries = listOf(ProvenanceEntry(locator = UriLocator("https://example.com/source"))), + ) + val result = ChunkPropositionResult.Success( + chunkId = "chunk-legacy", + suggestedPropositions = SuggestedPropositions( + chunkId = "chunk-legacy", + propositions = emptyList(), + ), + entityResolutions = Resolutions( + chunkIds = setOf("chunk-legacy"), + resolutions = emptyList(), + ), + propositions = listOf(proposition), + revisionResults = emptyList(), + ) + every { propositionPipeline.processChunk(any(), any()) } returns result + + mockMvc.perform( + post("/api/v1/contexts/test-context/extract") + .contentType(MediaType.APPLICATION_JSON) + .content("""{"text":"Revisionless fact","sourceId":"legacy-parent"}""") + ) + .andExpect(status().isOk) + .andExpect(jsonPath("$.propositions[0].provenance[0].sourceRevision").doesNotExist()) + + mockMvc.perform( + post("/api/v1/contexts/test-context/extract") + .contentType(MediaType.APPLICATION_JSON) + .content( + """ + { + "text": "Revisionless fact", + "sourceId": "locator-parent", + "sourceLocator": {"kind":"uri","value":"https://example.com/source"} + } + """.trimIndent() + ) + ) + .andExpect(status().isOk) + .andExpect(jsonPath("$.propositions[0].provenance[0].sourceRevision").doesNotExist()) + + verify(exactly = 1) { + propositionPipeline.processChunk( + match { it.parentId == "legacy-parent" }, + match { it.sourceLocator == null && it.sourceRevision == null }, + ) + } + verify(exactly = 1) { + propositionPipeline.processChunk( + match { it.parentId == "locator-parent" }, + match { + it.sourceLocator?.key() == "uri:https://example.com/source" && + it.sourceRevision == null + }, + ) + } + } + @Test fun `POST extract saves propositions to repository`() { val contextId = "test-context" @@ -306,7 +558,10 @@ class PropositionPipelineControllerTest { contentChunker = chunker, ) val mvc = MockMvcBuilders.standaloneSetup(fileController) - .setMessageConverters(MappingJackson2HttpMessageConverter(objectMapper)) + .setMessageConverters( + StringHttpMessageConverter(), + MappingJackson2HttpMessageConverter(objectMapper), + ) .build() mvc.perform( @@ -317,7 +572,115 @@ class PropositionPipelineControllerTest { .andExpect(jsonPath("$.chunksProcessed").value(2)) .andExpect(jsonPath("$.entities.failed[0]").value("chunk-bad")) - verify(exactly = 1) { propositionPipeline.process(any(), any()) } + verify(exactly = 1) { + propositionPipeline.process( + any(), + match { it.sourceLocator == null && it.sourceRevision == null }, + ) + } verify(exactly = 0) { propositionPipeline.processChunk(any(), any()) } } + + @Test + fun `POST extract file rejects revision without locator before reading or pipeline`() { + val reader = mockk() + val chunker = mockk() + val fileController = PropositionPipelineController( + propositionPipeline = propositionPipeline, + propositionRepository = propositionRepository, + entityResolver = entityResolver, + schemaRegistry = schemaRegistry, + contentReader = reader, + contentChunker = chunker, + objectMapper = objectMapper, + ) + val mvc = MockMvcBuilders.standaloneSetup(fileController) + .setMessageConverters( + StringHttpMessageConverter(), + MappingJackson2HttpMessageConverter(objectMapper), + ) + .build() + + mvc.perform( + multipart("/api/v1/contexts/test-context/extract/file") + .file(MockMultipartFile("file", "empty.txt", "text/plain", byteArrayOf())) + .file(MockMultipartFile("sourceRevision", "", "text/plain", "r1".toByteArray())) + ).andExpect(status().isBadRequest) + + verify(exactly = 0) { reader.parseContent(any(), any()) } + verify(exactly = 0) { propositionPipeline.process(any(), any()) } + } + + @Test + fun `POST extract file carries locator and opaque revision through batch path`() { + val contextId = "test-context" + val sourceRevision = "mailbox:v2|雪" + val reader = mockk() + val chunker = mockk() + val document = mockk(relaxed = true) + every { reader.parseContent(any(), any()) } returns document + every { chunker.chunk(any()) } returns listOf( + Chunk.create(text = "mail content", parentId = "legacy-source"), + ) + val processedChunkIds = mutableListOf>() + every { + propositionPipeline.process(any(), match { it.sourceRevision != null }) + } answers { + processedChunkIds += firstArg>().map { it.id } + PropositionResults(chunkResults = emptyList(), allPropositions = emptyList()) + } + + val fileController = PropositionPipelineController( + propositionPipeline = propositionPipeline, + propositionRepository = propositionRepository, + entityResolver = entityResolver, + schemaRegistry = schemaRegistry, + contentReader = reader, + contentChunker = chunker, + objectMapper = objectMapper, + ) + val mvc = MockMvcBuilders.standaloneSetup(fileController) + .setMessageConverters( + StringHttpMessageConverter(), + MappingJackson2HttpMessageConverter(objectMapper), + ) + .build() + + fun revisionedRequest() = + multipart("/api/v1/contexts/$contextId/extract/file") + .file(MockMultipartFile("file", "mail.txt", "text/plain", "content".toByteArray())) + .file(MockMultipartFile("sourceId", "", "text/plain", "legacy-source".toByteArray())) + .file( + MockMultipartFile( + "sourceLocator", + "", + "application/json", + """{"kind":"connector","value":"message-42","connectorId":"gmail"}""".toByteArray(), + ) + ) + .file( + MockMultipartFile( + "sourceRevision", + "", + "text/plain;charset=UTF-8", + sourceRevision.toByteArray(), + ) + ) + + repeat(2) { + mvc.perform(revisionedRequest()).andExpect(status().isOk) + } + + assertEquals(processedChunkIds[0], processedChunkIds[1]) + verify(exactly = 2) { + propositionPipeline.process( + match { chunks -> chunks.all { it.parentId == "legacy-source" } }, + match { + it.sourceLocator?.key() == "connector:gmail:message-42" && + it.sourceRevision?.sourceKey == "connector:gmail:message-42" && + it.sourceRevision?.sourceRevision == sourceRevision + }, + ) + } + } } diff --git a/dice/src/test/resources/compat/source-revision-legacy-client.jar b/dice/src/test/resources/compat/source-revision-legacy-client.jar new file mode 100644 index 00000000..d1ea31e8 Binary files /dev/null and b/dice/src/test/resources/compat/source-revision-legacy-client.jar differ diff --git a/dice/src/test/resources/compat/source-revision-legacy-client.kt b/dice/src/test/resources/compat/source-revision-legacy-client.kt new file mode 100644 index 00000000..a2230861 --- /dev/null +++ b/dice/src/test/resources/compat/source-revision-legacy-client.kt @@ -0,0 +1,92 @@ +/* + * Compiled against the unmodified origin/main dice jar, then run with the candidate jar. + * + * LINKED proves an unchanged approved JVM constructor descriptor. Both constructor probes use + * fully specified arguments so this fixture makes no claim about Kotlin's synthetic default + * constructor ABI. NoSuchMethodError on the old copy descriptors is expected and deliberately + * records, rather than broadens, the approved source/JSON/Java compatibility boundary. + */ +package com.embabel.dice.compat + +import com.embabel.agent.core.ContextId +import com.embabel.agent.core.DataDictionary +import com.embabel.dice.common.Relations +import com.embabel.dice.common.SourceAnalysisContext +import com.embabel.dice.common.resolver.AlwaysCreateEntityResolver +import com.embabel.dice.provenance.ContentAddressedLocator +import com.embabel.dice.provenance.ProvenanceEntry + +fun main() { + val locator = ContentAddressedLocator("legacy-client", null) + + val provenance = probe("ProvenanceEntry.constructor.full") { + ProvenanceEntry(locator, "chunk", 0, 5, "hash") + } + probe("ProvenanceEntry.constructor.nullable") { + ProvenanceEntry(locator, null, null, null, null) + } + provenance?.let { entry -> + probe("ProvenanceEntry.copy.direct") { + entry.copy(locator, "chunk-2", 0, 7, "hash-2") + } + probe("ProvenanceEntry.copy.default") { + entry.copy(contentHash = "hash-2") + } + } + + val context = probe("SourceAnalysisContext.constructor.full") { + SourceAnalysisContext( + DataDictionary.fromClasses("legacy-client"), + AlwaysCreateEntityResolver, + ContextId("legacy-client"), + emptyList(), + Relations.empty(), + emptyMap(), + null, + null, + false, + emptyMap(), + ) + } + probe("SourceAnalysisContext.constructor.alternate") { + SourceAnalysisContext( + DataDictionary.fromClasses("legacy-client-defaults"), + AlwaysCreateEntityResolver, + ContextId("legacy-client-defaults"), + emptyList(), + Relations.empty(), + emptyMap(), + null, + null, + false, + emptyMap(), + ) + } + context?.let { analysisContext -> + probe("SourceAnalysisContext.copy.direct") { + analysisContext.copy( + analysisContext.schema, + analysisContext.entityResolver, + analysisContext.contextId, + analysisContext.knownEntities, + analysisContext.relations, + analysisContext.promptVariables, + analysisContext.sourceLocator, + analysisContext.perspective, + analysisContext.mintNewEntities, + analysisContext.mintedEntityProperties, + ) + } + probe("SourceAnalysisContext.copy.default") { + analysisContext.copy(promptVariables = mapOf("legacy" to true)) + } + } +} + +private fun probe(name: String, call: () -> T): T? = + try { + call().also { println("$name=LINKED") } + } catch (error: LinkageError) { + println("$name=${error::class.java.simpleName}") + null + } diff --git a/dice/src/test/resources/provenance/proposition-revisioned.json b/dice/src/test/resources/provenance/proposition-revisioned.json new file mode 100644 index 00000000..3504f994 --- /dev/null +++ b/dice/src/test/resources/provenance/proposition-revisioned.json @@ -0,0 +1,49 @@ +{ + "id": "revisioned", + "contextId": "ctx-json", + "text": "A revisioned proposition", + "mentions": [], + "confidence": 0.8, + "decay": 0.0, + "importance": 0.5, + "reasoning": null, + "grounding": [], + "created": "2026-01-02T00:00:00Z", + "contentRevised": "2026-01-02T00:00:00Z", + "metadataRevised": "2026-01-02T00:00:00Z", + "pinned": false, + "lastAccessed": "2026-01-02T00:00:00Z", + "status": "ACTIVE", + "level": 0, + "sourceIds": [], + "reinforceCount": 0, + "metadata": {}, + "uri": null, + "temporal": null, + "provenanceEntries": [ + { + "locator": { + "kind": "uri", + "uri": "https://example.com/revisioned", + "display": "Revisioned source" + }, + "chunkId": "chunk-r1", + "startOffset": 2, + "endOffset": 12, + "contentHash": "hash-r1", + "sourceRevision": "r1" + }, + { + "locator": { + "kind": "uri", + "uri": "https://example.com/revisioned", + "display": "Revisioned source" + }, + "chunkId": "chunk-literal-null", + "startOffset": 13, + "endOffset": 25, + "contentHash": "hash-null", + "sourceRevision": "null" + } + ] +} diff --git a/dice/src/test/resources/provenance/proposition-revisionless.json b/dice/src/test/resources/provenance/proposition-revisionless.json new file mode 100644 index 00000000..6ff1265d --- /dev/null +++ b/dice/src/test/resources/provenance/proposition-revisionless.json @@ -0,0 +1,36 @@ +{ + "id": "revisionless", + "contextId": "ctx-json", + "text": "A revisionless proposition", + "mentions": [], + "confidence": 0.75, + "decay": 0.0, + "importance": 0.5, + "reasoning": null, + "grounding": [], + "created": "2026-01-01T00:00:00Z", + "contentRevised": "2026-01-01T00:00:00Z", + "metadataRevised": "2026-01-01T00:00:00Z", + "pinned": false, + "lastAccessed": "2026-01-01T00:00:00Z", + "status": "ACTIVE", + "level": 0, + "sourceIds": [], + "reinforceCount": 0, + "metadata": {}, + "uri": null, + "temporal": null, + "provenanceEntries": [ + { + "locator": { + "kind": "uri", + "uri": "https://example.com/legacy", + "display": "Legacy source" + }, + "chunkId": "chunk-legacy", + "startOffset": 0, + "endOffset": 10, + "contentHash": "hash-legacy" + } + ] +}