otel: make Go/Python/TS/.NET tool spans survive ingest, and stop the zero cost

PARITY-STATUS.md

@@ -70,13 +70,31 @@ All five servers carry the transport core: frame dispatch, per-turn engine, sess
 | Shared scenario conformance corpus | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Postgres conversation store | ✅ | ✅ | ✅ | ✅ | ✅ |
 | Server `gen_ai.*` OTel telemetry (chat + tool spans · redacted tool args · env-gated OTLP) | ✅ | ✅ | ✅ | ✅ | ✅ |
+| └ ingest-joinable spans (`gen_ai.operation.name` · self-identifying tool spans) [^ingest] | ✅ | ✅ | ✅ | ✅ | ✅ |
+| └ cost on the span (`gen_ai.usage.cost_usd` / `smooai.gen_ai.cost_unavailable`) | ✅ | ✅ | ✅ | ✅ | ✅ |
+| └ usage/cost provenance + `gen_ai.response.id` [^provenance] | ✅ | — | — | — | — |
 | Second storage backend (DynamoDB + S3 Vectors) | ✅ | — | — | — | — |
 | Persistent checkpoint / knowledge / ACL-knowledge stores [^knowledge] | ✅ | ✅ | ◐ | ◐ | ◐ |
 | Deep ingestion + ACL surface | ✅ | ✅ | ◐ | ◐ | ◐ |
 | Backplane `attach`/`detach` | ✅ | — | ✅ | ✅ | ✅ |
 | Backplane `publish` (event fan-out) | ✅ | — | ✅ | ✅ | — |
 | **Cross-pod backplane (Redis / NATS)** | ✅ | — | — | — | — |
 
+[^ingest]: The api-prime OTLP ingest builds a span's attribute set from the resource
+    attrs plus **that span's own**, with no inheritance from the parent. A tool span
+    without its own `gen_ai.system` therefore fails the ingest's LLM-event gate and is
+    **discarded** — Rust's were, for their entire existence (zero rows with
+    `operation_name = 'tool'`, all time). `gen_ai.operation.name` must be literally
+    `"chat"` / `"tool"`: the ingest takes the attribute verbatim when present and only
+    derives it from the span name as a fallback, so any other spelling lands in the
+    column and matches nothing.
+
+[^provenance]: Rust-only because it needs engine support that exists only in the Rust
+    core (1.10.0): `usage_estimated` / `cost_estimated` on `AgentEvent::Completed`, plus
+    capturing the gateway's `chatcmpl-…` response id. Until the other four cores carry
+    the same fields, those engines cannot distinguish a measured token count from an
+    estimated one, and have no join key to `LiteLLM_SpendLogs`. Tracked in th-73c8b5.
+
 [^knowledge]: The durable **knowledge + ACL-knowledge** Postgres stores now ship in **Go, TypeScript and Python** too ([PRs #442, #443, #444](https://github.com/SmooAI/smooth-operator/pulls)) — on the shared `knowledge_vectors` table + pgvector — alongside Rust and .NET (which already had them, e.g. Python's `postgres_knowledge.py`: `PostgresVectorKnowledge` / `PostgresAclKnowledge`). These three cells stay **◐ rather than ✅** for two honest reasons: the persistent **checkpoint** store is still pending in Go/TS/Python, and the **TS + Python** knowledge stores are shipped and contract-tested but **not yet wired into the live dispatcher** (a sync-engine vs async-pg bridge). The **second storage backend** (DynamoDB + S3-Vectors, the row above) remains Rust-only.
 
 **The operational consequence:** only the **Rust** server scales past one replica today. Go, TypeScript and Python run an in-memory backplane — correct for a single process, silently wrong the moment you run two pods, because an event published on pod A never reaches a socket held by pod B. C# has no backplane surface at all.

dotnet/server/src/Telemetry.cs

@@ -40,6 +40,46 @@ public static class Telemetry
     public const string GenAiToolArguments = "gen_ai.tool.call.arguments";
     public const string GenAiAgentName = "gen_ai.agent.name";
 
+    /// <summary>
+    /// <c>gen_ai.operation.name</c> — the operation a span represents.
+    /// <para>
+    /// The api-prime OTLP ingest takes this attribute VERBATIM when present and only derives it
+    /// from the span name as a fallback, and its queries filter on <c>operation_name = 'tool'</c>.
+    /// So the values must be exactly <see cref="OperationChat"/> / <see cref="OperationTool"/> —
+    /// a spelling like <c>execute_tool</c> would land in the column and match nothing.
+    /// </para></summary>
+    public const string GenAiOperationName = "gen_ai.operation.name";
+
+    /// <summary>
+    /// <c>gen_ai.usage.cost_usd</c> — the turn's cost in USD.
+    /// <para>
+    /// Recorded ONLY when positive. A zero is ambiguous: the gateway answers 0 for a model it has
+    /// no price for, and local pricing returns the free tier for anything it does not recognise, so
+    /// a zero means "not measured", never "free". Exporting it would render a paid turn as a
+    /// confident $0.00.
+    /// </para></summary>
+    public const string GenAiUsageCostUsd = "gen_ai.usage.cost_usd";
+
+    /// <summary><c>smooai.gen_ai.cost_unavailable</c> — why <see cref="GenAiUsageCostUsd"/> is
+    /// absent. Set INSTEAD of the cost, never alongside it. Same attribute name and values across
+    /// every engine so a consumer never special-cases per language.</summary>
+    public const string CostUnavailable = "smooai.gen_ai.cost_unavailable";
+
+    /// <summary><see cref="CostUnavailable"/> value: no price could be established.</summary>
+    public const string CostUnavailableUnpriced = "unpriced";
+
+    /// <summary><c>smooai.org_id</c> — the owning org, matching every other engine.
+    /// <para>ponytail: declared here but never set — the .NET server has no org concept at all
+    /// (no <c>orgId</c> anywhere in <c>dotnet/server/src</c>), so wiring it is a plumbing change
+    /// through FrameDispatcher, not a span tag. Set it here once that exists.</para></summary>
+    public const string SmooaiOrgId = "smooai.org_id";
+
+    /// <summary><see cref="GenAiOperationName"/> value on a <see cref="SpanChat"/> span.</summary>
+    public const string OperationChat = "chat";
+
+    /// <summary><see cref="GenAiOperationName"/> value on a <see cref="SpanTool"/> span.</summary>
+    public const string OperationTool = "tool";
+
     /// <summary>Span name for the per-turn GenAI chat span (<c>gen_ai.chat</c>).</summary>
     public const string SpanChat = "gen_ai.chat";
 

dotnet/server/src/TurnRunner.cs

@@ -146,13 +146,20 @@ private static string ConfiguredModel() =>
     /// <summary>Emit a <c>gen_ai.tool</c> child span (parented to the ambient turn span) for one tool
     /// call, carrying the tool name and its redacted JSON arguments. No-op when nothing is sampling
     /// <see cref="Telemetry.Source"/> (<c>StartActivity</c> returns null).</summary>
-    private static void EmitToolSpan(FunctionCallContent call)
+    private static void EmitToolSpan(FunctionCallContent call, string conversationId)
     {
         using var toolSpan = Telemetry.Source.StartActivity(Telemetry.SpanTool);
         if (toolSpan is null)
         {
             return;
         }
+        // The OTLP ingest builds a span's attributes from the resource attrs plus THAT span's own,
+        // with no inheritance from the parent — so a child repeats its identifiers or it cannot be
+        // joined. Omitting gen_ai.system is worse than losing the join: the ingest's LLM-event gate
+        // keys on it, so bare tool spans are DISCARDED. Rust's were, for their entire existence.
+        toolSpan.SetTag(Telemetry.GenAiSystem, Telemetry.SystemName);
+        toolSpan.SetTag(Telemetry.GenAiOperationName, Telemetry.OperationTool);
+        toolSpan.SetTag(Telemetry.GenAiConversationId, conversationId);
         toolSpan.SetTag(Telemetry.GenAiToolName, call.Name);
         var args = call.Arguments is null ? "{}" : JsonSerializer.Serialize(call.Arguments);
         toolSpan.SetTag(Telemetry.GenAiToolArguments, Telemetry.RedactToolArguments(args));
@@ -226,6 +233,7 @@ public async Task<TurnResult> RunAsync(string conversationId, string requestId,
         // host (env-gated on OTEL_EXPORTER_OTLP_ENDPOINT) or a test's ActivityListener.
         using var turnActivity = Telemetry.Source.StartActivity(Telemetry.SpanChat);
         turnActivity?.SetTag(Telemetry.GenAiSystem, Telemetry.SystemName);
+        turnActivity?.SetTag(Telemetry.GenAiOperationName, Telemetry.OperationChat);
         turnActivity?.SetTag(Telemetry.GenAiRequestModel, ConfiguredModel());
         turnActivity?.SetTag(Telemetry.GenAiConversationId, conversationId);
         turnActivity?.SetTag(Telemetry.GenAiAgentName, Telemetry.AgentName);
@@ -480,7 +488,7 @@ public async Task<TurnResult> RunAsync(string conversationId, string requestId,
                             toolNames[call.CallId] = call.Name;
                             // `gen_ai.tool` child span (nests under the turn span), mirroring the Rust
                             // runner emitting one gen_ai.tool span per tool call with redacted args.
-                            EmitToolSpan(call);
+                            EmitToolSpan(call, conversationId);
                             // DEFER a confirmation-gated tool's toolCall chunk: it is emitted from the
                             // gate AFTER write_confirmation_required, so the wire order matches the
                             // canonical (Rust) server. Non-gated tools emit their chunk inline as before.
@@ -516,12 +524,31 @@ public async Task<TurnResult> RunAsync(string conversationId, string requestId,
             _interactionPark?.Clear(sessionId);
         }
 
-        // Record token usage on the turn span (omitted when the engine reported none, per the GenAI
-        // conventions), mirroring the Rust runner's turn_span.record of the usage fields.
-        if (turnActivity is not null && sawUsage)
+        // Token counts and cost on the turn span, recording only what was actually measured.
+        //
+        // `sawUsage` alone was NOT enough: a usage chunk carrying null counts sets it while both
+        // totals resolve to 0 via `?? 0`, so this published `input_tokens = 0` on a grounded turn.
+        // Every other engine guards on the counts themselves. Absent is honest; 0 is a lie.
+        if (turnActivity is not null)
         {
-            turnActivity.SetTag(Telemetry.GenAiUsageInputTokens, promptTokens);
-            turnActivity.SetTag(Telemetry.GenAiUsageOutputTokens, completionTokens);
+            if (sawUsage && (promptTokens > 0 || completionTokens > 0))
+            {
+                turnActivity.SetTag(Telemetry.GenAiUsageInputTokens, promptTokens);
+                turnActivity.SetTag(Telemetry.GenAiUsageOutputTokens, completionTokens);
+            }
+
+            // Cost is judged separately from the counts: the gateway reports it on an HTTP header
+            // while usage arrives on an SSE chunk, so either can turn up without the other. A
+            // non-positive cost becomes an explicit "unpriced" marker, never a confident $0.00.
+            var costUsd = TurnUsageFrom(agent, sawUsage, promptTokens, completionTokens)?.CostUsd ?? 0;
+            if (costUsd > 0 && double.IsFinite(costUsd))
+            {
+                turnActivity.SetTag(Telemetry.GenAiUsageCostUsd, costUsd);
+            }
+            else
+            {
+                turnActivity.SetTag(Telemetry.CostUnavailable, Telemetry.CostUnavailableUnpriced);
+            }
         }
 
         // 5. Persist the outbound reply.

dotnet/server/tests/TelemetryTests.cs

@@ -121,6 +121,26 @@ public async Task StreamingTurnEmitsGenAiSpansWithModelAndToolArgs()
         var args = toolSpan.GetTagItem(Telemetry.GenAiToolArguments) as string ?? string.Empty;
         Assert.Contains("return policy refund window", args);
         Assert.Equal(chatSpan.Id, toolSpan.ParentId);
+
+        // Being a child is NOT enough. The OTLP ingest builds a span's attributes from
+        // the resource attrs plus THAT span's own, with no parent inheritance, so the
+        // tool span repeats the identifiers itself — and without gen_ai.system it fails
+        // the ingest's LLM-event gate outright and is discarded, which is what happened
+        // to Rust's tool spans for their entire existence.
+        Assert.Equal(Telemetry.SystemName, toolSpan.GetTagItem(Telemetry.GenAiSystem));
+        Assert.Equal(Telemetry.OperationTool, toolSpan.GetTagItem(Telemetry.GenAiOperationName));
+        Assert.Equal(conversationId, toolSpan.GetTagItem(Telemetry.GenAiConversationId));
+
+        // Must be exactly "chat"/"tool" — the ingest takes the attribute verbatim when
+        // present and its queries filter on operation_name = 'tool'.
+        Assert.Equal(Telemetry.OperationChat, chatSpan.GetTagItem(Telemetry.GenAiOperationName));
+
+        // Cost: exactly one of the two is ever set. This scripted turn is unpriced, so
+        // the marker must be there INSTEAD of a $0.00 — a missing price must never read
+        // as free. Before this, .NET was the one engine that shipped a literal
+        // `new TurnUsage(0, ...)` on the fallback path.
+        Assert.Null(chatSpan.GetTagItem(Telemetry.GenAiUsageCostUsd));
+        Assert.Equal(Telemetry.CostUnavailableUnpriced, chatSpan.GetTagItem(Telemetry.CostUnavailable));
     }
 
     [Fact]

go/server/telemetry.go

@@ -42,6 +42,34 @@ const (
 	// SmooaiOrgID is `smooai.org_id` — the owning org. Matches the monorepo TS chat
 	// handler's attribute exactly so the observability studio groups Rust + Go turns by org.
 	SmooaiOrgID = "smooai.org_id"
+	// GenAIOperationName is `gen_ai.operation.name` — the operation a span represents.
+	//
+	// The api-prime OTLP ingest takes this attribute VERBATIM when present and only
+	// derives it from the span name as a fallback, and its queries filter on
+	// `operation_name = 'tool'`. So the values must be exactly OperationChat /
+	// OperationTool — a spelling like "execute_tool" would land in the column and
+	// match nothing.
+	GenAIOperationName = "gen_ai.operation.name"
+	// GenAIUsageCostUSD is `gen_ai.usage.cost_usd` — the turn's cost in USD.
+	//
+	// Recorded ONLY when positive. A zero is ambiguous: the gateway answers 0 for a
+	// model it has no price for, and local pricing returns the free tier for anything
+	// it does not recognise, so a zero means "not measured", never "free". Exporting
+	// it would render a paid turn as a confident $0.00.
+	GenAIUsageCostUSD = "gen_ai.usage.cost_usd"
+	// CostUnavailable is `smooai.gen_ai.cost_unavailable` — why GenAIUsageCostUSD is
+	// absent. Set INSTEAD of the cost, never alongside it. Same attribute name and
+	// values across every engine so a consumer never special-cases per language.
+	CostUnavailable = "smooai.gen_ai.cost_unavailable"
+	// CostUnavailableUnpriced is the CostUnavailable value for "no price could be
+	// established for this model".
+	CostUnavailableUnpriced = "unpriced"
+)
+
+// OperationChat / OperationTool are the GenAIOperationName values.
+const (
+	OperationChat = "chat"
+	OperationTool = "tool"
 )
 
 // SystemName is emitted for GenAISystem and used as the tracer + service name.

go/server/telemetry_test.go

@@ -112,6 +112,34 @@ func TestStreamingTurnEmitsGenAISpans(t *testing.T) {
 		t.Errorf("gen_ai.tool span should be a child of gen_ai.chat; parent=%s chat=%s",
 			tool.Parent.SpanID(), chat.SpanContext.SpanID())
 	}
+
+	// Being a child is NOT enough. The OTLP ingest builds a span's attributes from the
+	// resource attrs plus THAT span's own, with no parent inheritance, so the tool span
+	// repeats the identifiers itself — and without gen_ai.system it fails the ingest's
+	// LLM-event gate outright and is discarded, which is what happened to Rust's tool
+	// spans for their entire existence (zero rows with operation_name='tool', all time).
+	assertAttr(t, tool.Attributes, GenAISystem, SystemName)
+	assertAttr(t, tool.Attributes, GenAIOperationName, OperationTool)
+	assertAttr(t, tool.Attributes, GenAIConversationID, session.ConversationID)
+	assertAttr(t, tool.Attributes, SmooaiOrgID, "org-telemetry")
+
+	// Must be exactly "chat"/"tool" — the ingest takes the attribute verbatim when
+	// present and its queries filter on operation_name = 'tool'.
+	assertAttr(t, chat.Attributes, GenAIOperationName, OperationChat)
+
+	// Cost: exactly one of the two is ever set. The mock turn IS priced (local
+	// ModelPricing knows openai/gpt-4o), so the cost lands and the marker must not —
+	// a zero must never be exported as a real cost, and a real cost must never carry
+	// an "unavailable" marker beside it.
+	cost, hasCost := attr(chat.Attributes, GenAIUsageCostUSD)
+	if !hasCost {
+		t.Errorf("a priced turn must record %s; got attrs %+v", GenAIUsageCostUSD, chat.Attributes)
+	} else if cost == "0" || cost == "0.000000" {
+		t.Errorf("%s must never be exported as zero — that means unpriced, not free", GenAIUsageCostUSD)
+	}
+	if _, ok := attr(chat.Attributes, CostUnavailable); ok {
+		t.Errorf("%s must not be set alongside a real cost", CostUnavailable)
+	}
 }
 
 func assertAttr(t *testing.T, kvs []attribute.KeyValue, key, want string) {

go/server/turn_runner.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"math"
 	"os"
 	"strings"
 	"sync/atomic"
@@ -200,6 +201,7 @@ func (r *TurnRunner) Run(ctx context.Context, sessionID, conversationID, request
 	tr := otel.Tracer(SystemName)
 	ctx, turnSpan := tr.Start(ctx, SpanChat, oteltrace.WithAttributes(
 		attribute.String(GenAISystem, SystemName),
+		attribute.String(GenAIOperationName, OperationChat),
 		attribute.String(GenAIRequestModel, r.spanModel()),
 		attribute.String(GenAIConversationID, conversationID),
 		attribute.String(GenAIAgentName, AgentName),
@@ -444,19 +446,43 @@ consume:
 	// GenAI conventions) and emit one `gen_ai.tool` child span per tool call — carrying
 	// the redacted arguments, measured latency, and an ERROR status on failure. Mirrors the
 	// Rust runner's post-turn span emission.
-	if usage != nil && (usage.PromptTokens > 0 || usage.CompletionTokens > 0) {
-		turnSpan.SetAttributes(
-			attribute.Int(GenAIUsageInputTokens, usage.PromptTokens),
-			attribute.Int(GenAIUsageOutputTokens, usage.CompletionTokens),
-		)
+	if usage != nil {
+		// Counts are omitted when the engine reported none: absent is honest, 0 is a
+		// lie (a grounded turn always consumes prompt tokens).
+		if usage.PromptTokens > 0 || usage.CompletionTokens > 0 {
+			turnSpan.SetAttributes(
+				attribute.Int(GenAIUsageInputTokens, usage.PromptTokens),
+				attribute.Int(GenAIUsageOutputTokens, usage.CompletionTokens),
+			)
+		}
+		// Cost is judged separately from the counts: the gateway reports it on an HTTP
+		// header while usage arrives on an SSE chunk, so either can turn up without the
+		// other. A non-positive cost becomes an explicit "unpriced" marker, never $0.00.
+		if usage.CostUSD > 0 && !math.IsInf(usage.CostUSD, 0) && !math.IsNaN(usage.CostUSD) {
+			turnSpan.SetAttributes(attribute.Float64(GenAIUsageCostUSD, usage.CostUSD))
+		} else {
+			turnSpan.SetAttributes(attribute.String(CostUnavailable, CostUnavailableUnpriced))
+		}
 	}
 	for _, rec := range toolRecords {
-		_, toolSpan := tr.Start(ctx, SpanTool, oteltrace.WithAttributes(
+		// The OTLP ingest builds a span's attributes from the resource attrs plus THAT
+		// span's own, with no inheritance from the parent — so a child repeats its
+		// identifiers or it cannot be joined. Omitting gen_ai.system is worse than
+		// losing the join: the ingest's LLM-event gate keys on it, so bare tool spans
+		// are DISCARDED. Rust's were, for their entire existence.
+		toolAttrs := []attribute.KeyValue{
+			attribute.String(GenAISystem, SystemName),
+			attribute.String(GenAIOperationName, OperationTool),
+			attribute.String(GenAIConversationID, conversationID),
 			attribute.String(GenAIToolName, rec.name),
 			attribute.String(GenAIToolArguments, redactToolArguments(rec.arguments)),
 			attribute.Int64("duration_ms", rec.durationMs),
 			attribute.Bool("is_error", rec.isError),
-		))
+		}
+		if r.orgID != "" {
+			toolAttrs = append(toolAttrs, attribute.String(SmooaiOrgID, r.orgID))
+		}
+		_, toolSpan := tr.Start(ctx, SpanTool, oteltrace.WithAttributes(toolAttrs...))
 		if rec.isError {
 			toolSpan.SetStatus(codes.Error, rec.errText)
 		}