From e32d6bfb7c81e7404fd756d1c5dce863cfedd53b Mon Sep 17 00:00:00 2001 From: Alan Marx Date: Fri, 31 Jul 2026 13:11:20 -0700 Subject: [PATCH 1/4] feat(testing): add Langfuse::Testing module for in-memory span capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces `require "langfuse/testing"` + `include Langfuse::Testing` as a supported, public way to assert against emitted spans in tests without real network calls or reaching into private SDK internals. - `OtelSetup.test_mode` flag causes `build_exporter` to return a fresh `InMemorySpanExporter` on each provider build; survives `shutdown` cycles - `Langfuse::Testing#emitted_langfuse_spans` — flushes and returns recorded spans - `Langfuse::Testing#reset_langfuse` — flushes and clears the span buffer - `docs/TESTING.md` covers RSpec and Minitest integration patterns Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 3 + docs/TESTING.md | 108 +++++++++++++++++++++++++++++++ lib/langfuse/otel_setup.rb | 22 ++++++- lib/langfuse/testing.rb | 31 +++++++++ spec/langfuse/otel_setup_spec.rb | 16 +++++ spec/langfuse/testing_spec.rb | 38 +++++++++++ 6 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 docs/TESTING.md create mode 100644 lib/langfuse/testing.rb create mode 100644 spec/langfuse/testing_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a2a4f..05e2bf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- `Langfuse::Testing` module (`require "langfuse/testing"`) — include in test classes to capture spans in memory without network calls or private ivar access; provides `emitted_langfuse_spans` and `reset_langfuse` instance methods + ## [0.10.1] - 2026-05-05 ### Changed diff --git a/docs/TESTING.md b/docs/TESTING.md new file mode 100644 index 0000000..15cd1e3 --- /dev/null +++ b/docs/TESTING.md @@ -0,0 +1,108 @@ +# Testing Guide + +This guide covers how to safely incorporate Langfuse into your application test suite. + +## Overview + +Langfuse can be configured to use an in-memory exporter in your test environemnt so that you can run real code end-to-end without making real network requests. This allows you to exercise the full Langfuse observation pipeline and write assertions against produced spans. Stubs break as the SDK evolves and they test the wrong thing — that *a method was called*, not that *the right telemetry was produced*. By asserting against real spans your tests can catch regressions iinvolving span naming, attribute values, and export filtering that stubs would silently miss. + +## Setup + +Add `require "langfuse/testing"` to your test helper. Requiring the file automatically wires an in-memory exporter into the SDK — no extra configuration beyond the usual keys. + +```ruby +require "langfuse/testing" + +Langfuse.configure do |config| + config.public_key = "pk-test" + config.secret_key = "sk-test" +end +``` + +Then include `Langfuse::Testing` wherever you need span assertions. It provides two instance methods: + +| Method | Description | +|---|---| +| `emitted_langfuse_spans` | Instance method — flushes pending batches and returns all recorded spans | +| `reset_langfuse` | Module method — flushes and discards all recorded spans; call this in test setup | + +## RSpec + +```ruby +# spec/support/langfuse_helpers.rb +require "langfuse/testing" + +Langfuse.configure do |config| + config.public_key = "pk-test" + config.secret_key = "sk-test" +end + +RSpec.configure do |config| + config.include Langfuse::Testing, :langfuse + config.before(:each, :langfuse) { reset_langfuse } +end +``` + +Tag examples that need span assertions with `:langfuse`: + +```ruby +RSpec.describe SummarizationService, :langfuse do + it "emits a generation span" do + SummarizationService.new.call("hello world") + + span = emitted_langfuse_spans.find { |s| s.name == "summarize" } + expect(span).not_to be_nil + expect(span.attributes["langfuse.observation.type"]).to eq("generation") + end +end +``` + +## Minitest + +```ruby +# test/support/langfuse_test_helper.rb +require "langfuse/testing" + +Langfuse.configure do |config| + config.public_key = "pk-test" + config.secret_key = "sk-test" +end + +module LangfuseTestHelper + include Langfuse::Testing + + def setup + super + reset_langfuse + end +end +``` + +Include it in any test class that needs it: + +```ruby +class SummarizationServiceTest < ActiveSupport::TestCase + include LangfuseTestHelper + + test "emits a generation span" do + SummarizationService.new.call("hello world") + + span = emitted_langfuse_spans.find { |s| s.name == "summarize" } + assert span, "expected a summarize span" + assert_equal "generation", span.attributes["langfuse.observation.type"] + end +end +``` + +## Available Span Data + +`emitted_langfuse_spans` returns an array of [`OpenTelemetry::SDK::Trace::SpanData`](https://opentelemetry.io/docs/specs/otel/trace/sdk/) values. Useful fields: + +| Field | Description | +|---|---| +| `span.name` | Span name passed to `Langfuse.observe` | +| `span.attributes` | Hash of all attributes set on the span | +| `span.status.code` | `:ok`, `:error`, or `:unset` | +| `span.events` | Array of timed events recorded on the span | +| `span.parent_span_id` | Non-zero for child spans | +| `span.start_timestamp` / `span.end_timestamp` | Monotonic timestamps | diff --git a/lib/langfuse/otel_setup.rb b/lib/langfuse/otel_setup.rb index e4bfaa1..0115dd2 100644 --- a/lib/langfuse/otel_setup.rb +++ b/lib/langfuse/otel_setup.rb @@ -26,6 +26,19 @@ class << self # @return [OpenTelemetry::SDK::Trace::TracerProvider, nil] The configured internal tracer provider attr_reader :tracer_provider + # @return [OpenTelemetry::Exporter::OTLP::Exporter, OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter, nil] + # The active span exporter for the current provider — +OTLP::Exporter+ in production, + # +InMemorySpanExporter+ when +test_mode+ is enabled. +nil+ until the first +setup+ + # call completes and cleared on +shutdown+. + # @api private + attr_reader :span_exporter + + # When true, +build_exporter+ returns a fresh +InMemorySpanExporter+ on each provider + # build instead of the OTLP exporter. Set by +Langfuse::Testing+ at require time and + # survives +shutdown+ cycles. + # @api private + attr_accessor :test_mode + # Initialize Langfuse's internal tracer provider without mutating global OpenTelemetry state. # # @param config [Langfuse::Config] The Langfuse configuration @@ -61,6 +74,7 @@ def shutdown(timeout: 30) provider = @tracer_provider @tracer_provider = nil @config_snapshot = nil + @span_exporter = nil end provider&.shutdown(timeout: timeout) end @@ -130,13 +144,19 @@ def build_tracer_provider(config) provider = OpenTelemetry::SDK::Trace::TracerProvider.new( sampler: build_sampler(config.sample_rate) ) + + @span_exporter = build_exporter(config) + provider.add_span_processor( - SpanProcessor.new(config: config, exporter: build_exporter(config)) + SpanProcessor.new(config: config, exporter: @span_exporter) ) + provider end def build_exporter(config) + return OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new if @test_mode + OpenTelemetry::Exporter::OTLP::Exporter.new( endpoint: "#{config.base_url}/api/public/otel/v1/traces", headers: build_headers(config.public_key, config.secret_key), diff --git a/lib/langfuse/testing.rb b/lib/langfuse/testing.rb new file mode 100644 index 0000000..c581d9c --- /dev/null +++ b/lib/langfuse/testing.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require "langfuse" +require "opentelemetry/sdk" + +module Langfuse + # Mixin for capturing Langfuse spans in tests without real network calls. + # See docs/TESTING.md for setup and usage examples. + module Testing + # Set test_mode once at require time. It survives shutdown cycles so every + # provider build gets a fresh InMemorySpanExporter automatically. + OtelSetup.test_mode = true + + # Discard all recorded spans so the next test starts clean. + # Flushes pending batches first to avoid cross-test span leakage. + # + # @return [void] + def reset_langfuse + Langfuse.force_flush + OtelSetup.span_exporter&.reset + end + + # Return all spans emitted since the last reset, flushing any pending batches first. + # + # @return [Array] + def emitted_langfuse_spans + Langfuse.force_flush + OtelSetup.span_exporter&.finished_spans || [] + end + end +end diff --git a/spec/langfuse/otel_setup_spec.rb b/spec/langfuse/otel_setup_spec.rb index 7f76798..9730db6 100644 --- a/spec/langfuse/otel_setup_spec.rb +++ b/spec/langfuse/otel_setup_spec.rb @@ -88,6 +88,22 @@ ) end + it "uses an in-memory exporter when test_mode is set" do + original_test_mode = described_class.test_mode + described_class.test_mode = true + allow(described_class).to receive(:build_exporter).and_call_original + + described_class.setup(config) + described_class.tracer_provider.tracer(Langfuse::LANGFUSE_TRACER_NAME).start_span("test-span").finish + described_class.force_flush(timeout: 1) + + expect(described_class.span_exporter).to be_a(OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter) + expect(described_class.span_exporter.finished_spans.map(&:name)).to eq(["test-span"]) + ensure + described_class.test_mode = original_test_mode + described_class.shutdown(timeout: 1) + end + context "with sample_rate below 1.0" do before do config.sample_rate = 0.1 diff --git a/spec/langfuse/testing_spec.rb b/spec/langfuse/testing_spec.rb new file mode 100644 index 0000000..c42d2ea --- /dev/null +++ b/spec/langfuse/testing_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require "spec_helper" +require "langfuse/testing" + +RSpec.describe Langfuse::Testing do + include described_class + + before { reset_langfuse } + + it "sets test_mode at require time" do + expect(Langfuse::OtelSetup.test_mode).to be true + end + + describe "#emitted_langfuse_spans" do + it "returns finished spans after flushing" do + Langfuse.observe("test-span").end + + expect(emitted_langfuse_spans.map(&:name)).to eq(["test-span"]) + end + + it "returns an empty array when no spans have been recorded" do + expect(emitted_langfuse_spans).to eq([]) + end + end + + describe "#reset_langfuse" do + it "discards all recorded spans" do + Langfuse.observe("test-span").end + + emitted_langfuse_spans # flush so span lands in exporter + + reset_langfuse + + expect(Langfuse::OtelSetup.span_exporter.finished_spans).to be_empty + end + end +end From 8d28b91d338b7aaefc87dda306c9f2fb24041c89 Mon Sep 17 00:00:00 2001 From: kadekillary Date: Fri, 31 Jul 2026 14:45:05 -0700 Subject: [PATCH 2/4] style: fix rubocop layout offenses --- lib/langfuse/otel_setup.rb | 4 ++-- spec/langfuse/testing_spec.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/langfuse/otel_setup.rb b/lib/langfuse/otel_setup.rb index 0115dd2..26ca26f 100644 --- a/lib/langfuse/otel_setup.rb +++ b/lib/langfuse/otel_setup.rb @@ -145,12 +145,12 @@ def build_tracer_provider(config) sampler: build_sampler(config.sample_rate) ) - @span_exporter = build_exporter(config) + @span_exporter = build_exporter(config) provider.add_span_processor( SpanProcessor.new(config: config, exporter: @span_exporter) ) - + provider end diff --git a/spec/langfuse/testing_spec.rb b/spec/langfuse/testing_spec.rb index c42d2ea..26955c8 100644 --- a/spec/langfuse/testing_spec.rb +++ b/spec/langfuse/testing_spec.rb @@ -27,7 +27,7 @@ describe "#reset_langfuse" do it "discards all recorded spans" do Langfuse.observe("test-span").end - + emitted_langfuse_spans # flush so span lands in exporter reset_langfuse From 0c3ed15d0f329410d5e6e51d234780986a273c2e Mon Sep 17 00:00:00 2001 From: Alan Marx Date: Fri, 31 Jul 2026 17:34:12 -0700 Subject: [PATCH 3/4] fix(otel): assign span_exporter atomically with provider publication @span_exporter was set in build_tracer_provider before the provider won the setup mutex. On a lost race the losing provider is shut down (stopping its exporter) while @span_exporter still pointed at it, causing emitted_langfuse_spans and reset_langfuse to read a stopped exporter. build_exporter is now called in setup before build_tracer_provider, and @span_exporter is assigned inside publish_provider under the mutex only when the provider wins. rollback_provider also clears it. Co-Authored-By: Claude Sonnet 4.6 --- lib/langfuse/otel_setup.rb | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/lib/langfuse/otel_setup.rb b/lib/langfuse/otel_setup.rb index 26ca26f..ce2aefd 100644 --- a/lib/langfuse/otel_setup.rb +++ b/lib/langfuse/otel_setup.rb @@ -50,8 +50,9 @@ def setup(config) candidate_provider = nil provider = nil created = false - candidate_provider = build_tracer_provider(config) - provider, created = publish_provider(candidate_provider, tracing_config_snapshot(config)) + candidate_exporter = build_exporter(config) + candidate_provider = build_tracer_provider(config, candidate_exporter) + provider, created = publish_provider(candidate_provider, candidate_exporter, tracing_config_snapshot(config)) unless created candidate_provider.shutdown(timeout: 30) return existing_provider_for(config) @@ -109,7 +110,7 @@ def existing_provider_for(config) @tracer_provider end - def publish_provider(provider, snapshot) + def publish_provider(provider, exporter, snapshot) created = false current = nil @@ -120,6 +121,7 @@ def publish_provider(provider, snapshot) else @tracer_provider = provider @config_snapshot = snapshot + @span_exporter = exporter current = provider created = true end @@ -134,23 +136,18 @@ def rollback_provider(provider) @tracer_provider = nil @config_snapshot = nil + @span_exporter = nil end provider.shutdown(timeout: 1) rescue StandardError nil end - def build_tracer_provider(config) + def build_tracer_provider(config, exporter) provider = OpenTelemetry::SDK::Trace::TracerProvider.new( sampler: build_sampler(config.sample_rate) ) - - @span_exporter = build_exporter(config) - - provider.add_span_processor( - SpanProcessor.new(config: config, exporter: @span_exporter) - ) - + provider.add_span_processor(SpanProcessor.new(config: config, exporter: exporter)) provider end From 880b37591b7f34bb46ff3a96c1814a97f1e20f54 Mon Sep 17 00:00:00 2001 From: Alan Marx Date: Fri, 31 Jul 2026 17:47:23 -0700 Subject: [PATCH 4/4] test(otel): verify span_exporter is not overwritten on publish race Co-Authored-By: Claude Sonnet 4.6 --- spec/langfuse/otel_setup_spec.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/spec/langfuse/otel_setup_spec.rb b/spec/langfuse/otel_setup_spec.rb index 9730db6..ece2c9c 100644 --- a/spec/langfuse/otel_setup_spec.rb +++ b/spec/langfuse/otel_setup_spec.rb @@ -79,6 +79,18 @@ expect(described_class.setup(config)).to equal(existing_provider) end + it "does not overwrite span_exporter when losing the publish race" do + described_class.setup(config) + winning_exporter = described_class.span_exporter + + losing_exporter = instance_double(OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter) + losing_provider = instance_double(OpenTelemetry::SDK::Trace::TracerProvider) + _, created = described_class.send(:publish_provider, losing_provider, losing_exporter, {}) + + expect(created).to be false + expect(described_class.span_exporter).to equal(winning_exporter) + end + it "validates should_export_span in setup" do config.should_export_span = "bad"