diff --git a/BUILD.bazel b/BUILD.bazel index e1b39f02..89e2ac21 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -127,6 +127,7 @@ cc_library( "include/datadog/logger.h", "include/datadog/null_collector.h", "include/datadog/optional.h", + "include/datadog/otel_tracestate.h", "include/datadog/propagation_behavior_extract.h", "include/datadog/propagation_style.h", "include/datadog/rate.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index bcea1b83..787f91fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,6 +152,7 @@ target_sources(dd-trace-cpp-objects include/datadog/logger.h include/datadog/null_collector.h include/datadog/optional.h + include/datadog/otel_tracestate.h include/datadog/propagation_behavior_extract.h include/datadog/propagation_style.h include/datadog/rate.h diff --git a/include/datadog/otel_tracestate.h b/include/datadog/otel_tracestate.h new file mode 100644 index 00000000..7610b29e --- /dev/null +++ b/include/datadog/otel_tracestate.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace datadog { +namespace tracing { + +struct OtelTraceState { + // Raw value of the `ot` tracestate member. + std::string value; + // Number of other-vendor members to emit before `ot`. + std::size_t position; +}; + +} // namespace tracing +} // namespace datadog diff --git a/include/datadog/sampling_decision.h b/include/datadog/sampling_decision.h index cfccbf74..fa79d548 100644 --- a/include/datadog/sampling_decision.h +++ b/include/datadog/sampling_decision.h @@ -39,6 +39,9 @@ struct SamplingDecision { // The per-second maximum allowed number of "keeps" configured for the limiter // consulted in this decision, if any. Optional limiter_max_per_second; + // Whether the sample rate alone, before the rate limiter, would keep this + // trace. + Optional was_probability_sampled; // The provenance of this decision. Origin origin; }; diff --git a/include/datadog/trace_segment.h b/include/datadog/trace_segment.h index f7fa958a..7918fad1 100644 --- a/include/datadog/trace_segment.h +++ b/include/datadog/trace_segment.h @@ -33,6 +33,7 @@ #include #include "optional.h" +#include "otel_tracestate.h" #include "propagation_style.h" #include "runtime_id.h" #include "sampling_decision.h" @@ -77,6 +78,7 @@ class TraceSegment { std::vector> spans_; std::size_t num_finished_spans_; Optional sampling_decision_; + const Optional otel_w3c_tracestate_; const Optional additional_w3c_tracestate_; const Optional additional_datadog_w3c_tracestate_; @@ -99,6 +101,7 @@ class TraceSegment { Optional origin, std::size_t tags_header_max_size, std::vector> trace_tags, Optional sampling_decision, + Optional otel_w3c_tracestate, Optional additional_w3c_tracestate, Optional additional_datadog_w3c_tracestate, std::unique_ptr local_root, diff --git a/src/datadog/extracted_data.h b/src/datadog/extracted_data.h index a26274d7..be451d50 100644 --- a/src/datadog/extracted_data.h +++ b/src/datadog/extracted_data.h @@ -4,6 +4,7 @@ // extracted from trace context. It's an implementation detail of this library. #include +#include #include #include @@ -26,11 +27,13 @@ struct ExtractedData { // refering to the latest datadog parent ID. Optional datadog_w3c_parent_id; // If this `ExtractedData` was created on account of `PropagationStyle::W3C`, - // then `additional_w3c_tracestate` contains the parts of the "tracestate" - // header that are not the "dd" (Datadog) entry. If there are no other parts, - // then `additional_w3c_tracestate` is null. + // then `additional_w3c_tracestate` contains the entries of the "tracestate" + // header other than the "dd" (Datadog) and "ot" (OpenTelemetry) entries. + // If there are no such entries, then `additional_w3c_tracestate` is null. // `additional_w3c_tracestate` is used for the `W3C` injection style. Optional additional_w3c_tracestate; + // The retained OpenTelemetry `ot` tracestate member, if present. + Optional otel_w3c_tracestate; // If this `ExtractedData` was created on account of `PropagationStyle::W3C`, // and if the "tracestate" header contained a "dd" (Datadog) entry, then // `additional_datadog_w3c_tracestate` contains fields from within the "dd" diff --git a/src/datadog/extraction_util.cpp b/src/datadog/extraction_util.cpp index b5405ece..e9a7c27e 100644 --- a/src/datadog/extraction_util.cpp +++ b/src/datadog/extraction_util.cpp @@ -283,6 +283,7 @@ ExtractedData merge( if (w3c != contexts.end() && w3c->second.trace_id == result.trace_id) { result.additional_w3c_tracestate = w3c->second.additional_w3c_tracestate; + result.otel_w3c_tracestate = w3c->second.otel_w3c_tracestate; result.additional_datadog_w3c_tracestate = w3c->second.additional_datadog_w3c_tracestate; result.headers_examined.insert(result.headers_examined.end(), diff --git a/src/datadog/sampling_util.h b/src/datadog/sampling_util.h index 19f424f4..3d6500c6 100644 --- a/src/datadog/sampling_util.h +++ b/src/datadog/sampling_util.h @@ -4,6 +4,7 @@ // `TraceSampler` and `SpanSampler`. #include +#include #include #include @@ -11,6 +12,22 @@ namespace datadog { namespace tracing { +inline bool is_probability_mechanism(int mechanism) { + switch (static_cast(mechanism)) { + case SamplingMechanism::DEFAULT: + case SamplingMechanism::AGENT_RATE: + case SamplingMechanism::REMOTE_RATE_AUTO: + case SamplingMechanism::RULE: + case SamplingMechanism::REMOTE_RATE_USER_DEFINED: + case SamplingMechanism::REMOTE_RATE_EMERGENCY: + case SamplingMechanism::REMOTE_RULE: + case SamplingMechanism::REMOTE_ADAPTIVE_RULE: + return true; + default: + return false; + } +} + // Return a hash value for the specified `value`. `value` is one of the // following: // diff --git a/src/datadog/trace_sampler.cpp b/src/datadog/trace_sampler.cpp index d455b9c4..9784cde5 100644 --- a/src/datadog/trace_sampler.cpp +++ b/src/datadog/trace_sampler.cpp @@ -50,8 +50,10 @@ SamplingDecision TraceSampler::decide(const SpanData& span) { decision.mechanism = int(rule.mechanism); decision.limiter_max_per_second = limiter_max_per_second_; decision.configured_rate = rule.rate; - const std::uint64_t threshold = max_id_from_rate(rule.rate); - if (knuth_hash(span.trace_id.low) <= threshold) { + const std::uint64_t threshold = max_id_from_rate(*decision.configured_rate); + decision.was_probability_sampled = + knuth_hash(span.trace_id.low) <= threshold; + if (*decision.was_probability_sampled) { if (rule.bypass_limiter) { decision.priority = int(SamplingPriority::USER_KEEP); return decision; @@ -91,7 +93,8 @@ SamplingDecision TraceSampler::decide(const SpanData& span) { } const std::uint64_t threshold = max_id_from_rate(*decision.configured_rate); - if (knuth_hash(span.trace_id.low) <= threshold) { + decision.was_probability_sampled = knuth_hash(span.trace_id.low) <= threshold; + if (*decision.was_probability_sampled) { decision.priority = int(SamplingPriority::AUTO_KEEP); } else { decision.priority = int(SamplingPriority::AUTO_DROP); diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 555f648c..fc70445c 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include "endpoint_inferral.h" #include "hex.h" #include "platform_util.h" +#include "sampling_util.h" #include "span_data.h" #include "span_sampler.h" #include "tag_propagation.h" @@ -55,15 +57,12 @@ Cache cache_singleton; // Encode the specified `trace_tags`. If the encoded value is not longer than // the specified `tags_header_max_size`, then set it as the "x-datadog-tags" -// header using the specified `writer`. If the encoded value is oversized, then -// write a diagnostic to the specified `logger` and set a propagation error tag -// on the specified `local_root_tags`. -void inject_trace_tags( +// header using the specified `writer`. Return true if the encoded value is +// oversized. +bool inject_trace_tags( DictWriter& writer, const std::vector>& trace_tags, - std::size_t tags_header_max_size, - std::unordered_map& local_root_tags, - Logger& logger) { + std::size_t tags_header_max_size, Logger& logger) { const std::string encoded_trace_tags = encode_tags(trace_tags); if (encoded_trace_tags.size() > tags_header_max_size) { @@ -76,10 +75,13 @@ void inject_trace_tags( message += std::to_string(encoded_trace_tags.size()); message += " bytes."; logger.log_error(message); - local_root_tags[tags::internal::propagation_error] = "inject_max_size"; - } else if (!encoded_trace_tags.empty()) { + return true; + } + + if (!encoded_trace_tags.empty()) { writer.set("x-datadog-tags", encoded_trace_tags); } + return false; } void maybe_calculate_http_endpoint(HttpEndpointCalculationMode renaming_mode, @@ -151,6 +153,59 @@ Optional format_rate(double rate, Logger& logger) { return std::string(begin, end); } +Optional resolve_otel_tracestate_value( + TraceID trace_id, const SamplingDecision& decision, + const Optional& inherited) { + if (decision.origin != SamplingDecision::Origin::LOCAL) { + return inherited ? sanitize_otel_tracestate(inherited->value) : nullopt; + } + + const StringView raw = + inherited ? StringView(inherited->value) : StringView{}; + const bool probability_sampling_is_unavailable_or_dropped = + !decision.was_probability_sampled || + (*decision.was_probability_sampled && decision.priority <= 0); + if (!decision.mechanism || !decision.configured_rate || + !is_probability_mechanism(*decision.mechanism) || + probability_sampling_is_unavailable_or_dropped) { + return rewrite_otel_tracestate(raw, extract_otel_random_value(raw), + nullopt); + } + + constexpr std::uint64_t max_value = UINT64_C(1) << 56; + std::uint64_t threshold = static_cast( + std::round((1.0 - decision.configured_rate->value()) * + static_cast(max_value))); + threshold = std::min(threshold, max_value - 1); + + std::uint64_t random_value = (~knuth_hash(trace_id.low)) >> 8; + if (*decision.was_probability_sampled && random_value < threshold) { + random_value = threshold; + } else if (!*decision.was_probability_sampled && random_value >= threshold) { + random_value = threshold == 0 ? 0 : threshold - 1; + } + + return rewrite_otel_tracestate(raw, random_value, threshold); +} + +Optional resolve_otel_tracestate( + TraceID trace_id, const SamplingDecision& decision, + const Optional& inherited) { + Optional resolved = + resolve_otel_tracestate_value(trace_id, decision, inherited); + if (!resolved) { + return nullopt; + } + + const bool otel_w3c_tracestate_is_unchanged = + inherited && *resolved == inherited->value; + constexpr std::size_t first_non_datadog_tracestate_position = 0; + const std::size_t position = otel_w3c_tracestate_is_unchanged + ? inherited->position + : first_non_datadog_tracestate_position; + return OtelTraceState{std::move(*resolved), position}; +} + } // anonymous namespace TraceSegment::TraceSegment( @@ -166,6 +221,7 @@ TraceSegment::TraceSegment( std::size_t tags_header_max_size, std::vector> trace_tags, Optional sampling_decision, + Optional otel_w3c_tracestate, Optional additional_w3c_tracestate, Optional additional_datadog_w3c_tracestate, std::unique_ptr local_root, @@ -184,6 +240,7 @@ TraceSegment::TraceSegment( trace_tags_(std::move(trace_tags)), num_finished_spans_(0), sampling_decision_(std::move(sampling_decision)), + otel_w3c_tracestate_(std::move(otel_w3c_tracestate)), additional_w3c_tracestate_(std::move(additional_w3c_tracestate)), additional_datadog_w3c_tracestate_( std::move(additional_datadog_w3c_tracestate)), @@ -214,16 +271,16 @@ Optional TraceSegment::sampling_decision() const { return sampling_decision_; } -Optional> TraceSegment::w3c_link_context( +Optional TraceSegment::w3c_link_context( const SpanData& span) const { - int sampling_priority; + SamplingDecision sampling_decision; std::vector> trace_tags; { std::lock_guard lock(mutex_); if (!sampling_decision_) { return nullopt; } - sampling_priority = sampling_decision_->priority; + sampling_decision = *sampling_decision_; trace_tags = trace_tags_; const Optional trace_source_tag = @@ -233,11 +290,15 @@ Optional> TraceSegment::w3c_link_context( } } + const Optional resolved_otel_w3c_tracestate = + resolve_otel_tracestate(span.trace_id, sampling_decision, + otel_w3c_tracestate_); return std::make_pair( - encode_tracestate(span.span_id, sampling_priority, origin_, trace_tags, - additional_datadog_w3c_tracestate_, + encode_tracestate(span.span_id, sampling_decision.priority, origin_, + trace_tags, additional_datadog_w3c_tracestate_, + resolved_otel_w3c_tracestate, additional_w3c_tracestate_), - sampling_priority > 0 ? 1u : 0u); + sampling_decision.priority > 0 ? 1u : 0u); } Logger& TraceSegment::logger() const { return *logger_; } @@ -443,28 +504,25 @@ bool TraceSegment::inject(DictWriter& writer, const SpanData& span, // and trace tags might change when that happens ("_dd.p.dm"). // So, we lock here, make a sampling decision if necessary, and then copy the // decision and trace tags before unlocking. - int sampling_priority; + SamplingDecision sampling_decision; std::vector> trace_tags; + Optional trace_source_tag; { std::lock_guard lock(mutex_); make_sampling_decision_if_null(); assert(sampling_decision_); - sampling_priority = sampling_decision_->priority; + sampling_decision = *sampling_decision_; trace_tags = trace_tags_; - } - std::unordered_map& local_root_tags = - spans_.front()->tags; - - const Optional trace_source_tag = - find_trace_source_tag(local_root_tags); + trace_source_tag = find_trace_source_tag(spans_.front()->tags); + } // When tracing (the product) is disabled, skip tracing context propagation // when: // - the local root span is NOT created by another product (no `_dd.p.ts`) // - sampling priority is DROP if (!tracing_enabled_) { - if (!trace_source_tag && sampling_priority <= 0) { + if (!trace_source_tag && sampling_decision.priority <= 0) { writer.erase("x-datadog-trace-id"); writer.erase("x-datadog-parent-id"); writer.erase("x-datadog-sampling-priority"); @@ -486,18 +544,21 @@ bool TraceSegment::inject(DictWriter& writer, const SpanData& span, trace_tags.emplace_back(tags::internal::trace_source, *trace_source_tag); } + bool trace_tags_too_large = false; for (const auto style : injection_styles_) { switch (style) { case PropagationStyle::DATADOG: writer.set("x-datadog-trace-id", std::to_string(span.trace_id.low)); writer.set("x-datadog-parent-id", std::to_string(span.span_id)); writer.set("x-datadog-sampling-priority", - std::to_string(sampling_priority)); + std::to_string(sampling_decision.priority)); if (origin_) { writer.set("x-datadog-origin", *origin_); } - inject_trace_tags(writer, trace_tags, tags_header_max_size_, - local_root_tags, *logger_); + if (inject_trace_tags(writer, trace_tags, tags_header_max_size_, + *logger_)) { + trace_tags_too_large = true; + } telemetry::counter::increment(metrics::tracer::trace_context::injected, {"header_style:datadog"}); @@ -509,32 +570,47 @@ bool TraceSegment::inject(DictWriter& writer, const SpanData& span, writer.set("x-b3-traceid", hex_padded(span.trace_id.low)); } writer.set("x-b3-spanid", hex_padded(span.span_id)); - writer.set("x-b3-sampled", std::to_string(int(sampling_priority > 0))); + writer.set("x-b3-sampled", + std::to_string(int(sampling_decision.priority > 0))); if (origin_) { writer.set("x-datadog-origin", *origin_); } - inject_trace_tags(writer, trace_tags, tags_header_max_size_, - local_root_tags, *logger_); + if (inject_trace_tags(writer, trace_tags, tags_header_max_size_, + *logger_)) { + trace_tags_too_large = true; + } telemetry::counter::increment(metrics::tracer::trace_context::injected, {"header_style:b3multi"}); break; - case PropagationStyle::W3C: - writer.set( - "traceparent", - encode_traceparent(span.trace_id, span.span_id, sampling_priority)); + case PropagationStyle::W3C: { + const Optional resolved_otel_w3c_tracestate = + resolve_otel_tracestate(span.trace_id, sampling_decision, + otel_w3c_tracestate_); + writer.set("traceparent", + encode_traceparent(span.trace_id, span.span_id, + sampling_decision.priority)); writer.set( "tracestate", - encode_tracestate(span.span_id, sampling_priority, origin_, + encode_tracestate(span.span_id, sampling_decision.priority, origin_, trace_tags, additional_datadog_w3c_tracestate_, + resolved_otel_w3c_tracestate, additional_w3c_tracestate_)); telemetry::counter::increment(metrics::tracer::trace_context::injected, {"header_style:tracecontext"}); break; + } default: break; } } + if (trace_tags_too_large) { + std::lock_guard lock(mutex_); + std::unordered_map& local_root_tags = + spans_.front()->tags; + local_root_tags[tags::internal::propagation_error] = "inject_max_size"; + } + return true; } diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 8c0e6bd7..4872dcea 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -247,7 +247,8 @@ Span Tracer::create_span(const SpanConfig& config) { logger_, collector_, config_manager_->trace_sampler(), span_sampler_, defaults, config_manager_, runtime_id_, injection_styles_, hostname_, nullopt /* origin */, tags_header_max_size_, std::move(trace_tags), - nullopt /* sampling_decision */, nullopt /* additional_w3c_tracestate */, + nullopt /* sampling_decision */, nullopt /* otel_w3c_tracestate */, + nullopt /* additional_w3c_tracestate */, nullopt /* additional_datadog_w3c_tracestate*/, std::move(span_data), resource_renaming_mode_, tracing_enabled_); Span span{span_data_ptr, segment, @@ -483,6 +484,7 @@ Expected Tracer::extract_span(const DictReader& reader, injection_styles_, hostname_, std::move(merged_context.origin), tags_header_max_size_, std::move(merged_context.trace_tags), std::move(sampling_decision), + std::move(merged_context.otel_w3c_tracestate), std::move(merged_context.additional_w3c_tracestate), std::move(merged_context.additional_datadog_w3c_tracestate), std::move(span_data), resource_renaming_mode_, tracing_enabled_); diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index acba657f..54a78faf 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -33,11 +33,109 @@ auto verboten(int lowest_ascii, int highest_ascii, }; } -constexpr bool is_hexdiglc(const char c) { +constexpr bool is_hexdig(const char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } +constexpr bool is_lowercase_hexdig(const char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); +} + +constexpr std::size_t otel_random_value_size = 14; +constexpr std::size_t min_otel_threshold_size = 1; +constexpr std::size_t max_otel_threshold_size = otel_random_value_size; +constexpr std::size_t max_w3c_tracestate_member_value_size = 256; +constexpr std::size_t max_datadog_tracestate_value_size = 512; +constexpr std::size_t max_w3c_tracestate_members = 32; + +Optional parse_otel_value(StringView value, + std::size_t minimum_size, + std::size_t maximum_size) { + if (value.size() < minimum_size || value.size() > maximum_size || + !std::all_of(value.begin(), value.end(), is_lowercase_hexdig)) { + return nullopt; + } + + const Expected parsed = parse_uint64(value, 16); + if (parsed.if_error()) { + return nullopt; + } + return *parsed; +} + +Optional parse_otel_random_value(StringView value) { + return parse_otel_value(value, otel_random_value_size, + otel_random_value_size); +} + +Optional parse_otel_threshold(StringView value) { + return parse_otel_value(value, min_otel_threshold_size, + max_otel_threshold_size); +} + +template +void for_each_otel_item(StringView raw, Function&& function) { + std::size_t begin = 0; + while (begin < raw.size()) { + const std::size_t end = raw.find(';', begin); + const StringView item = raw.substr(begin, end - begin); + const std::size_t separator = item.find(':'); + const StringView key = item.substr(0, separator); + const StringView value = separator == StringView::npos + ? StringView{} + : item.substr(separator + 1); + function(item, key, value); + if (end == StringView::npos) { + return; + } + begin = end + 1; + } +} + +template +void for_each_w3c_tracestate_member(StringView tracestate, + Function&& function) { + std::size_t begin = 0; + while (begin < tracestate.size()) { + const std::size_t end = tracestate.find(',', begin); + const StringView member = trim(tracestate.substr(begin, end - begin)); + if (!function(member) || end == StringView::npos) { + return; + } + begin = end + 1; + } +} + +void append_otel_item(std::string& result, StringView item) { + if (item.empty()) { + return; + } + + const std::size_t separator_size = result.empty() ? 0 : 1; + if (result.size() + separator_size + item.size() > + max_w3c_tracestate_member_value_size) { + return; + } + + if (separator_size) { + result += ';'; + } + append(result, item); +} + +std::string format_otel_hex(std::uint64_t value) { + return hex_padded(value).substr(2); +} + +std::string format_otel_threshold(std::uint64_t threshold) { + std::string result = format_otel_hex(threshold); + while (result.size() > 1 && result.back() == '0') { + result.pop_back(); + } + return result; +} + // Populate the specified `result` with data extracted from the "traceparent" // entry of the specified `headers`. Return `nullopt` on success. Return a value // for the `tags::internal::w3c_extraction_error` tag if an error occurs. @@ -64,7 +162,7 @@ Optional extract_traceparent(ExtractedData& result, beg = i + 1; internal_state = state::trace_id; - } else if (!is_hexdiglc(traceparent[i])) { + } else if (!is_hexdig(traceparent[i])) { return "invalid_version"; } } break; @@ -124,86 +222,25 @@ Optional extract_traceparent(ExtractedData& result, return nullopt; } -// `struct PartiallyParsedTracestat` contains the separated Datadog-specific and -// non-Datadog-specific portions of tracestate. -struct PartiallyParsedTracestate { - StringView datadog_value; - std::string other_entries; -}; - -// Return the separate Datadog-specific and non-Datadog-specific portions of the -// specified `tracestate`. If `tracestate` does not have a Datadog-specific -// portion, return `nullopt`. -Optional parse_tracestate(StringView tracestate) { - const std::size_t begin = 0; - const std::size_t end = tracestate.size(); - std::size_t pair_begin = begin; - while (pair_begin < end) { - const std::size_t pair_end = tracestate.find(',', pair_begin); - // Note that since this `pair` is `strip`ped, `pair_begin` is not - // necessarily equal to `pair.begin()` (similarly for the ends). - const auto pair = - trim(tracestate.substr(pair_begin, pair_end - pair_begin)); - if (pair.empty()) { - pair_begin = (pair_end == StringView::npos) ? end : pair_end + 1; - continue; - } - - const auto kv_separator = pair.find('='); - if (kv_separator == StringView::npos) { - // This is an invalid entry because it contains a non-whitespace character - // but not a "=". - // Let's move on to the next entry. - pair_begin = (pair_end == StringView::npos) ? end : pair_end + 1; - continue; - } - - const auto key = pair.substr(0, kv_separator); - if (key != "dd") { - // On to the next. - pair_begin = (pair_end == StringView::npos) ? end : pair_end + 1; - continue; - } - - PartiallyParsedTracestate result; - result.datadog_value = pair.substr(kv_separator + 1); - // `result->other_entries` is whatever was before the "dd" entry and - // whatever is after the "dd" entry, but without an extra comma in the - // middle. - if (pair_begin != 0) { - // There's a prefix - append(result.other_entries, tracestate.substr(0, pair_begin - 1)); - if (pair_end != StringView::npos && pair_end + 1 < end) { - // and a suffix - append(result.other_entries, tracestate.substr(pair_end)); - } - } else if (pair_end != StringView::npos && pair_end + 1 < end) { - // There's just a suffix - append(result.other_entries, tracestate.substr(pair_end + 1)); - } - - return result; - } - - return nullopt; -} // Fill the specified `result` with information parsed from the specified -// `datadog_value`. `datadog_value` is the value of the "dd" entry in the -// "tracestate" header. +// `datadog_trace_state`. `datadog_trace_state` is the value of the "dd" entry +// in the W3C "tracestate" header. // -// `parse_datadog_tracestate` populates the following `ExtractedData` fields: +// `parse_datadog_trace_state` populates the following `ExtractedData` fields: // // - `origin` // - `trace_tags` // - `sampling_priority` // - `datadog_w3c_parent_id` // - `additional_datadog_w3c_tracestate` -void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { - const std::size_t end = datadog_value.size(); +void parse_datadog_trace_state(ExtractedData& result, + StringView datadog_trace_state) { + const std::size_t end = datadog_trace_state.size(); std::size_t pair_begin = 0; while (pair_begin < end) { - const std::size_t pair_end = datadog_value.find(';', pair_begin); - const auto pair = datadog_value.substr(pair_begin, pair_end - pair_begin); + const std::size_t pair_end = datadog_trace_state.find(';', pair_begin); + const auto pair = + datadog_trace_state.substr(pair_begin, pair_end - pair_begin); pair_begin = (pair_end == StringView::npos) ? end : pair_end + 1; if (pair.empty()) { continue; @@ -272,45 +309,73 @@ void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { } } -// Fill the specified `result` with information parsed from the "tracestate" -// element of the specified `headers`, if present. +// Fill the specified `result` with information parsed from the specified +// `ot_tracestate`. `ot_tracestate` is the value of the "ot" entry in the W3C +// "tracestate" header. // -// `extract_tracestate` populates the `additional_w3c_tracestate` field of -// `ExtractedData`, in addition to those populated by -// `parse_datadog_tracestate`. -void extract_tracestate( - ExtractedData& result, const DictReader& headers, - std::unordered_map& span_tags) { - const auto maybe_tracestate = headers.lookup("tracestate"); - if (!maybe_tracestate || maybe_tracestate->empty()) { - return; +// `parse_ot_tracestate` preserves the first OpenTelemetry state in a header. +void parse_ot_tracestate(ExtractedData& result, StringView ot_tracestate, + std::size_t position) { + if (!result.otel_w3c_tracestate) { + result.otel_w3c_tracestate = + OtelTraceState{std::string(ot_tracestate), position}; } +} - const auto tracestate = trim(*maybe_tracestate); - result.tracestate_full = tracestate; - - auto maybe_parsed = parse_tracestate(tracestate); - if (!maybe_parsed) { - // No "dd" entry in `tracestate`, so there's nothing to extract. - if (!tracestate.empty()) { - result.additional_w3c_tracestate = std::string{tracestate}; +void parse_w3c_tracestate_member( + ExtractedData& result, StringView member, + std::unordered_map& span_tags, + std::string& other_w3c_tracestate, + std::size_t& other_w3c_tracestate_member_count) { + const std::size_t separator = member.find('='); + const StringView key = member.substr(0, separator); + const StringView member_value = separator == StringView::npos + ? StringView{} + : member.substr(separator + 1); + if (key == "dd") { + if (member_value.size() > max_datadog_tracestate_value_size) { + span_tags[tags::internal::propagation_error] = "extract_max_size"; + } else { + parse_datadog_trace_state(result, member_value); + } + } else if (key == "ot") { + if (member_value.size() > max_w3c_tracestate_member_value_size) { + span_tags[tags::internal::propagation_error] = "extract_max_size"; + } else { + parse_ot_tracestate(result, member_value, + other_w3c_tracestate_member_count); + } + } else { + if (!other_w3c_tracestate.empty()) { + other_w3c_tracestate += ','; + } + if (!member.empty()) { + append(other_w3c_tracestate, member); + ++other_w3c_tracestate_member_count; } - return; - } - - auto& [datadog_value, other_entries] = *maybe_parsed; - if (!other_entries.empty()) { - result.additional_w3c_tracestate = std::move(other_entries); } +} - // If the "dd" vendor entry's value exceeds 512 bytes, drop it and record a - // propagation error tag. - if (datadog_value.size() > 512) { - span_tags[tags::internal::propagation_error] = "extract_max_size"; - return; +// Fill the specified `result` with information parsed from the specified W3C +// `tracestate`. +// +// `parse_w3c_tracestate` populates `additional_w3c_tracestate`, in addition to +// the fields populated by `parse_datadog_trace_state` and +// `parse_ot_tracestate`. +void parse_w3c_tracestate( + ExtractedData& result, StringView w3c_tracestate, + std::unordered_map& span_tags) { + std::string other_w3c_tracestate; + std::size_t other_w3c_tracestate_member_count = 0; + for_each_w3c_tracestate_member(w3c_tracestate, [&](StringView member) { + parse_w3c_tracestate_member(result, member, span_tags, other_w3c_tracestate, + other_w3c_tracestate_member_count); + return true; + }); + + if (!other_w3c_tracestate.empty()) { + result.additional_w3c_tracestate = std::move(other_w3c_tracestate); } - - parse_datadog_tracestate(result, datadog_value); } } // namespace @@ -340,7 +405,12 @@ Expected extract_w3c( } result.datadog_w3c_parent_id = "0000000000000000"; - extract_tracestate(result, headers, span_tags); + const Optional maybe_tracestate = headers.lookup("tracestate"); + if (maybe_tracestate && !maybe_tracestate->empty()) { + const StringView tracestate = trim(*maybe_tracestate); + result.tracestate_full = tracestate; + parse_w3c_tracestate(result, tracestate, span_tags); + } return result; } @@ -411,12 +481,11 @@ std::string encode_datadog_tracestate( result += *additional_datadog_w3c_tracestate; } - const std::size_t max_size = 256; - while (result.size() > max_size) { + while (result.size() > max_w3c_tracestate_member_value_size) { const auto last_semicolon_index = result.rfind(';'); // This assumption is safe, because `result` always begins with - // "dd=s:", and that's fewer than `max_size` characters for any - // ``. + // "dd=s:", and that's fewer than + // `max_w3c_tracestate_member_value_size` characters for any ``. assert(last_semicolon_index != std::string::npos); result.resize(last_semicolon_index); } @@ -424,19 +493,112 @@ std::string encode_datadog_tracestate( return result; } +Optional sanitize_otel_tracestate(StringView raw) { + std::string result; + for_each_otel_item(raw, + [&](StringView item, StringView key, StringView value) { + if ((key == "rv" && !parse_otel_random_value(value)) || + (key == "th" && !parse_otel_threshold(value))) { + return; + } + append_otel_item(result, item); + }); + if (result.empty()) { + return nullopt; + } + return result; +} + +Optional extract_otel_random_value(StringView raw) { + Optional result; + for_each_otel_item(raw, [&](StringView, StringView key, StringView value) { + if (!result && key == "rv") { + result = parse_otel_random_value(value); + } + }); + return result; +} + +Optional rewrite_otel_tracestate( + StringView raw, Optional random_value, + Optional threshold) { + std::string result; + if (random_value) { + const std::string item = "rv:" + format_otel_hex(*random_value); + append_otel_item(result, item); + } + if (threshold) { + const std::string item = "th:" + format_otel_threshold(*threshold); + append_otel_item(result, item); + } + + for_each_otel_item(raw, [&](StringView item, StringView key, StringView) { + if (key != "rv" && key != "th") { + append_otel_item(result, item); + } + }); + + if (result.empty()) { + return nullopt; + } + return result; +} + +// Append other vendors and the `ot` member in their specified order. +void append_tracestate_entries( + std::string& result, StringView entries, std::size_t& member_count, + const Optional& otel_w3c_tracestate) { + std::size_t other_w3c_tracestate_position = 0; + const auto append_otel_tracestate = [&]() { + result += ",ot="; + result += otel_w3c_tracestate->value; + ++member_count; + }; + + for_each_w3c_tracestate_member(entries, [&](StringView entry) { + if (entry.empty()) { + return true; + } + if (otel_w3c_tracestate && + otel_w3c_tracestate->position == other_w3c_tracestate_position) { + append_otel_tracestate(); + if (member_count >= max_w3c_tracestate_members) { + return false; + } + } + + result += ','; + append(result, entry); + ++member_count; + ++other_w3c_tracestate_position; + return member_count < max_w3c_tracestate_members; + }); + + if (otel_w3c_tracestate && + otel_w3c_tracestate->position == other_w3c_tracestate_position && + member_count < max_w3c_tracestate_members) { + append_otel_tracestate(); + } +} + std::string encode_tracestate( uint64_t span_id, int sampling_priority, const Optional& origin, const std::vector>& trace_tags, const Optional& additional_datadog_w3c_tracestate, + const Optional& otel_w3c_tracestate, const Optional& additional_w3c_tracestate) { std::string result = encode_datadog_tracestate(span_id, sampling_priority, origin, trace_tags, additional_datadog_w3c_tracestate); - if (additional_w3c_tracestate) { - result += ','; - result += *additional_w3c_tracestate; + std::size_t member_count = 1; + if (additional_w3c_tracestate || otel_w3c_tracestate) { + const StringView entries = additional_w3c_tracestate + ? StringView(*additional_w3c_tracestate) + : StringView{}; + append_tracestate_entries(result, entries, member_count, + otel_w3c_tracestate); } return result; diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 823cce39..8c04179f 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include @@ -37,12 +39,21 @@ Expected extract_w3c( std::string encode_traceparent(TraceID trace_id, std::uint64_t span_id, int sampling_priority); +Optional sanitize_otel_tracestate(StringView raw); + +Optional extract_otel_random_value(StringView raw); + +Optional rewrite_otel_tracestate( + StringView raw, Optional random_value, + Optional threshold); + // Return a value for the "tracestate" header containing the specified fields. std::string encode_tracestate( uint64_t span_id, int sampling_priority, const Optional& origin, const std::vector>& trace_tags, const Optional& additional_datadog_w3c_tracestate, + const Optional& otel_w3c_tracestate, const Optional& additional_w3c_tracestate); } // namespace tracing diff --git a/test/test_span.cpp b/test/test_span.cpp index 6608e60b..6b6f0d40 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -792,7 +792,8 @@ TEST_SPAN("injecting W3C tracestate header") { {"x-datadog-origin", "France"}, }, // The "s:-1" and "t.ksr:0" comes from the 0% sample rate. - "dd=s:-1;p:$parent_id;o:France;t.ksr:0"}, + "dd=s:-1;p:$parent_id;o:France;t.ksr:0,ot=rv:f0948a54d43b8e;th:" + "ffffffffffffff"}, {__LINE__, "trace tags", @@ -802,7 +803,8 @@ TEST_SPAN("injecting W3C tracestate header") { {"x-datadog-tags", "_dd.p.foo=x,_dd.p.bar=y,ignored=wrong_prefix"}, }, // The "s:-1" and "t.ksr:0" comes from the 0% sample rate. - "dd=s:-1;p:$parent_id;t.foo:x;t.bar:y;t.ksr:0"}, + "dd=s:-1;p:$parent_id;t.foo:x;t.bar:y;t.ksr:0,ot=rv:f0948a54d43b8e;" + "th:ffffffffffffff"}, {__LINE__, "extra fields", @@ -832,7 +834,7 @@ TEST_SPAN("injecting W3C tracestate header") { }, // The "s:-1" comes from the 0% sample rate. "dd=s:-1;p:$parent_id;o:France_ is a country~nation_ so is " - "______.;t.ksr:0", + "______.;t.ksr:0,ot=rv:f0948a54d43b8e;th:ffffffffffffff", }, {__LINE__, @@ -843,7 +845,8 @@ TEST_SPAN("injecting W3C tracestate header") { {"x-datadog-tags", "_dd.p.a;d台北x =foo,_dd.p.ok=bar"}, }, // The "s:-1" comes from the 0% sample rate. - "dd=s:-1;p:$parent_id;t.a_d______x_:foo;t.ok:bar;t.ksr:0"}, + "dd=s:-1;p:$parent_id;t.a_d______x_:foo;t.ok:bar;t.ksr:0,ot=rv:" + "f0948a54d43b8e;th:ffffffffffffff"}, {__LINE__, "replace invalid characters in trace tag value", @@ -854,7 +857,7 @@ TEST_SPAN("injecting W3C tracestate header") { }, // The "s:-1" comes from the 0% sample rate. "dd=s:-1;p:$parent_id;t.wacky:hello fr_d_ how are " - "_________?;t.ksr:0"}, + "_________?;t.ksr:0,ot=rv:f0948a54d43b8e;th:ffffffffffffff"}, {__LINE__, "replace equal signs with tildes in trace tag value", @@ -864,7 +867,8 @@ TEST_SPAN("injecting W3C tracestate header") { {"x-datadog-tags", "_dd.p.base64_thingy=d2Fra2EhIHdhaw=="}, }, // The "s:-1" comes from the 0% sample rate. - "dd=s:-1;p:$parent_id;t.base64_thingy:d2Fra2EhIHdhaw~~;t.ksr:0"}, + "dd=s:-1;p:$parent_id;t.base64_thingy:d2Fra2EhIHdhaw~~;t.ksr:0,ot=" + "rv:f0948a54d43b8e;th:ffffffffffffff"}, {__LINE__, "oversized origin truncates it and subsequent fields", @@ -883,7 +887,7 @@ TEST_SPAN("injecting W3C tracestate header") { {"x-datadog-tags", "_dd.p.foo=bar,_dd.p.honk=honk"}, }, // The "s:-1" comes from the 0% sample rate. - "dd=s:-1;p:$parent_id"}, + "dd=s:-1;p:$parent_id,ot=rv:f0948a54d43b8e;th:ffffffffffffff"}, {__LINE__, "oversized trace tag truncates it and subsequent fields", @@ -901,7 +905,8 @@ TEST_SPAN("injecting W3C tracestate header") { "ooooooooooooooooooong,_dd.p.lost=forever"}, }, // The "s:-1" comes from the 0% sample rate. - "dd=s:-1;p:$parent_id;t.foo:bar"}, + "dd=s:-1;p:$parent_id;t.foo:bar,ot=rv:f0948a54d43b8e;" + "th:ffffffffffffff"}, {__LINE__, "oversized extra field truncates itself and subsequent fields", @@ -928,6 +933,15 @@ TEST_SPAN("injecting W3C tracestate header") { }, // The "s:0" comes from the sampling decision in `traceparent_drop`. "dd=s:0;p:$parent_id,foo=bar,boing=boing"}, + + {__LINE__, + "unmodified OpenTelemetry member preserves vendor order", + { + {"traceparent", traceparent_drop}, + {"tracestate", "foo=bar,ot=future:value,boing=boing"}, + }, + // The "s:0" comes from the sampling decision in `traceparent_drop`. + "dd=s:0;p:$parent_id,foo=bar,ot=future:value,boing=boing"}, })); CAPTURE(test_case.name); @@ -955,6 +969,127 @@ TEST_SPAN("injecting W3C tracestate header") { REQUIRE(logger->error_count() == 0); } +TEST_SPAN("OpenTelemetry consistent probability sampling") { + class Generator : public IDGenerator { + const TraceID trace_id_; + + public: + explicit Generator(TraceID trace_id) : trace_id_(trace_id) {} + TraceID trace_id(const TimePoint&) const override { return trace_id_; } + std::uint64_t span_id() const override { return trace_id_.low; } + }; + + SECTION("local probability decisions emit a consistent rv and th") { + struct TestCase { + double rate; + std::uint64_t trace_id; + bool sampled; + std::string expected_ot; + }; + + const auto test_case = GENERATE(values({ + {0.01, 1, false, "rv:f0948a54d43b8e;th:fd70a3d70a3d7"}, + {0.1, 1, true, "rv:f0948a54d43b8e;th:e6666666666668"}, + {0.2, 1, true, "rv:f0948a54d43b8e;th:ccccccccccccd"}, + {0.5, 1, true, "rv:f0948a54d43b8e;th:8"}, + {0.99, 1, true, "rv:f0948a54d43b8e;th:028f5c28f5c29"}, + {0.1, UINT64_C(0x03A93EE8B1999F00), true, + "rv:e6666666666668;th:e6666666666668"}, + {0.05, UINT64_C(5401449561355763072), false, + "rv:f333333333332f;th:f333333333333"}, + })); + + CAPTURE(test_case.rate); + CAPTURE(test_case.trace_id); + CAPTURE(test_case.expected_ot); + + TracerConfig config; + config.service = "testsvc"; + config.collector = std::make_shared(); + config.logger = std::make_shared(); + config.telemetry.enabled = false; + config.injection_styles = {PropagationStyle::W3C}; + config.trace_sampler.sample_rate = test_case.rate; + config.trace_sampler.max_per_second = 100; + + const Expected finalized = finalize_config(config); + REQUIRE(finalized); + Tracer tracer{*finalized, + std::make_shared(TraceID(test_case.trace_id))}; + + Span span = tracer.create_span(); + MockDictWriter writer; + span.inject(writer); + + const auto tracestate = writer.items.find("tracestate"); + REQUIRE(tracestate != writer.items.end()); + REQUIRE(tracestate->second.find("dd=") == 0); + REQUIRE(tracestate->second.find("ot=" + test_case.expected_ot) != + std::string::npos); + const std::string& traceparent = writer.items.at("traceparent"); + REQUIRE(traceparent.substr(traceparent.size() - 3) == + (test_case.sampled ? "-01" : "-00")); + } + + SECTION("non-probability decisions retain inherited rv but erase th") { + TracerConfig config; + config.service = "testsvc"; + config.collector = std::make_shared(); + config.logger = std::make_shared(); + config.telemetry.enabled = false; + config.extraction_styles = {PropagationStyle::W3C}; + config.injection_styles = {PropagationStyle::W3C}; + + const Expected finalized = finalize_config(config); + REQUIRE(finalized); + Tracer tracer{*finalized}; + + const std::unordered_map input_headers{ + {"traceparent", + "00-00000000000000000000000000000001-0000000000000001-00"}, + {"tracestate", "ot=rv:1234567890abcd;th:e6666666666668"}, + }; + MockDictReader reader{input_headers}; + Expected span = tracer.extract_span(reader); + REQUIRE(span); + span->trace_segment().override_sampling_priority( + int(SamplingPriority::USER_KEEP)); + + MockDictWriter writer; + span->inject(writer); + REQUIRE(writer.items.at("tracestate").find("ot=rv:1234567890abcd") != + std::string::npos); + REQUIRE(writer.items.at("tracestate").find("th:") == std::string::npos); + } + + SECTION("rate-limiter demotion clears locally generated sampling values") { + TracerConfig config; + config.service = "testsvc"; + config.collector = std::make_shared(); + config.logger = std::make_shared(); + config.telemetry.enabled = false; + config.injection_styles = {PropagationStyle::W3C}; + config.trace_sampler.sample_rate = 1.0; + config.trace_sampler.max_per_second = 0.1; + + const Expected finalized = finalize_config(config); + REQUIRE(finalized); + Tracer tracer{*finalized, std::make_shared(TraceID(1))}; + + { + Span span = tracer.create_span(); + MockDictWriter writer; + span.inject(writer); + REQUIRE(writer.items.at("tracestate").find("ot=") != std::string::npos); + } + + Span span = tracer.create_span(); + MockDictWriter writer; + span.inject(writer); + REQUIRE(writer.items.at("tracestate").find("ot=") == std::string::npos); + } +} + TEST_SPAN("128-bit trace ID injection") { TracerConfig config; config.service = "testsvc"; diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index 55f8d15c..4ab69071 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -1291,6 +1291,33 @@ TEST_TRACER("span extraction") { "extract_max_size"); } + SECTION( + "'extract_max_size' propagation error if tracestate \"ot\" vendor " + "value is oversized on extract") { + constexpr std::size_t max_w3c_tracestate_member_value_size = 256; + const std::string ot_value(max_w3c_tracestate_member_value_size + 1, 'a'); + std::unordered_map span_tags; + MockLogger logger; + CAPTURE(logger.entries); + CAPTURE(span_tags); + + std::unordered_map headers{ + {"traceparent", + "00-00000000000000000000000000000001-0000000000000001-00"}, + {"tracestate", "dd=s:1,ot=" + ot_value + ",vendorx=keepme"}}; + MockDictReader reader{headers}; + + const auto extracted = extract_w3c(reader, span_tags, logger); + REQUIRE(extracted); + REQUIRE(extracted->otel_w3c_tracestate == nullopt); + REQUIRE(extracted->additional_w3c_tracestate == "vendorx=keepme"); + + REQUIRE(logger.entries.empty()); + REQUIRE(span_tags.count(tags::internal::propagation_error) == 1); + REQUIRE(span_tags.at(tags::internal::propagation_error) == + "extract_max_size"); + } + SECTION("W3C Phase 3 support - Preferring tracecontext") { // Tests behavior from system-test // test_headers_tracecontext.py::test_tracestate_w3c_p_extract_datadog_w3c @@ -1727,6 +1754,69 @@ TEST_TRACER("restart extraction link uses metadata from the selected context") { REQUIRE(link.context.flags == Optional(1u)); } +TEST_TRACER("OpenTelemetry tracestate sampling values") { + SECTION("malformed sampling values are removed") { + const Optional normalized = + sanitize_otel_tracestate("rv:1234567890abcd;th:ABC;future:value"); + REQUIRE(normalized); + REQUIRE(*normalized == "rv:1234567890abcd;future:value"); + + REQUIRE(!sanitize_otel_tracestate("rv:1234567890ABCD;th:A")); + } + + SECTION("sampling values are replaced without altering other values") { + const Optional rewritten = rewrite_otel_tracestate( + "rv:bad;future:value;th:bad", UINT64_C(0xf0948a54d43b8e), + UINT64_C(0xe6666666666668)); + REQUIRE(rewritten); + REQUIRE(*rewritten == "rv:f0948a54d43b8e;th:e6666666666668;future:value"); + + const Optional no_threshold = + rewrite_otel_tracestate("rv:1234567890abcd;th:e6666666666668", + UINT64_C(0x1234567890abcd), nullopt); + REQUIRE(no_threshold); + REQUIRE(*no_threshold == "rv:1234567890abcd"); + } + + SECTION("the OpenTelemetry member is separated from other vendors") { + const std::unordered_map headers{ + {"traceparent", + "00-00000000000000000000000000000001-0000000000000001-01"}, + {"tracestate", + "dd=s:2,ot=rv:1234567890abcd;th:e6666666666668;future:value," + "congo=t61rcWkgMzE"}, + }; + MockDictReader reader{headers}; + std::unordered_map span_tags; + MockLogger logger; + + const Expected extracted = + extract_w3c(reader, span_tags, logger); + REQUIRE(extracted); + REQUIRE(extracted->otel_w3c_tracestate); + REQUIRE(extracted->otel_w3c_tracestate->value == + "rv:1234567890abcd;th:e6666666666668;future:value"); + REQUIRE(extracted->additional_w3c_tracestate == "congo=t61rcWkgMzE"); + } + + SECTION("the first OpenTelemetry member is retained") { + const std::unordered_map headers{ + {"traceparent", + "00-00000000000000000000000000000001-0000000000000001-01"}, + {"tracestate", "ot=first,ot=second"}, + }; + MockDictReader reader{headers}; + std::unordered_map span_tags; + MockLogger logger; + + const Expected extracted = + extract_w3c(reader, span_tags, logger); + REQUIRE(extracted); + REQUIRE(extracted->otel_w3c_tracestate); + REQUIRE(extracted->otel_w3c_tracestate->value == "first"); + } +} + TEST_TRACER("baggage usage") { TracerConfig config; config.logger = std::make_shared(); @@ -2037,9 +2127,12 @@ TEST_TRACER("heterogeneous extraction") { {{"x-datadog-trace-id", "48"}, {"x-datadog-parent-id", "64"}, {"x-datadog-origin", "Kansas"}, {"x-datadog-sampling-priority", "2"}, {"traceparent", "00-00000000000000000000000000000030-0000000000000040-01"}, - {"tracestate", "competitor=stuff,dd=o:Nebraska;s:1;ah:choo"}}, // origin is different + {"tracestate", "competitor=stuff,dd=o:Nebraska;s:1;ah:choo," + "ot=rv:1234567890abcd;th:e6666666666668;future:value"}}, // origin is different {{"traceparent", "00-00000000000000000000000000000030-000000000000002a-01"}, - {"tracestate", "dd=s:2;p:000000000000002a;o:Kansas;ah:choo,competitor=stuff"}}}, + {"tracestate", "dd=s:2;p:000000000000002a;o:Kansas;ah:choo," + "competitor=stuff,ot=rv:1234567890abcd;th:e6666666666668;" + "future:value"}}}, {__LINE__, "ignore interlopers", {PropagationStyle::DATADOG, PropagationStyle::B3, PropagationStyle::W3C}, @@ -2060,7 +2153,8 @@ TEST_TRACER("heterogeneous extraction") { {{"x-datadog-trace-id", "48"}, {"x-datadog-parent-id", "64"}, {"x-datadog-origin", "Kansas"}, {"x-datadog-sampling-priority", "2"}, {"traceparent", "00-00000000000000000000000000000031-0000000000000040-01"}, - {"tracestate", "competitor=stuff,dd=o:Nebraska;s:1;ah:choo"}}, + {"tracestate", "competitor=stuff,dd=o:Nebraska;s:1;ah:choo," + "ot=rv:1234567890abcd;th:e6666666666668;future:value"}}, {{"traceparent", "00-00000000000000000000000000000030-000000000000002a-01"}, {"tracestate", "dd=s:2;p:000000000000002a;o:Kansas"}}},