Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion tracer/src/Datadog.Trace/Agent/OtlpSpanStatsSerializer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#nullable enable

using System;
using System.Globalization;
using System.IO;
using System.Text;
using Datadog.Trace.Logging;
Expand Down Expand Up @@ -325,7 +326,7 @@ private static void WriteIntKvJson(JsonTextWriter writer, string key, long value
writer.WritePropertyName("value");
writer.WriteStartObject();
writer.WritePropertyName("intValue");
writer.WriteValue(value.ToString());
writer.WriteValue(value.ToString(CultureInfo.InvariantCulture));
Comment thread
anna-git marked this conversation as resolved.
writer.WriteEndObject();
writer.WriteEndObject();
}
Expand Down
151 changes: 144 additions & 7 deletions tracer/src/Datadog.Trace/OpenTelemetry/OtlpMapper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
using Datadog.Trace.FeatureFlags;
using Datadog.Trace.OpenTelemetry.Common;
using Datadog.Trace.Processors;
using Datadog.Trace.Sampling;
using Datadog.Trace.Tagging;
using Datadog.Trace.Telemetry.Metrics;
using Datadog.Trace.Vendors.Datadog.Sketches;
Expand Down Expand Up @@ -237,6 +238,66 @@ public static int EmitAttributesFromSpan<TState>(in SpanModel spanModel, int lim
}
}

ITagProcessor[]? tagProcessors = null;
if (spanModel.Span.Context.TraceContext?.Tracer is Tracer tracer)
{
tagProcessors = tracer.TracerManager?.TagProcessors;
}

// Sampling decision attributes (same tags and placement as the MessagePack path) let the Agent
// honor the tracer's decision instead of applying its own OTLP sampler. They are written before
// span tags so customer tags are dropped first when the attribute limit is reached.

// add "_sampling_priority_v1" to all "chunk orphans" (spans whose parents are not found in the same chunk).
// Written first so a large number of trace tags can't push it past the attribute limit.
// Written as an OTLP int; the Agent converts int and double attributes to the same span metric.
if (spanModel is { IsChunkOrphan: true, TraceChunk.SamplingPriority: { } samplingPriority })
{
WriteAttribute(ref state, writeKeyValue, new KeyValue(Trace.Metrics.SamplingPriority, samplingPriority), limit, ref count, ref droppedAttributesCount);
}

if (spanModel.TraceChunk.Tags is { Count: > 0 } traceTags &&
(spanModel.IsFirstSpanInChunk || spanModel.IsLocalRoot))
{
var traceTagWriter = new TraceTagWriter<TState>(state, writeKeyValue, tagProcessors, spanModel.IsLocalRoot, spanModel.IsFirstSpanInChunk, count, limit);
traceTags.Enumerate(ref traceTagWriter);
Comment thread
zacharycmontoya marked this conversation as resolved.
Comment thread
anna-git marked this conversation as resolved.
count = traceTagWriter.Count;
droppedAttributesCount += traceTagWriter.DroppedCount;
state = traceTagWriter.State;
}

if (spanModel.IsLocalRoot)
{
// TODO: unlike SpanMessagePackFormatter, we don't write "process_id" or "_dd.tracer_kr" on the local root yet.
// "process_id" likely belongs on the OTLP resource instead (OTel semantic convention "process.pid").
// It's still open whether "_dd.tracer_kr" (tracer keep rate: moving average of the share of traces
// sent vs. dropped before reaching the Agent) should be exported over OTLP at all.

// add agent or rule sampling rate
Comment thread
zacharycmontoya marked this conversation as resolved.
if (spanModel.TraceChunk is { AppliedSamplingRate: { } samplingRate, SamplingMechanism: { } samplingMechanism })
{
var samplingRateTagName = samplingMechanism switch
{
SamplingMechanism.AgentRate => Trace.Metrics.SamplingAgentDecision, // "_dd.agent_psr"
SamplingMechanism.LocalTraceSamplingRule
or SamplingMechanism.RemoteAdaptiveSamplingRule
or SamplingMechanism.RemoteUserSamplingRule => Trace.Metrics.SamplingRuleDecision, // "_dd.rule_psr"
_ => null
};

if (samplingRateTagName is not null)
{
WriteAttribute(ref state, writeKeyValue, new KeyValue(samplingRateTagName, samplingRate), limit, ref count, ref droppedAttributesCount);
}
}

// add rate limiter rate
if (spanModel.TraceChunk.RateLimiterRate is { } limitSamplingRate)
{
WriteAttribute(ref state, writeKeyValue, new KeyValue(Trace.Metrics.SamplingLimitDecision, limitSamplingRate), limit, ref count, ref droppedAttributesCount);
}
}

// Notes for later:
// - Do we actually need to add _dd.base_service tag even though the OTLP span shares the same service name?

Expand All @@ -247,12 +308,6 @@ public static int EmitAttributesFromSpan<TState>(in SpanModel spanModel, int lim
// AAS tags need to be set on any span for the backend to properly handle the billing.

// Write span tags
ITagProcessor[]? tagProcessors = null;
if (spanModel.Span.Context.TraceContext?.Tracer is Tracer tracer)
{
tagProcessors = tracer.TracerManager?.TagProcessors;
}

var tagWriter = new TagWriter<TState>(state, writeKeyValue, tagProcessors, count, limit, openTelemetrySemanticsEnabled);
spanModel.Span.Tags.EnumerateTags(ref tagWriter, openTelemetrySemanticsEnabled);
count = tagWriter.Count;
Expand All @@ -270,12 +325,94 @@ public static int EmitAttributesFromSpan<TState>(in SpanModel spanModel, int lim
// if (model.IsLocalRoot)
// add the "apm.enabled" tag with a value of 0
// if (Security.Instance.AppsecEnabled && model.IsLocalRoot && span.Context.TraceContext?.WafExecuted is true)
// add "_sampling_priority_v1" tag to all "chunk orphans"
// add "_dd.top_level" to top-level spans (aka service-entry spans)

return droppedAttributesCount;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void WriteAttribute<TState>(ref TState state, KeyValueWriter<TState> writeKeyValue, KeyValue keyValue, int limit, ref int count, ref int droppedCount)
{
if (count < limit)
{
writeKeyValue(ref state, keyValue);
count++;
}
else
{
droppedCount++;
}
}

internal struct TraceTagWriter<TState> : TraceTagCollection.ITagEnumerator
{
private readonly KeyValueWriter<TState> _writeKeyValue;
private readonly ITagProcessor[]? _tagProcessors;
private readonly bool _isLocalRoot;
private readonly bool _isFirstSpanInChunk;
private readonly int _limit;

public TState State;
public int Count;
public int DroppedCount;

[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal TraceTagWriter(TState state, KeyValueWriter<TState> writeKeyValue, ITagProcessor[]? tagProcessors, bool isLocalRoot, bool isFirstSpanInChunk, int count, int limit)
{
State = state;
_writeKeyValue = writeKeyValue;
_tagProcessors = tagProcessors;
_isLocalRoot = isLocalRoot;
_isFirstSpanInChunk = isFirstSpanInChunk;
_limit = limit;

Count = count;
DroppedCount = 0;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Next(KeyValuePair<string, string> item)
{
// TagPropagation.ParseHeader accepts the "_dd.p." prefix case-insensitively and keeps the original casing,
// so classify keys the same way here. Note: SpanMessagePackFormatter.TraceTagWriter and the header
// injection in TagPropagation still compare ordinally.

// the upper 64 bits of the trace id are already part of the 128-bit OTLP traceId
if (string.Equals(item.Key, Tags.Propagated.TraceIdUpper, StringComparison.OrdinalIgnoreCase))
{
return;
}

var isPropagatedTag = item.Key.StartsWith(TagPropagation.PropagatedTagPrefix, StringComparison.OrdinalIgnoreCase);

// add propagated trace tags to the first span in every chunk,
// add non-propagated trace tags to the root span
if ((isPropagatedTag && _isFirstSpanInChunk) || (!isPropagatedTag && _isLocalRoot))
{
if (Count < _limit)
{
var key = item.Key;
var value = item.Value;

if (_tagProcessors is not null)
{
for (var i = 0; i < _tagProcessors.Length; i++)
{
_tagProcessors[i]?.ProcessMeta(ref key, ref value);
}
}

_writeKeyValue(ref State, new KeyValue(key, value));
Count++;
}
else
{
DroppedCount++;
}
}
}
}

internal struct TagWriter<TState> : IItemProcessor<string>, IItemProcessor<int>, IItemProcessor<double>, IItemProcessor<byte[]>
{
private readonly KeyValueWriter<TState> _writeKeyValue;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -314,15 +314,19 @@ internal static void WriteAnyValue(JsonTextWriter writer, object? value, bool ex
writer.WriteValue(boolValue);
break;

case int intValue:
writer.WritePropertyName("intValue");
writer.WriteValue(IntStringCache.ToInvariantString(intValue));
break;

case byte:
case sbyte:
case short:
case ushort:
case int:
case uint:
case long:
writer.WritePropertyName("intValue");
writer.WriteValue(value.ToString());
writer.WriteValue(Convert.ToString(value, CultureInfo.InvariantCulture));
break;

case double doubleValue:
Expand Down Expand Up @@ -489,7 +493,8 @@ static Action<KeyValue> WriteKeyValue(JsonTextWriter writer)
}

// flags (optional)
if (spanModel.Span.Context.SamplingPriority is int samplingPriority)
// Local spans carry the sampling decision on the trace chunk, not on their own context.
if ((spanModel.TraceChunk.SamplingPriority ?? spanModel.Span.Context.SamplingPriority) is int samplingPriority)
{
writer.WritePropertyName("flags");
writer.WriteValue(SamplingPriorityValues.IsKeep(samplingPriority) ? 1 : 0);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -555,8 +555,9 @@ private int WriteSpan(byte[] bytes, int writePosition, in SpanModel spanModel, s
writePosition = WriteSpanStatus(bytes, writePosition, statusCode.Value, spanModel.Span.GetTag(Tags.ErrorMsg));
}

// flags (field 16, fixed32) — only when sampling priority is known
if (spanModel.Span.Context.SamplingPriority is int samplingPriority)
// flags (field 16, fixed32) — only when sampling priority is known.
// Local spans carry the sampling decision on the trace chunk, not on their own context.
if ((spanModel.TraceChunk.SamplingPriority ?? spanModel.Span.Context.SamplingPriority) is int samplingPriority)
Comment thread
anna-git marked this conversation as resolved.
{
// Bit 0: trace flag for "sampled" (set when priority keeps the trace)
uint flags = SamplingPriorityValues.IsKeep(samplingPriority) ? 1u : 0u;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,15 @@ public static void NormalizeSpans(JToken tracesRequests, OtlpFieldNames names, l
span[parentSpanIdKey] = "normalized-parent-span-id";
}

// The http/json exporter writes "flags" after "parentSpanId", but the http/protobuf payload
// carries it as field 16, so the test agent renders it last. Move it to the end so both
// protocols produce the same snapshot.
if (((JObject)span).Property("flags") is { } flags)
{
flags.Remove();
((JObject)span).Add(flags);
}

// Our JSON and Protobuf OTLP exporters differ in serialization behavior when there are no attributes.
// Standardize them here by removing an empty array
if (span["attributes"] is JArray attributes && attributes.Count == 0)
Expand Down
Loading
Loading