[Test Optimization] Report sessions where VSTest ran no tests as skipped

tracer/build/supported_calltargets.g.json

@@ -7881,6 +7881,31 @@
     "IsAdoNetIntegration": false,
     "InstrumentationCategory": 1
   },
+  {
+    "IntegrationName": "DotnetTest",
+    "AssemblyName": "vstest.console",
+    "TargetTypeName": "Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.RunTestsArgumentExecutor+TestRunRequestEventsRegistrar",
+    "TargetMethodName": "TestRunRequest_OnRunCompletion",
+    "TargetReturnType": "System.Void",
+    "TargetParameterTypes": [
+      "System.Object",
+      "Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs"
+    ],
+    "MinimumVersion": {
+      "Item1": 15,
+      "Item2": 0,
+      "Item3": 0
+    },
+    "MaximumVersion": {
+      "Item1": 15,
+      "Item2": 65535,
+      "Item3": 65535
+    },
+    "InstrumentationTypeName": "Datadog.Trace.ClrProfiler.AutoInstrumentation.Testing.DotnetTest.TestRunRequestOnRunCompletionIntegration",
+    "IntegrationKind": 0,
+    "IsAdoNetIntegration": false,
+    "InstrumentationCategory": 1
+  },
   {
     "IntegrationName": "DotnetTest",
     "AssemblyName": "vstest.console.arm64",
@@ -7930,6 +7955,31 @@
     "IsAdoNetIntegration": false,
     "InstrumentationCategory": 1
   },
+  {
+    "IntegrationName": "DotnetTest",
+    "AssemblyName": "vstest.console.arm64",
+    "TargetTypeName": "Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.RunTestsArgumentExecutor+TestRunRequestEventsRegistrar",
+    "TargetMethodName": "TestRunRequest_OnRunCompletion",
+    "TargetReturnType": "System.Void",
+    "TargetParameterTypes": [
+      "System.Object",
+      "Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs"
+    ],
+    "MinimumVersion": {
+      "Item1": 15,
+      "Item2": 0,
+      "Item3": 0
+    },
+    "MaximumVersion": {
+      "Item1": 15,
+      "Item2": 65535,
+      "Item3": 65535
+    },
+    "InstrumentationTypeName": "Datadog.Trace.ClrProfiler.AutoInstrumentation.Testing.DotnetTest.TestRunRequestOnRunCompletionIntegration",
+    "IntegrationKind": 0,
+    "IsAdoNetIntegration": false,
+    "InstrumentationCategory": 1
+  },
   {
     "IntegrationName": "ElasticsearchNet",
     "AssemblyName": "Elasticsearch.Net",

tracer/src/Datadog.Trace/Ci/Ipc/Messages/SessionExecutedTestsMessage.cs

@@ -0,0 +1,37 @@
+// <copyright file="SessionExecutedTestsMessage.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+#nullable enable
+
+using Datadog.Trace.Vendors.Newtonsoft.Json;
+
+namespace Datadog.Trace.Ci.Ipc.Messages;
+
+/// <summary>
+/// Number of tests executed by a test runner, sent to the test session that owns the run.
+/// </summary>
+internal sealed class SessionExecutedTestsMessage
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="SessionExecutedTestsMessage"/> class for IPC deserialization.
+    /// </summary>
+    public SessionExecutedTestsMessage()
+    {
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="SessionExecutedTestsMessage"/> class.
+    /// </summary>
+    /// <param name="executedTests">Number of executed tests, or null when the runner could not count them.</param>
+    public SessionExecutedTestsMessage(long? executedTests)
+    {
+        ExecutedTests = executedTests;
+    }
+
+    /// <summary>
+    /// Gets or sets the number of executed tests, or null when the runner could not count them.
+    /// </summary>
+    [JsonProperty("executed_tests")]
+    public long? ExecutedTests { get; set; }
+}

tracer/src/Datadog.Trace/Ci/Tags/TestTags.cs

@@ -91,6 +91,21 @@ internal static class TestTags
     /// </summary>
     public const string SkipReason = "test.skip_reason";
 
+    /// <summary>
+    /// Skip reason value for a test session that ran no tests
+    /// </summary>
+    public const string SkipReasonNoTestsExecuted = "No tests were executed";
+
+    /// <summary>
+    /// Reason why a test session ran no tests
+    /// </summary>
+    public const string SessionEmptyReason = "test.session.empty_reason";
+
+    /// <summary>
+    /// Session empty reason value when the test runner executed no tests
+    /// </summary>
+    public const string SessionEmptyReasonZeroTests = "zero_tests";
+
     /// <summary>
     /// Test output message
     /// </summary>

tracer/src/Datadog.Trace/Ci/TestSession.cs

@@ -72,6 +72,12 @@ public sealed class TestSession
     /// </summary>
     private long _lastCoverageIpcMessageTicks;
 
+    /// <summary>
+    /// Tracks the tests executed by test runners: 0 until one reports, 1 while every report counted zero tests,
+    /// and 2 once a runner executed tests or could not count them.
+    /// </summary>
+    private int _executedTestsState;
+
     private TestSession(string? command, string? workingDirectory, string? framework, DateTimeOffset? startDate, bool propagateEnvironmentVariables)
     {
         // First we make sure that CI Visibility is initialized.
@@ -440,6 +446,15 @@ internal bool InternalClose(TestStatus status, TimeSpan? duration)
         }
 
         WaitForActiveIpcCallbacks(TimeSpan.FromMilliseconds(100));
+
+        // Test runners report over IPC, and disposing the server above reads every report already sent.
+        if (status == TestStatus.Pass && Volatile.Read(ref _executedTestsState) == 1)
+        {
+            Tags.Status = TestTags.StatusSkip;
+            SetTag(TestTags.SkipReason, TestTags.SkipReasonNoTestsExecuted);
+            SetTag(TestTags.SessionEmptyReason, TestTags.SessionEmptyReasonZeroTests);
+        }
+
         // Coverlet XML fallback can be processed in-process before the final close path.
         // In that case there is no persisted fallback result to wait for.
         var waitForCoverletXmlFallback =
@@ -529,6 +544,23 @@ internal TestModule CreateModule(string name, string framework, string framework
         return new TestModule(name, framework, frameworkVersion, startDate, Tags);
     }
 
+    /// <summary>
+    /// Records the number of tests executed by a test runner, such as vstest.console. A session that closes as passed
+    /// is reported as skipped when every runner that reported executed zero tests.
+    /// </summary>
+    /// <param name="executedTests">Number of executed tests, or null when the runner could not count them.</param>
+    internal void RecordExecutedTests(long? executedTests)
+    {
+        if (executedTests == 0)
+        {
+            Interlocked.CompareExchange(ref _executedTestsState, 1, 0);
+        }
+        else
+        {
+            Volatile.Write(ref _executedTestsState, 2);
+        }
+    }
+
     internal bool EnableIpcServer()
     {
         if (Tags.SessionId == 0)
@@ -783,6 +815,11 @@ private void OnIpcMessageReceived(object message)
                     SetTag(tagMessage.Name, tagMessage.NumberValue);
                 }
             }
+            else if (message is SessionExecutedTestsMessage executedTestsMessage)
+            {
+                _testOptimization.Log.Information<long?>("TestSession.ReceiveMessage (executed tests): {ExecutedTests}", executedTestsMessage.ExecutedTests);
+                RecordExecutedTests(executedTestsMessage.ExecutedTests);
+            }
             else if (message is SessionCodeCoverageReferenceMessage codeCoverageReferenceMessage)
             {
                 Interlocked.Increment(ref _coverageIpcMessageCount);

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/DotnetTest/CoverageGetCoverageResultIntegration.cs

@@ -12,9 +12,7 @@
 using Datadog.Trace.Ci.Ipc.Messages;
 using Datadog.Trace.ClrProfiler.CallTarget;
 using Datadog.Trace.DuckTyping;
-using Datadog.Trace.Propagators;
 using Datadog.Trace.Telemetry;
-using Datadog.Trace.Util;
 using Datadog.Trace.Vendors.Newtonsoft.Json.Utilities;
 
 namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Testing.DotnetTest;
@@ -72,7 +70,7 @@ internal static CallTargetReturn<TReturn> OnMethodEnd<TTarget, TReturn>(TTarget
                 CoverletCoverageBackfill.CoverletCoverageBackfillRollback? rollback = null;
                 try
                 {
-                    if (!TryGetParentSessionSpanId(out var sessionSpanId))
+                    if (!DotnetCommon.TryGetParentSessionSpanId(out var sessionSpanId))
                     {
                         Common.Log.Warning("CoverageGetCoverageResultIntegration: Could not find the parent test session context for Coverlet coverage IPC.");
                         TelemetryFactory.Metrics.RecordCountCIVisibilityCodeCoverageErrors();
@@ -184,25 +182,9 @@ rollback is not null &&
         return new CallTargetReturn<TReturn>(returnValue);
     }
 
-    private static bool TryGetParentSessionSpanId(out ulong sessionSpanId)
-    {
-        var context = Tracer.Instance.TracerManager.SpanContextPropagator.Extract(
-            EnvironmentHelpers.GetEnvironmentVariables(),
-            new DictionaryGetterAndSetter(DictionaryGetterAndSetter.EnvironmentVariableKeyProcessor));
-
-        if (context.SpanContext is { } sessionContext)
-        {
-            sessionSpanId = sessionContext.SpanId;
-            return true;
-        }
-
-        sessionSpanId = 0;
-        return false;
-    }
-
     private static void RecordCoverletCoverageIpcFailure()
     {
-        if (TryGetParentSessionSpanId(out var sessionSpanId))
+        if (DotnetCommon.TryGetParentSessionSpanId(out var sessionSpanId))
         {
             CoverageBackfillDataStore.RecordCoverageIpcFailure(sessionSpanId, nameof(CodeCoverageReportSource.Coverlet));
         }

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/DotnetTest/DotnetCommon.cs

@@ -166,11 +166,7 @@ internal static bool IsMsBuildTask
             }
 
             // Let's detect if we already have a session for this test process
-            var context = Tracer.Instance.TracerManager.SpanContextPropagator.Extract(
-                EnvironmentHelpers.GetEnvironmentVariables(),
-                new DictionaryGetterAndSetter(DictionaryGetterAndSetter.EnvironmentVariableKeyProcessor));
-
-            if (context.SpanContext is not null)
+            if (TryGetParentSessionSpanId(out _))
             {
                 // Session found in the environment variables
                 // let's capture current Coverlet XML attachments for stale-attachment filtering and bail out.
@@ -236,6 +232,27 @@ internal static bool IsMsBuildTask
         internal static bool IsVSTestArtifactsPostprocessCommand(string? commandLine)
             => commandLine?.IndexOf(VSTestArtifactsPostprocessArgument, StringComparison.OrdinalIgnoreCase) >= 0;
 
+        /// <summary>
+        /// Gets the test session that was propagated to this process through environment variables, by a parent process or by this one.
+        /// </summary>
+        /// <param name="sessionSpanId">Span id of the propagated test session.</param>
+        /// <returns>True when a test session context was found in the environment variables.</returns>
+        internal static bool TryGetParentSessionSpanId(out ulong sessionSpanId)
+        {
+            var context = Tracer.Instance.TracerManager.SpanContextPropagator.Extract(
+                EnvironmentHelpers.GetEnvironmentVariables(),
+                new DictionaryGetterAndSetter(DictionaryGetterAndSetter.EnvironmentVariableKeyProcessor));
+
+            if (context.SpanContext is { } sessionContext)
+            {
+                sessionSpanId = sessionContext.SpanId;
+                return true;
+            }
+
+            sessionSpanId = 0;
+            return false;
+        }
+
         /// <summary>
         /// Captures Coverlet collector XML attachments that already exist before the test command writes current results.
         /// </summary>
@@ -279,6 +296,9 @@ internal static void FinalizeSession(TestSession? session, int exitCode, Excepti
             if (exception is not null)
             {
                 session.SetErrorInfo(exception);
+
+                // The exit code is 0 when the runner throws, so the session can't be reported as empty.
+                session.RecordExecutedTests(null);
             }
 
             var codeCoveragePath = EnvironmentHelpers.GetEnvironmentVariable(Configuration.ConfigurationKeys.CIVisibility.CodeCoveragePath);
@@ -678,11 +698,7 @@ internal static bool TryProcessInjectedSessionCoverletCollectorXmlReports()
 
         private static CoverletCollectorXmlProcessingResult TryProcessInjectedSessionCoverletCollectorXmlReports(bool recordIpcFailureOnFailure)
         {
-            var context = Tracer.Instance.TracerManager.SpanContextPropagator.Extract(
-                EnvironmentHelpers.GetEnvironmentVariables(),
-                new DictionaryGetterAndSetter(DictionaryGetterAndSetter.EnvironmentVariableKeyProcessor));
-
-            if (context.SpanContext is not { } sessionContext)
+            if (!TryGetParentSessionSpanId(out var sessionSpanId))
             {
                 return CoverletCollectorXmlProcessingResult.NotApplicable;
             }
@@ -700,11 +716,11 @@ private static CoverletCollectorXmlProcessingResult TryProcessInjectedSessionCov
                     commandLine,
                     workingDirectory,
                     GetCurrentProcessStartTime(),
-                    (CoverletCollectorXmlReport coverageReport, ExternalCoverageXmlBackfill.CoverageBackfillValidationState validationState, out ExternalCoverageXmlResult result) => TryProcessCoverletCollectorXmlForCurrentProcess(sessionContext.SpanId, coverageReport, validationState, out result),
-                    (coverageReports, coverageResult, backfillValidated) => SendInjectedSessionCoverageResult(sessionContext.SpanId, coverageReports, coverageResult, backfillValidated));
+                    (CoverletCollectorXmlReport coverageReport, ExternalCoverageXmlBackfill.CoverageBackfillValidationState validationState, out ExternalCoverageXmlResult result) => TryProcessCoverletCollectorXmlForCurrentProcess(sessionSpanId, coverageReport, validationState, out result),
+                    (coverageReports, coverageResult, backfillValidated) => SendInjectedSessionCoverageResult(sessionSpanId, coverageReports, coverageResult, backfillValidated));
                 if (result != CoverletCollectorXmlProcessingResult.Processed && recordIpcFailureOnFailure)
                 {
-                    RecordCoverletXmlFallbackIpcFailure(sessionContext.SpanId);
+                    RecordCoverletXmlFallbackIpcFailure(sessionSpanId);
                 }
 
                 return result;
@@ -714,7 +730,7 @@ private static CoverletCollectorXmlProcessingResult TryProcessInjectedSessionCov
                 Log.Warning(ex, "RunCiCommand: Error while processing injected-session Coverlet collector XML fallback.");
                 if (recordIpcFailureOnFailure)
                 {
-                    RecordCoverletXmlFallbackIpcFailure(sessionContext.SpanId);
+                    RecordCoverletXmlFallbackIpcFailure(sessionSpanId);
                 }
 
                 return CoverletCollectorXmlProcessingResult.FailedClosed;

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/DotnetTest/ManagedVanguardStopIntegration.cs

@@ -17,7 +17,6 @@
 using Datadog.Trace.Ci.Ipc.Messages;
 using Datadog.Trace.ClrProfiler.CallTarget;
 using Datadog.Trace.DuckTyping;
-using Datadog.Trace.Propagators;
 using Datadog.Trace.Telemetry;
 using Datadog.Trace.Util;
 
@@ -63,7 +62,7 @@ internal static CallTargetReturn OnMethodEnd<TTarget>(TTarget instance, Exceptio
                 return CallTargetReturn.GetDefault();
             }
 
-            if (!TryGetParentSessionSpanId(out var parentSessionSpanId))
+            if (!DotnetCommon.TryGetParentSessionSpanId(out var parentSessionSpanId))
             {
                 Common.Log.Warning("ManagedVanguardStopIntegration: Could not find the parent test session context for Microsoft CodeCoverage IPC.");
                 RecordMicrosoftCoverageIpcFailure();
@@ -263,7 +262,7 @@ private static string GetMicrosoftCoverageResultId(ulong sessionSpanId, IReadOnl
     private static void RecordMicrosoftCoverageIpcFailure()
     {
         TelemetryFactory.Metrics.RecordCountCIVisibilityCodeCoverageErrors();
-        if (TryGetParentSessionSpanId(out var sessionSpanId))
+        if (DotnetCommon.TryGetParentSessionSpanId(out var sessionSpanId))
         {
             CoverageBackfillDataStore.RecordCoverageIpcFailure(sessionSpanId, nameof(CodeCoverageReportSource.MicrosoftCodeCoverage));
         }
@@ -275,22 +274,6 @@ private static void RecordMicrosoftCoverageIpcFailure(ulong sessionSpanId)
         CoverageBackfillDataStore.RecordCoverageIpcFailure(sessionSpanId, nameof(CodeCoverageReportSource.MicrosoftCodeCoverage));
     }
 
-    private static bool TryGetParentSessionSpanId(out ulong sessionSpanId)
-    {
-        var context = Tracer.Instance.TracerManager.SpanContextPropagator.Extract(
-            EnvironmentHelpers.GetEnvironmentVariables(),
-            new DictionaryGetterAndSetter(DictionaryGetterAndSetter.EnvironmentVariableKeyProcessor));
-
-        if (context.SpanContext is { } sessionContext)
-        {
-            sessionSpanId = sessionContext.SpanId;
-            return true;
-        }
-
-        sessionSpanId = 0;
-        return false;
-    }
-
     private static bool TryReadCoverageXmlReportBackup(string filePath, out CoverageXmlReportBackup backup)
     {
         try