Warm up container before parallel test dispatch

build/scripts/ParallelTestExecution.psm1

@@ -564,6 +564,60 @@ function Merge-TenantTestResults {
     }
 }
 
+<#
+.SYNOPSIS
+    Runs the first test app alone and awaits it before the parallel fan-out. Returns the remaining
+    apps still to dispatch.
+.DESCRIPTION
+    Concurrent per-tenant company-opens can race on the first use of the container's process-wide
+    GDI+ state, so the first open is serialized: one app runs alone and is awaited to completion,
+    then the caller fans out the rest. Warms once per container - no-op for a single app or tenant.
+    A transient failure on the warmed-up app flows into $State.transient and is re-queued normally.
+.PARAMETER Pending
+    The ordered list of app names still to dispatch. The first app is consumed for the warmup.
+.PARAMETER AppIdByName
+    Map of app name -> extensionId. If the first app's id cannot be resolved, warmup is skipped.
+.PARAMETER Tenants
+    All available tenant ids. Warmup dispatches onto the first one.
+.PARAMETER State
+    The parallel execution state object; mutated (jobs/hasFailures/transient) as the warmup runs.
+.OUTPUTS
+    [string[]] The remaining app names to dispatch (first app removed if it was warmed up).
+#>
+function Invoke-WarmupDispatch {
+    param(
+        [Parameter(Mandatory=$true)][Hashtable]$Parameters,
+        [Parameter(Mandatory=$true)][AllowEmptyCollection()][string[]]$Pending,
+        [Parameter(Mandatory=$true)][Hashtable]$AppIdByName,
+        [Parameter(Mandatory=$true)][AllowEmptyCollection()][string[]]$Tenants,
+        [Parameter(Mandatory=$true)][string]$ScriptPath,
+        [string]$TestType,
+        [Parameter(Mandatory=$true)]$State
+    )
+
+    # Only serialize when there is a fan-out to protect: >1 app AND >1 tenant.
+    if ($Pending.Count -le 1 -or $Tenants.Count -le 1) {
+        return @($Pending)
+    }
+
+    $warmupApp = $Pending[0]
+    $warmupAppId = $AppIdByName[$warmupApp]
+    if (-not $warmupAppId) {
+        # Leave the app in the queue so the main loop emits its usual appId warning.
+        return @($Pending)
+    }
+
+    Write-Host "Warming up: dispatching first app '$warmupApp' on '$($Tenants[0])' alone and awaiting completion before parallel fan-out."
+    Start-TestAppDispatch -Parameters $Parameters -AppName $warmupApp -AppId $warmupAppId -Tenant $Tenants[0] `
+        -ScriptPath $ScriptPath -TestType $TestType -State $State -Verb 'Dispatching'
+
+    # Await the single job so the process is warm before anything runs in parallel. A transient
+    # failure here lands in $State.transient and the caller's loop re-queues it.
+    if (-not (Wait-ForAllTestJobs -state $State)) { $State.hasFailures = $true }
+
+    return @($Pending | Select-Object -Skip 1)
+}
+
 <#
 .SYNOPSIS
     Dispatches test apps in parallel across all available tenants in a BC container.
@@ -633,12 +687,16 @@ function Invoke-ParallelTestExecution {
     $state = [PSCustomObject]@{ jobs = @(); dispatched = $true; completed = $false; finalResult = $false; hasFailures = $false; transient = @(); retried = @{} }
     $state | ConvertTo-Json -Depth 5 | Set-Content $stateFile -Force
 
-    # Single dispatch loop. $pending is processed FIFO and $appNamesToTest arrives ordered
-    # longest-first (see TestConfiguration.json), which is the LPT schedule that keeps the
-    # tail short. The retry cap lives in Receive-TestJobResult: an app already in
-    # $state.retried gets classified as Failed (not Transient) on a second failure.
+    # Single dispatch loop, FIFO. TestConfiguration.json lists the smallest app first (a cheap
+    # serial warmup) and the rest longest-first (LPT, keeps the tail short). The retry cap lives in
+    # Receive-TestJobResult: an app already in $state.retried is classified as Failed on a re-fail.
     $pending = @($appNamesToTest)
 
+    # Run the first app alone and await it to warm the container before parallelizing the rest.
+    # No-op for single-app/single-tenant.
+    $pending = @(Invoke-WarmupDispatch -Parameters $parameters -Pending $pending -AppIdByName $appIdByName `
+        -Tenants $tenants -ScriptPath $scriptPath -TestType $testType -State $state)
+
     while ($pending.Count -gt 0 -or $state.jobs.Count -gt 0 -or $state.transient.Count -gt 0) {
         # Promote any transient failures back into the dispatch queue. They go to the FRONT:
         # a platform race normally kills a job within a minute of dispatch, so the victim is
@@ -732,4 +790,4 @@ function Invoke-PerProjectTestRun {
     return (. $script -parameters $parameters -TestType $testType -AppNamesToTest $appNamesToTest)
 }
 
-Export-ModuleMember -Function Invoke-ParallelTestExecution, Get-AvailableBcTenants, Get-CachedTestRunResult, Get-InstalledTestAppNames, Get-AppNamesForBucket, Invoke-PerProjectTestRun, Get-AppNameFromMetadata
+Export-ModuleMember -Function Invoke-ParallelTestExecution, Get-AvailableBcTenants, Get-CachedTestRunResult, Get-InstalledTestAppNames, Get-AppNamesForBucket, Invoke-PerProjectTestRun, Get-AppNameFromMetadata, Invoke-WarmupDispatch

build/scripts/TestConfiguration.json

@@ -1,5 +1,6 @@
 {
   "LegacyTests-Bucket1": [
+    "Tests-Resource",
     "Tests-Workflow",
     "AlCosting",
     "Tests-ERM-Application",
@@ -10,6 +11,7 @@
     "Tests-SINGLESERVER",
     "Tests-Job",
     "Tests-ERM-Purchase",
+    "Tests-Local",
     "Tests-ERM-Sales",
     "Tests-ERM-Finance",
     "Tests-Dimension",
@@ -22,11 +24,10 @@
     "Tests-Cost Accounting",
     "Tests-User",
     "Tests-Cash Flow",
-    "Tests-Resource",
-    "Tests-Azure AI",
-    "Tests-Local"
+    "Tests-Azure AI"
   ],
   "LegacyTests-Bucket2": [
+    "Tests-Upgrade",
     "Tests-SCM-Assembly",
     "Tests-SCM",
     "Tests-SCM-Workflow",
@@ -49,7 +50,6 @@
     "Tests-Monitor Sensitive Fields",
     "Performance Toolkit Samples",
     "Tests-Integration-Internal",
-    "Tests-DotNet-Internal",
-    "Tests-Upgrade"
+    "Tests-DotNet-Internal"
   ]
 }

build/scripts/tests/ParallelTestExecution.Test.ps1

@@ -139,3 +139,117 @@ Describe "ParallelTestExecution transient retry scheduling" {
         }
     }
 }
+
+Describe "ParallelTestExecution warmup dispatch" {
+    BeforeAll {
+        Import-Module (Join-Path $PSScriptRoot '../ParallelTestExecution.psm1') -Force
+    }
+
+    It "dispatches the first app alone and awaits it before fanning out the rest" {
+        # The first app must run alone and be awaited before any parallel dispatch. This asserts
+        # exactly that ordering: dispatch(first) -> wait -> rest.
+        InModuleScope ParallelTestExecution {
+            $script:events = [System.Collections.Generic.List[string]]::new()
+
+            Mock Get-AvailableBcTenants { @('default', 'tenant2') }
+            Mock Get-BcContainerAppInfo {
+                @('Big', 'Medium', 'Small') | ForEach-Object {
+                    [PSCustomObject]@{ IsInstalled = $true; Name = $_; AppId = "id-$_" }
+                }
+            }
+            Mock Wait-ForFreeTenant { 'tenant2' }
+            Mock Merge-TenantTestResults { }
+            Mock Start-TestAppDispatch { $script:events.Add("dispatch:$AppName") }
+            Mock Wait-ForAllTestJobs { $script:events.Add('wait'); $true }
+
+            $params = @{ containerName = "ut-$([guid]::NewGuid().ToString('N'))"; tenant = 'default' }
+            $null = Invoke-ParallelTestExecution -parameters $params -scriptPath 'unused.ps1' `
+                -testType 'Legacy' -appNamesToTest @('Big', 'Medium', 'Small')
+
+            # First app dispatched alone, then awaited, before any other app is dispatched.
+            $script:events[0] | Should -Be 'dispatch:Big'
+            $script:events[1] | Should -Be 'wait'
+            $waitIndex = $script:events.IndexOf('wait')
+            $script:events.IndexOf('dispatch:Medium') | Should -BeGreaterThan $waitIndex
+            $script:events.IndexOf('dispatch:Small')  | Should -BeGreaterThan $waitIndex
+        }
+    }
+
+    It "skips the warmup dispatch when only one tenant is available" {
+        InModuleScope ParallelTestExecution {
+            $script:events = [System.Collections.Generic.List[string]]::new()
+
+            Mock Get-AvailableBcTenants { @('default') }
+            Mock Get-BcContainerAppInfo {
+                @('Big', 'Medium') | ForEach-Object {
+                    [PSCustomObject]@{ IsInstalled = $true; Name = $_; AppId = "id-$_" }
+                }
+            }
+            Mock Wait-ForFreeTenant { 'default' }
+            Mock Merge-TenantTestResults { }
+            Mock Start-TestAppDispatch { $script:events.Add("dispatch:$AppName") }
+            Mock Wait-ForAllTestJobs { $script:events.Add('wait'); $true }
+
+            $params = @{ containerName = "ut-$([guid]::NewGuid().ToString('N'))"; tenant = 'default' }
+            $null = Invoke-ParallelTestExecution -parameters $params -scriptPath 'unused.ps1' `
+                -testType 'Legacy' -appNamesToTest @('Big', 'Medium')
+
+            # No warmup dispatch: the two apps are dispatched by the normal loop with no leading
+            # solo dispatch+await. (Start-TestAppDispatch is mocked so no jobs accumulate, hence
+            # the loop's terminal Wait-ForAllTestJobs is not reached.)
+            $script:events | Should -Be @('dispatch:Big', 'dispatch:Medium')
+            $script:events | Should -Not -Contain 'wait'
+        }
+    }
+
+    It "returns the pending list unchanged when there is a single tenant" {
+        InModuleScope ParallelTestExecution {
+            $state = [PSCustomObject]@{ jobs = @(); hasFailures = $false; transient = @(); retried = @{} }
+            $result = Invoke-WarmupDispatch -Parameters @{ containerName = 'c' } `
+                -Pending @('A', 'B', 'C') -AppIdByName @{ A = 'id-A'; B = 'id-B'; C = 'id-C' } `
+                -Tenants @('default') -ScriptPath 'unused.ps1' -TestType 'Legacy' -State $state
+            $result | Should -Be @('A', 'B', 'C')
+        }
+    }
+
+    It "returns the pending list unchanged when there is a single app" {
+        InModuleScope ParallelTestExecution {
+            Mock Start-TestAppDispatch { }
+            Mock Wait-ForAllTestJobs { $true }
+            $state = [PSCustomObject]@{ jobs = @(); hasFailures = $false; transient = @(); retried = @{} }
+            $result = Invoke-WarmupDispatch -Parameters @{ containerName = 'c' } `
+                -Pending @('Only') -AppIdByName @{ Only = 'id-Only' } `
+                -Tenants @('default', 'tenant2') -ScriptPath 'unused.ps1' -TestType 'Legacy' -State $state
+            $result | Should -Be @('Only')
+            Should -Invoke Start-TestAppDispatch -Times 0
+        }
+    }
+
+    It "warms up the first app and returns the remaining apps" {
+        InModuleScope ParallelTestExecution {
+            Mock Start-TestAppDispatch { }
+            Mock Wait-ForAllTestJobs { $true }
+            $state = [PSCustomObject]@{ jobs = @(); hasFailures = $false; transient = @(); retried = @{} }
+            $result = Invoke-WarmupDispatch -Parameters @{ containerName = 'c' } `
+                -Pending @('A', 'B', 'C') -AppIdByName @{ A = 'id-A'; B = 'id-B'; C = 'id-C' } `
+                -Tenants @('default', 'tenant2') -ScriptPath 'unused.ps1' -TestType 'Legacy' -State $state
+            $result | Should -Be @('B', 'C')
+            Should -Invoke Start-TestAppDispatch -Times 1
+        }
+    }
+
+    It "leaves a transient warmup failure in State.transient for the caller to re-queue" {
+        InModuleScope ParallelTestExecution {
+            Mock Start-TestAppDispatch { }
+            # Simulate Wait-ForAllTestJobs classifying the warmup app as a transient race.
+            Mock Wait-ForAllTestJobs { $State.transient = @('A'); $true }
+            $state = [PSCustomObject]@{ jobs = @(); hasFailures = $false; transient = @(); retried = @{} }
+            $result = Invoke-WarmupDispatch -Parameters @{ containerName = 'c' } `
+                -Pending @('A', 'B', 'C') -AppIdByName @{ A = 'id-A'; B = 'id-B'; C = 'id-C' } `
+                -Tenants @('default', 'tenant2') -ScriptPath 'unused.ps1' -TestType 'Legacy' -State $state
+            $result | Should -Be @('B', 'C')
+            $state.transient | Should -Contain 'A'
+            $state.hasFailures | Should -BeFalse
+        }
+    }
+}