Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions src/TUnit.Engine/Framework/TUnitTestFramework.cs
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,22 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context)
return;
}

var message = context.CancellationToken.IsCancellationRequested
? "The test run was cancelled."
: "Test execution stopped due to fail-fast.";
await serviceProvider.Logger.LogErrorAsync(message);
if (!context.CancellationToken.IsCancellationRequested)
{
// Fail-fast (or the engine's own cancellation) fired without the platform's token,
// so, as with SIGTERM above, MTP would treat a re-thrown cancellation as unhandled
// and crash the host. The tests that ran are already reported; fail the session.
await serviceProvider.Logger.LogErrorAsync(serviceProvider.FailFastCancellationSource.IsCancellationRequested
? "Test execution stopped due to fail-fast."
: "The test run was cancelled.");
serviceProvider.SessionFailed = true;
return;
}

await serviceProvider.Logger.LogErrorAsync("The test run was cancelled.");

// Re-throw is safe here — MTP handles OperationCanceledException specially.
// Re-throw is safe here: MTP handles OperationCanceledException specially when its own
// token was cancelled.
throw;
}
catch (Exception e)
Expand Down
25 changes: 24 additions & 1 deletion src/TUnit.Engine/Scheduling/TestScheduler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ private async Task ProcessDynamicTestQueueAsync(CancellationToken cancellationTo
var dynamicTests = new List<AbstractExecutableTest>();

// Use async signaling instead of polling to eliminate IOCP overhead
while (await _dynamicTestQueue.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
while (await WaitForDynamicTestsAsync(cancellationToken).ConfigureAwait(false))
{
// Dequeue all currently available tests (batch processing)
while (_dynamicTestQueue.TryDequeue(out var test))
Expand All @@ -358,6 +358,13 @@ private async Task ProcessDynamicTestQueueAsync(CancellationToken cancellationTo
}
}

// Cancelled (fail-fast or run abort): the queued tests would only be cancelled too, and
// the phases would throw on the cancelled token, so stop here.
if (cancellationToken.IsCancellationRequested)
{
return;
}

// Process any remaining tests after queue completion
while (_dynamicTestQueue.TryDequeue(out var test))
{
Expand All @@ -376,6 +383,22 @@ private async Task ProcessDynamicTestQueueAsync(CancellationToken cancellationTo
}
}

// Ends the wait loop instead of faulting when the token is cancelled. The phases can finish
// without throwing after fail-fast fires (the failing test was the last to run), and
// ExecuteGroupedTestsAsync then awaits this processor on its normal path, so a cancelled
// wait would surface as the scheduler's exception.
private async ValueTask<bool> WaitForDynamicTestsAsync(CancellationToken cancellationToken)
{
try
{
return await _dynamicTestQueue.WaitToReadAsync(cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return false;
}
}

#if NET
[System.Diagnostics.CodeAnalysis.RequiresUnreferencedCode("Test execution involves reflection for hooks and initialization")]
#endif
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
using CliWrap.Buffered;
using Shouldly;
using TUnit.Engine.Tests.Enums;

namespace TUnit.Engine.Tests.Scheduling;

/// <summary>
/// With --fail-fast and --maximum-parallel-tests 1, the failing keyed test cancels the keyed
/// tests still waiting for a parallelism slot, so the scheduling phases throw. The
/// After(TestSession) hook must still be awaited, before shared fixtures are disposed,
/// instead of being left running in the background while the session tears down.
/// tests queued behind it, so the scheduling phases throw. The After(TestSession) hook must
/// still be awaited, before shared fixtures are disposed, instead of being left running in the
/// background while the session tears down. Fail-fast must also end the run as a failed session
/// rather than crash the test host with an unhandled cancellation.
/// </summary>
public class KeyedNotInParallelFailFastTests(TestMode testMode) : InvokableTestBase(testMode)
{
Expand All @@ -18,11 +20,13 @@ public async Task FailFast_WithWaitingKeyedTests_AwaitsSessionHooksBeforeFixture
{
var markerPath = Path.Combine(Path.GetTempPath(), $"tunit-7074-session-hook-{Guid.NewGuid():N}.txt");
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
var process = new ProcessResultCapture();

try
{
await RunTestsWithFilter("/*/*/KeyedNotInParallelFailFast7074/*",
[
_ => process.AssertEndedWithoutCrash(),
result => result.ResultSummary.Outcome.ShouldBe("Failed"),
result => result.ResultSummary.Counters.Failed.ShouldBeGreaterThanOrEqualTo(1),
// Fails runs first and holds the only slot, so fail-fast must cancel at least one
Expand All @@ -36,7 +40,8 @@ await RunTestsWithFilter("/*/*/KeyedNotInParallelFailFast7074/*",
.WithArgument("--maximum-parallel-tests")
.WithArgument("1")
.WithEnvironmentVariable(MarkerVariable, markerPath)
.WithForcefulCancellationToken(cts.Token));
.WithForcefulCancellationToken(cts.Token)
.Capture(process));
}
finally
{
Expand All @@ -54,11 +59,13 @@ public async Task FailFast_WithRunningDynamicTest_AwaitsDynamicTestBeforeFixture
{
var markerPath = Path.Combine(Path.GetTempPath(), $"tunit-7074-dynamic-{Guid.NewGuid():N}.txt");
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
var process = new ProcessResultCapture();

try
{
await RunTestsWithFilter("/*/*/KeyedNotInParallelFailFastDynamic7074/*",
[
_ => process.AssertEndedWithoutCrash(),
result => result.ResultSummary.Outcome.ShouldBe("Failed"),
result => result.ResultSummary.Counters.Failed.ShouldBeGreaterThanOrEqualTo(1),
_ => File.Exists(markerPath).ShouldBeTrue($"Marker file '{markerPath}' was not written: the session ended before the dynamic test finished"),
Expand All @@ -69,7 +76,8 @@ await RunTestsWithFilter("/*/*/KeyedNotInParallelFailFastDynamic7074/*",
.WithArgument("--maximum-parallel-tests")
.WithArgument("2")
.WithEnvironmentVariable(DynamicMarkerVariable, markerPath)
.WithForcefulCancellationToken(cts.Token));
.WithForcefulCancellationToken(cts.Token)
.Capture(process));
}
finally
{
Expand All @@ -79,4 +87,62 @@ await RunTestsWithFilter("/*/*/KeyedNotInParallelFailFastDynamic7074/*",
}
}
}

/// <summary>
/// When the failing test is the last one to run, the phases finish without throwing, so the
/// scheduler awaits the dynamic test queue processor on its normal path. Its wait on the
/// queue must end cleanly on the cancelled token instead of surfacing an
/// OperationCanceledException. The framework would turn that exception into a failed session
/// (exit code 10) rather than a crash, so only exit code 2 ("at least one test failed") proves
/// the scheduler itself ended cleanly.
/// </summary>
[Test]
public async Task FailFast_WhenFailingTestRunsLast_EndsSessionWithoutCrash()
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
var process = new ProcessResultCapture();

await RunTestsWithFilter("/*/*/KeyedNotInParallelFailFast7074/Fails",
[
_ => process.AssertEndedWithoutCrash(2),
result => result.ResultSummary.Outcome.ShouldBe("Failed"),
result => result.ResultSummary.Counters.Total.ShouldBe(1),
result => result.ResultSummary.Counters.Failed.ShouldBe(1),
],
new RunOptions()
.WithArgument("--fail-fast")
.WithForcefulCancellationToken(cts.Token)
.Capture(process));
}
}

internal sealed class ProcessResultCapture
{
public BufferedCommandResult? Result { get; set; }

public void AssertEndedWithoutCrash(params int[] expectedExitCodes)
{
if (expectedExitCodes.Length == 0)
{
// MTP's "at least one test failed" or "test session failed" codes, not a runtime crash.
expectedExitCodes = [2, 10];
}

var result = Result.ShouldNotBeNull();
var output = result.StandardOutput + result.StandardError;

// The runtime's crash banner for an exception that escaped Main.
output.ShouldNotContain("Unhandled exception. ");

result.ExitCode.ShouldBeOneOf(expectedExitCodes);
}
}

internal static class ProcessResultCaptureExtensions
{
public static RunOptions Capture(this RunOptions runOptions, ProcessResultCapture capture)
{
runOptions.OnExecutingDelegates.Add(async commandTask => capture.Result = await commandTask);
return runOptions;
}
}
23 changes: 14 additions & 9 deletions tests/TUnit.TestProject/Bugs/7074/KeyedNotInParallelFailFast.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,24 +16,29 @@ public ValueTask DisposeAsync()

/// <summary>
/// With --fail-fast and --maximum-parallel-tests 1, the failing keyed test cancels the keyed
/// tests still waiting for a parallelism slot. The After(TestSession) hook must still be awaited
/// before the session disposes shared fixtures, rather than left running in the background.
/// tests queued behind it, so the scheduling phases throw. The After(TestSession) hook must
/// still be awaited before the session disposes shared fixtures, rather than left running in
/// the background.
/// </summary>
[EngineTest(ExpectedResult.Failure)]
public class KeyedNotInParallelFailFast7074
{
[ClassDataSource<FailFastSessionFixture7074>(Shared = SharedType.PerTestSession)]
public required FailFastSessionFixture7074 Fixture { get; init; }

// Critical priority makes Fails the first keyed test scheduled, so it takes the only
// parallelism slot and fails while the Waits tests are still queued for that slot.
[Test, NotInParallel("7074.ff1"), ExecutionPriority(Priority.Critical)]
// All tests share one key. The scheduler hands keys out in priority order while it walks
// the tests, so Critical priority deterministically gives Fails the key first, and the
// Waits tests queue behind it. (With distinct keys every test would start at once and race
// for the single parallelism slot, so a Waits test could run before Fails.) When Fails
// releases the key, fail-fast has already cancelled the token, so each Waits test throws
// on its cancelled slot wait instead of running.
[Test, NotInParallel("7074.ff"), ExecutionPriority(Priority.Critical)]
public void Fails() => throw new InvalidOperationException("Intentional failure to trigger fail-fast");

[Test, NotInParallel("7074.ff2")] public Task Waits1() => Task.Delay(100);
[Test, NotInParallel("7074.ff3")] public Task Waits2() => Task.Delay(100);
[Test, NotInParallel("7074.ff4")] public Task Waits3() => Task.Delay(100);
[Test, NotInParallel("7074.ff5")] public Task Waits4() => Task.Delay(100);
[Test, NotInParallel("7074.ff")] public Task Waits1() => Task.Delay(100);
[Test, NotInParallel("7074.ff")] public Task Waits2() => Task.Delay(100);
[Test, NotInParallel("7074.ff")] public Task Waits3() => Task.Delay(100);
[Test, NotInParallel("7074.ff")] public Task Waits4() => Task.Delay(100);

// No-op unless the engine test opted in via the environment variable, so this hook is
// inert for every other TestProject invocation.
Expand Down
Loading