diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Redis/RedisServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Redis/RedisServerExecutorTests.cs index fa062f6097..623d20dd49 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Redis/RedisServerExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Redis/RedisServerExecutorTests.cs @@ -8,6 +8,7 @@ namespace VirtualClient.Actions using System.Linq; using System.Net; using System.Text; + using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -17,6 +18,7 @@ namespace VirtualClient.Actions using VirtualClient.Common; using VirtualClient.Common.Telemetry; using VirtualClient.Contracts; + using VirtualClient.TestExtensions; [TestFixture] [Category("Unit")] @@ -90,6 +92,8 @@ public async Task RedisServerExecutorConfirmsTheExpectedPackagesOnInitialization [Test] public async Task RedisMemtierServerExecutorExecutesExpectedProcessWhenBindingToCores() { + // OLD APPROACH: Manual command tracking with List + // This demonstrates the traditional way of tracking commands in tests using (var executor = new TestRedisServerExecutor(this.fixture.Dependencies, this.fixture.Parameters)) { List expectedCommands = new List() @@ -110,6 +114,7 @@ public async Task RedisMemtierServerExecutorExecutesExpectedProcessWhenBindingTo ); return this.memoryProcess; } + expectedCommands.Remove($"{exe} {arguments}"); return this.fixture.Process; }; @@ -119,6 +124,42 @@ public async Task RedisMemtierServerExecutorExecutesExpectedProcessWhenBindingTo } } + [Test] + [Category("POC")] + public async Task RedisMemtierServerExecutorExecutesExpectedProcessWhenBindingToCores_WithTracking() + { + // NEW APPROACH: Automatic tracking with fluent assertions + + // ? STEP 1: Enable automatic tracking + setup version check output (chainable!) + this.fixture + .TrackProcesses() + .SetupProcessOutput( + "redis-server.*--version", + "Redis server v=7.0.15 sha=00000000 malloc=jemalloc-5.1.0 bits=64 build=abc123"); + + using (var executor = new TestRedisServerExecutor(this.fixture.Dependencies, this.fixture.Parameters)) + { + // ? STEP 2: Execute (no manual tracking needed!) + await executor.ExecuteAsync(CancellationToken.None); + + // ? STEP 3: Assert with fluent, self-documenting assertions + this.fixture.Tracking.AssertCommandsExecutedInOrder( + // Verify chmod command + $@"sudo chmod \+x ""{Regex.Escape(this.mockRedisPackage.Path)}/src/redis-server""", + + // Verify redis-server startup with numactl binding + $@"sudo bash -c ""numactl -C 0 {Regex.Escape(this.mockRedisPackage.Path)}/src/redis-server --port 6379 --protected-mode no --io-threads 4 --maxmemory-policy noeviction --ignore-warnings ARM64-COW-BUG --save --daemonize yes""" + ); + + // ? OPTIONAL: Additional verification types + this.fixture.Tracking.AssertCommandExecutedTimes("chmod", 1); + this.fixture.Tracking.AssertCommandExecutedTimes("numactl", 1); + + // ? DEBUGGING: Detailed summary available on demand + // TestContext.WriteLine(this.fixture.Tracking.GetDetailedSummary()); + } + } + [Test] public async Task RedisMemtierServerExecutorExecutesExpectedProcessWhenBindingToCores_2_Server_Instances() { diff --git a/src/VirtualClient/VirtualClient.TestExtensions/MockFixtureTrackingAssertions.cs b/src/VirtualClient/VirtualClient.TestExtensions/MockFixtureTrackingAssertions.cs new file mode 100644 index 0000000000..7e111d1c45 --- /dev/null +++ b/src/VirtualClient/VirtualClient.TestExtensions/MockFixtureTrackingAssertions.cs @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.TestExtensions +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Text.RegularExpressions; + using NUnit.Framework; + using VirtualClient.Common; + + /// + /// Fluent assertion extensions for MockFixture tracking. + /// + public static class MockFixtureTrackingAssertions + { + /// + /// Asserts that specific commands were executed in the exact order specified. + /// Supports regex patterns for flexible matching. + /// + /// The fixture tracking instance. + /// The expected commands in order (supports regex patterns). + public static void AssertCommandsExecutedInOrder(this FixtureTracking tracking, params string[] expectedCommands) + { + var actualCommands = tracking.Commands.Select(c => c.FullCommand).ToList(); + + if (expectedCommands.Length > actualCommands.Count) + { + Assert.Fail( + $"Expected {expectedCommands.Length} commands but only {actualCommands.Count} were executed.\n\n" + + FormatCommandMismatch(expectedCommands, actualCommands)); + } + + for (int i = 0; i < expectedCommands.Length; i++) + { + bool matches = TryMatchCommand(actualCommands[i], expectedCommands[i]); + + if (!matches) + { + Assert.Fail( + $"Command mismatch at position {i}.\n\n" + + $"Expected: {expectedCommands[i]}\n" + + $"Actual: {actualCommands[i]}\n\n" + + FormatCommandMismatch(expectedCommands, actualCommands, i)); + } + } + + if (actualCommands.Count > expectedCommands.Length) + { + var extraCommands = actualCommands.Skip(expectedCommands.Length); + Assert.Fail( + $"Unexpected additional commands executed:\n {string.Join("\n ", extraCommands)}\n\n" + + FormatCommandMismatch(expectedCommands, actualCommands)); + } + } + + /// + /// Asserts that specific commands were executed (order-independent). + /// Supports regex patterns for flexible matching. + /// + /// The fixture tracking instance. + /// The expected commands (supports regex patterns). + public static void AssertCommandsExecuted(this FixtureTracking tracking, params string[] expectedCommands) + { + var actualCommands = tracking.Commands.Select(c => c.FullCommand).ToList(); + var unmatchedExpected = new List(expectedCommands); + var matchedActual = new HashSet(); + + foreach (string expected in expectedCommands) + { + var match = actualCommands.FirstOrDefault(actual => + !matchedActual.Contains(actual) && TryMatchCommand(actual, expected)); + + if (match != null) + { + unmatchedExpected.Remove(expected); + matchedActual.Add(match); + } + } + + if (unmatchedExpected.Any()) + { + Assert.Fail( + $"Expected commands not executed:\n {string.Join("\n ", unmatchedExpected)}\n\n" + + $"Actual commands executed:\n {string.Join("\n ", actualCommands)}\n\n" + + $"Debugging: {unmatchedExpected.Count} expected command(s) did not match any of the {actualCommands.Count} commands executed."); + } + } + + /// + /// Asserts that a command was executed exactly N times. + /// Supports regex patterns for flexible matching. + /// + /// The fixture tracking instance. + /// The command pattern (supports regex). + /// The expected number of executions. + public static void AssertCommandExecutedTimes(this FixtureTracking tracking, string commandPattern, int expectedCount) + { + int actualCount = tracking.Commands.Count(c => TryMatchCommand(c.FullCommand, commandPattern)); + + if (actualCount != expectedCount) + { + var matches = tracking.Commands + .Where(c => TryMatchCommand(c.FullCommand, commandPattern)) + .Select(c => c.FullCommand); + + Assert.Fail( + $"Expected '{commandPattern}' to be executed {expectedCount} time(s), but was executed {actualCount} time(s).\n\n" + + $"Matching commands:\n {string.Join("\n ", matches)}\n\n" + + $"All commands:\n {string.Join("\n ", tracking.Commands.Select(c => c.FullCommand))}"); + } + } + + /// + /// Asserts that a file operation occurred. + /// + /// The fixture tracking instance. + /// The operation type (e.g., "Read", "Write", "Exists"). + /// The file path pattern (supports regex). + public static void AssertFileOperation(this FixtureTracking tracking, string operation, string filePath) + { + bool occurred = tracking.FileOperations.Any(fo => + fo.Operation == operation && + TryMatchCommand(fo.FilePath, filePath)); + + if (!occurred) + { + var relevantOps = tracking.FileOperations + .Where(fo => fo.Operation == operation) + .Select(fo => fo.FilePath); + + Assert.Fail( + $"Expected {operation} operation on '{filePath}' but it did not occur.\n\n" + + $"Actual {operation} operations:\n {string.Join("\n ", relevantOps)}"); + } + } + + /// + /// Asserts that a package operation occurred. + /// + /// The fixture tracking instance. + /// The operation type (e.g., "GetPackage", "InstallPackage"). + /// The package name. + public static void AssertPackageOperation(this FixtureTracking tracking, string operation, string packageName) + { + bool occurred = tracking.PackageOperations.Any(po => + po.Operation == operation && + string.Equals(po.PackageName, packageName, StringComparison.OrdinalIgnoreCase)); + + if (!occurred) + { + var relevantOps = tracking.PackageOperations + .Where(po => po.Operation == operation) + .Select(po => po.PackageName); + + Assert.Fail( + $"Expected {operation} operation on package '{packageName}' but it did not occur.\n\n" + + $"Actual {operation} operations:\n {string.Join("\n ", relevantOps)}"); + } + } + + /// + /// Tries to match a command against a pattern (supports regex). + /// + private static bool TryMatchCommand(string actualCommand, string expectedPattern) + { + try + { + return Regex.IsMatch(actualCommand, expectedPattern, RegexOptions.IgnoreCase); + } + catch + { + // If regex fails, try exact match + return string.Equals(actualCommand, expectedPattern, StringComparison.OrdinalIgnoreCase); + } + } + + /// + /// Formats a command mismatch for display. + /// + private static string FormatCommandMismatch(string[] expected, List actual, int? highlightIndex = null) + { + var sb = new StringBuilder(); + + sb.AppendLine("Expected commands:"); + for (int i = 0; i < expected.Length; i++) + { + string marker = (highlightIndex.HasValue && i == highlightIndex.Value) ? " >>> " : " "; + sb.AppendLine($"{marker}{i + 1}. {expected[i]}"); + } + + sb.AppendLine(); + + sb.AppendLine("Actual commands executed:"); + for (int i = 0; i < actual.Count; i++) + { + string marker = (highlightIndex.HasValue && i == highlightIndex.Value) ? " >>> " : " "; + sb.AppendLine($"{marker}{i + 1}. {actual[i]}"); + } + + return sb.ToString(); + } + } +} diff --git a/src/VirtualClient/VirtualClient.TestFramework/FixtureTracking.cs b/src/VirtualClient/VirtualClient.TestFramework/FixtureTracking.cs new file mode 100644 index 0000000000..a97b5fd99e --- /dev/null +++ b/src/VirtualClient/VirtualClient.TestFramework/FixtureTracking.cs @@ -0,0 +1,231 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using VirtualClient.Common; + + /// + /// Centralized tracking for mock fixture interactions during testing. + /// Provides visibility into processes, file operations, and package operations + /// executed during test runs. + /// + public class FixtureTracking + { + /// + /// Initializes a new instance of the class. + /// + public FixtureTracking() + { + this.Processes = new List(); + this.Commands = new List(); + this.FileOperations = new List(); + this.PackageOperations = new List(); + } + + /// + /// All processes created during the test execution. + /// + public List Processes { get; } + + /// + /// Detailed command execution information. + /// + public List Commands { get; } + + /// + /// File system operations performed during the test. + /// + public List FileOperations { get; } + + /// + /// Package manager operations performed during the test. + /// + public List PackageOperations { get; } + + /// + /// Clears all tracked data. + /// + public void Clear() + { + this.Processes.Clear(); + this.Commands.Clear(); + this.FileOperations.Clear(); + this.PackageOperations.Clear(); + } + + /// + /// Returns a summary of tracked operations. + /// + public string GetSummary() + { + return $"Processes: {this.Processes.Count}, " + + $"Commands: {this.Commands.Count}, " + + $"FileOps: {this.FileOperations.Count}, " + + $"PackageOps: {this.PackageOperations.Count}"; + } + + /// + /// Returns a detailed summary of all tracked operations for debugging. + /// + public string GetDetailedSummary() + { + var sb = new StringBuilder(); + sb.AppendLine("=== Fixture Tracking Summary ==="); + sb.AppendLine($"Total Processes: {this.Processes.Count}"); + sb.AppendLine($"Total Commands: {this.Commands.Count}"); + sb.AppendLine($"Total File Operations: {this.FileOperations.Count}"); + sb.AppendLine($"Total Package Operations: {this.PackageOperations.Count}"); + sb.AppendLine(); + + if (this.Commands.Any()) + { + sb.AppendLine("Commands Executed:"); + foreach (var cmd in this.Commands) + { + sb.AppendLine($" [{cmd.ExecutedAt:HH:mm:ss.fff}] {cmd.FullCommand}"); + if (cmd.ExitCode.HasValue) + { + sb.AppendLine($" Exit Code: {cmd.ExitCode}"); + } + + if (!string.IsNullOrEmpty(cmd.StandardOutput)) + { + sb.AppendLine($" Output: {cmd.StandardOutput.Substring(0, Math.Min(100, cmd.StandardOutput.Length))}..."); + } + } + + sb.AppendLine(); + } + + if (this.FileOperations.Any()) + { + sb.AppendLine("File Operations:"); + foreach (var op in this.FileOperations.GroupBy(f => f.Operation)) + { + sb.AppendLine($" {op.Key}: {op.Count()} operations"); + } + + sb.AppendLine(); + } + + if (this.PackageOperations.Any()) + { + sb.AppendLine("Package Operations:"); + foreach (var op in this.PackageOperations) + { + sb.AppendLine($" [{op.OccurredAt:HH:mm:ss.fff}] {op.Operation}: {op.PackageName}"); + } + } + + return sb.ToString(); + } + } + + /// + /// Detailed information about a command execution. + /// + public class CommandExecutionInfo + { + /// + /// The full command including arguments. + /// + public string FullCommand { get; set; } + + /// + /// The executable/command name. + /// + public string FileName { get; set; } + + /// + /// The command line arguments. + /// + public string Arguments { get; set; } + + /// + /// The working directory where the command was executed. + /// + public string WorkingDirectory { get; set; } + + /// + /// The timestamp when the command was executed. + /// + public DateTime ExecutedAt { get; set; } + + /// + /// The exit code of the process (if available). + /// + public int? ExitCode { get; set; } + + /// + /// The standard output from the process. + /// + public string StandardOutput { get; set; } + + /// + /// The standard error from the process. + /// + public string StandardError { get; set; } + + /// + /// The process ID. + /// + public int ProcessId { get; set; } + + /// + /// Returns a string representation of the command. + /// + public override string ToString() => this.FullCommand; + } + + /// + /// Information about a file operation. + /// + public class FileOperationInfo + { + /// + /// The type of operation (Read, Write, Delete, Exists, etc.). + /// + public string Operation { get; set; } + + /// + /// The file path involved in the operation. + /// + public string FilePath { get; set; } + + /// + /// The timestamp when the operation occurred. + /// + public DateTime OccurredAt { get; set; } + } + + /// + /// Information about a package operation. + /// + public class PackageOperationInfo + { + /// + /// The type of operation (Install, GetPackage, Register, etc.). + /// + public string Operation { get; set; } + + /// + /// The name of the package. + /// + public string PackageName { get; set; } + + /// + /// The path to the package. + /// + public string PackagePath { get; set; } + + /// + /// The timestamp when the operation occurred. + /// + public DateTime OccurredAt { get; set; } + } +} diff --git a/src/VirtualClient/VirtualClient.TestFramework/MockFixture.cs b/src/VirtualClient/VirtualClient.TestFramework/MockFixture.cs index 0aadadf5ab..bca7faa31d 100644 --- a/src/VirtualClient/VirtualClient.TestFramework/MockFixture.cs +++ b/src/VirtualClient/VirtualClient.TestFramework/MockFixture.cs @@ -11,6 +11,7 @@ namespace VirtualClient using System.Net.Http; using System.Reflection; using System.Runtime.InteropServices; + using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -70,9 +71,15 @@ static MockFixture() public MockFixture() { this.experimentId = Guid.NewGuid().ToString(); + this.Tracking = new FixtureTracking(); this.Setup(Environment.OSVersion.Platform, Architecture.X64, useUnixStylePathsOnly: false); } + /// + /// Centralized tracking for all mock interactions during testing. + /// + public FixtureTracking Tracking { get; private set; } + /// /// A platform specific instance for the current OS and CPU architecture on which the /// testing operations are running. @@ -625,6 +632,164 @@ public virtual MockFixture Setup(PlatformID platform, Architecture architecture this.PackagesBlobManager.Object }); + // Clear tracking on setup + this.Tracking.Clear(); + + return this; + } + + /// + /// Enables automatic process tracking for test verification. + /// This method sets up tracking to automatically capture all processes created during test execution. + /// + /// If true, clears existing tracked processes before enabling tracking. + /// If true, captures stdout/stderr for each process. + /// The fixture instance for method chaining. + public MockFixture TrackProcesses(bool reset = true, bool captureOutput = true) + { + if (reset) + { + this.Tracking.Processes.Clear(); + this.Tracking.Commands.Clear(); + } + + // Wrap the ProcessManager's OnCreateProcess to add automatic tracking + var existingHandler = this.ProcessManager.OnCreateProcess; + + this.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + // Call the existing handler if it exists, otherwise use default process + InMemoryProcess process; + + if (existingHandler != null) + { + process = existingHandler(command, arguments, workingDir) as InMemoryProcess; + } + else + { + process = this.Process; + } + + // Ensure we have a valid process + if (process == null) + { + process = new InMemoryProcess + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = command, + Arguments = arguments, + WorkingDirectory = workingDir + }, + OnHasExited = () => true, + OnStart = () => true, + ExitCode = 0 + }; + } + + // Automatically track the process + this.Tracking.Processes.Add(process); + + // Track detailed command info + var commandInfo = new CommandExecutionInfo + { + FullCommand = $"{command} {arguments}".Trim(), + FileName = command, + Arguments = arguments, + WorkingDirectory = workingDir, + ExecutedAt = DateTime.UtcNow, + ProcessId = process.Id + }; + + this.Tracking.Commands.Add(commandInfo); + + // Optionally capture output when process completes + if (captureOutput) + { + var originalOnHasExited = process.OnHasExited; + process.OnHasExited = () => + { + bool result = originalOnHasExited(); + commandInfo.ExitCode = process.ExitCode; + commandInfo.StandardOutput = process.StandardOutput?.ToString(); + commandInfo.StandardError = process.StandardError?.ToString(); + return result; + }; + } + + return process; + }; + + return this; + } + + /// + /// Sets up process output injection for specific command patterns. + /// This allows you to inject stdout/stderr into processes that match a pattern without manually checking in OnCreateProcess. + /// + /// Regex pattern to match commands (e.g., "redis-server.*--version") + /// Output to inject into matching processes + /// Error output to inject (optional) + /// Exit code for matching processes (default: 0) + /// The fixture instance for method chaining. + public MockFixture SetupProcessOutput(string commandPattern, string standardOutput, string standardError = null, int exitCode = 0) + { + // Store the existing handler + var existingHandler = this.ProcessManager.OnCreateProcess; + + this.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + string fullCommand = $"{command} {arguments}".Trim(); + + // Check if this command matches the pattern + bool matches = false; + try + { + matches = Regex.IsMatch(fullCommand, commandPattern, RegexOptions.IgnoreCase); + } + catch + { + matches = fullCommand.Contains(commandPattern, StringComparison.OrdinalIgnoreCase); + } + + // If it matches, set up the output + if (matches) + { + var process = new InMemoryProcess + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = command, + Arguments = arguments, + WorkingDirectory = workingDir + }, + OnHasExited = () => true, + OnStart = () => true, + ExitCode = exitCode + }; + + if (!string.IsNullOrEmpty(standardOutput)) + { + process.StandardOutput = new ConcurrentBuffer(new StringBuilder(standardOutput)); + } + + if (!string.IsNullOrEmpty(standardError)) + { + process.StandardError = new ConcurrentBuffer(new StringBuilder(standardError)); + } + + return process; + } + + // Otherwise, call the existing handler or return default process + if (existingHandler != null) + { + return existingHandler(command, arguments, workingDir); + } + + return this.Process; + }; + return this; }