User-defined Benchmarks parameter for speccpu profiles

VERSION

@@ -1 +1 @@
-2.1.51
+2.1.52

src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs

@@ -155,6 +155,78 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithInstallationInWin
 
             Assert.AreEqual(processCount, expectedCommands.Count);
         }
+
+        [Test]
+        public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithSpecificBenchmarksInLinux()
+        {
+            this.SetupLinux();
+            ProcessStartInfo expectedInfo = new ProcessStartInfo();
+
+            this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
+            {
+                { nameof(SpecCpuExecutor.SpecProfile), "intrate" },
+                { nameof(SpecCpuExecutor.Benchmarks), "549.fotonik3d_r" },
+                { nameof(SpecCpuExecutor.PackageName), "speccpu" },
+                { nameof(SpecCpuExecutor.RunPeak), true },
+                { nameof(SpecCpuExecutor.Threads), 8 },
+                { nameof(SpecCpuExecutor.Copies), 4 }
+            };
+
+            int coreCount = Environment.ProcessorCount;
+            List<string> expectedCommands = new List<string>
+            {
+                $"sudo mount -t iso9660 -o ro,exec,loop {this.mockPackage.Path}/speccpu.iso {this.mockFixture.GetPackagePath()}/speccpu_mount",
+                $"sudo ./install.sh -f -d {this.mockPackage.Path}",
+                $"sudo gcc -dumpversion",
+                $"sudo chmod -R ugo=rwx {this.mockPackage.Path}",
+                $"sudo umount {this.mockFixture.GetPackagePath()}/speccpu_mount",
+                $"sudo bash runspeccpu.sh \"--config vc-linux-x64.cfg --iterations 2 --copies 4 --threads 8 --tune all --noreportable 549.fotonik3d_r\""
+            };
+
+            int processCount = 0;
+            this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) =>
+            {
+                Assert.AreEqual(expectedCommands.ElementAt(processCount), $"{exe} {arguments}");
+                processCount++;
+
+                if (exe == "sudo" && arguments == "gcc -dumpversion")
+                {
+                    return new InMemoryProcess
+                    {
+                        StartInfo = new ProcessStartInfo
+                        {
+                            FileName = exe,
+                            Arguments = arguments
+                        },
+                        StandardOutput = new ConcurrentBuffer(new StringBuilder("10")),
+                        ExitCode = 0,
+                        OnStart = () => true,
+                        OnHasExited = () => true
+                    };
+                }
+                else
+                {
+                    return new InMemoryProcess
+                    {
+                        StartInfo = new ProcessStartInfo
+                        {
+                            FileName = exe,
+                            Arguments = arguments
+                        },
+                        ExitCode = 0,
+                        OnStart = () => true,
+                        OnHasExited = () => true
+                    };
+                }
+            };
+
+            using (TestSpecCpuExecutor specCpuExecutor = new TestSpecCpuExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters))
+            {
+                await specCpuExecutor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false);
+            }
+
+            Assert.AreEqual(expectedCommands.Count, processCount);
+        }
 
         [Test]
         public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfilesInLinux()

src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs

@@ -399,14 +399,16 @@ private async Task UploadSpecCpuLogsAsync(CancellationToken cancellationToken)
 
         private string GetCommandLineArguments()
         {
+            List<string> suites = new List<string> { "intrate", "intspeed", "fprate", "fpspeed" };
+
             // runcpu arguments document: https://www.spec.org/cpu2017/Docs/runcpu.html#strict
             string configurationFile = this.GetConfigurationFileName();
 
             string cmd = @$"--config {configurationFile} --iterations {this.Iterations} --copies {this.Copies} --threads {this.Threads} --tune {this.tuning}";
 
             // For linux runs we are doing reportable. For windows since not all benchmarks could be run, it will be noreportable.
             // Iterations has to be either 2 or 3 for reportable runs. https://www.spec.org/cpu2017/Docs/config.html#reportable
-            bool reportable = (this.Platform == PlatformID.Unix) && (this.Iterations == 2 || this.Iterations == 3);
+            bool reportable = (this.Platform == PlatformID.Unix && suites.Contains(this.Benchmarks.ToLower())) && (this.Iterations == 2 || this.Iterations == 3);
             cmd = reportable ? $"{cmd} --reportable" : $"{cmd} --noreportable";
             cmd = $"{cmd} {this.Benchmarks}";
             return cmd;

src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json

@@ -15,6 +15,7 @@
         "RunPeak": false,
         "Threads": "{LogicalCoreCount}",
         "Copies": "{LogicalCoreCount}",
+        "Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"fprate\" : \"503 508 510 519 544 549 554\")}",
         "BaseOptimizingFlags": "-g -O3 -march=native -frecord-gcc-switches",
         "PeakOptimizingFlags": "-g -Ofast -march=native -flto -frecord-gcc-switches"
     },
@@ -25,7 +26,7 @@
                 "Scenario": "ExecuteSPECBenchmark",
                 "Iterations": "$.Parameters.Iterations",
                 "SpecProfile": "fprate",
-                "Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"fprate\" : \"503 508 510 519 544 549 554\")}",
+                "Benchmarks": "$.Parameters.Benchmarks",
                 "PackageName": "speccpu2017",
                 "RunPeak": "$.Parameters.RunPeak",
                 "Threads": "$.Parameters.Threads",

src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPSPEED.json

@@ -15,6 +15,7 @@
         "RunPeak": false,
         "Threads": "{LogicalCoreCount}",
         "Copies": "{LogicalCoreCount}",
+        "Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"fpspeed\" : \"603 619 644 654\")}",
         "BaseOptimizingFlags": "-g -O3 -march=native -frecord-gcc-switches",
         "PeakOptimizingFlags": "-g -Ofast -march=native -flto -frecord-gcc-switches"
     },
@@ -25,7 +26,7 @@
                 "Scenario": "ExecuteSPECBenchmark",
                 "Iterations": "$.Parameters.Iterations",
                 "SpecProfile": "fpspeed",
-                "Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"fpspeed\" : \"603 619 644 654\")}",
+                "Benchmarks": "$.Parameters.Benchmarks",
                 "PackageName": "speccpu2017",
                 "RunPeak": "$.Parameters.RunPeak",
                 "Threads": "$.Parameters.Threads",

src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json

@@ -15,6 +15,7 @@
         "RunPeak": false,
         "Threads": "{LogicalCoreCount}",
         "Copies": "{LogicalCoreCount}",
+        "Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intrate\" : \"505 525 541 548 557\")}",
         "BaseOptimizingFlags": "-g -O3 -march=native -frecord-gcc-switches",
         "PeakOptimizingFlags": "-g -Ofast -march=native -flto -frecord-gcc-switches"
     },
@@ -25,7 +26,7 @@
                 "Scenario": "ExecuteSPECBenchmark",
                 "Iterations": "$.Parameters.Iterations",
                 "SpecProfile": "intrate",
-                "Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intrate\" : \"505 525 541 548 557\")}",
+                "Benchmarks": "$.Parameters.Benchmarks",
                 "PackageName": "speccpu2017",
                 "RunPeak": "$.Parameters.RunPeak",
                 "Threads": "$.Parameters.Threads",

src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTSPEED.json

@@ -15,6 +15,7 @@
         "RunPeak": false,
         "Threads": "{LogicalCoreCount}",
         "Copies": "{LogicalCoreCount}",
+        "Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intspeed\" : \"641\")}",
         "BaseOptimizingFlags": "-g -O3 -march=native -frecord-gcc-switches",
         "PeakOptimizingFlags": "-g -Ofast -march=native -flto -frecord-gcc-switches"
     },
@@ -25,7 +26,7 @@
                 "Scenario": "ExecuteSPECBenchmark",
                 "Iterations": "$.Parameters.Iterations",
                 "SpecProfile": "intspeed",
-                "Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intspeed\" : \"641\")}",
+                "Benchmarks": "$.Parameters.Benchmarks",
                 "PackageName": "speccpu2017",
                 "RunPeak": "$.Parameters.RunPeak",
                 "Threads": "$.Parameters.Threads",