Support e.g. --workers=2x to use two workers per core.

This is mostly useful for tests performing a lot of I/O and sleeping,
when you don't know on which architecture they end up running.

The syntax can also be used to reduce CPU load (e.g. --workers=0.5x).

Bug: webrtc:9717
Change-Id: I26b4552576b1dd56a69c2223da39f4bb1115bbf6
Reviewed-on: https://webrtc-review.googlesource.com/101643
Commit-Queue: Yves Gerey <yvesg@webrtc.org>
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24830}

tools_webrtc/gtest-parallel-wrapper.py

@@ -15,8 +15,9 @@ gtest-parallel, renaming options and translating environment variables into
 flags. Developers should execute gtest-parallel directly.
 
 In particular, this translates the GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS
-environment variables to the --shard_index and --shard_count flags, and renames
+environment variables to the --shard_index and --shard_count flags, renames
-the --isolated-script-test-output flag to --dump_json_test_results.
+the --isolated-script-test-output flag to --dump_json_test_results,
+and interprets e.g. --workers=2x as 2 workers per core.
 
 Flags before '--' will be attempted to be understood as arguments to
 gtest-parallel. If gtest-parallel doesn't recognize the flag or the flag is
@@ -63,6 +64,7 @@ Will be converted into:
 
 import argparse
 import collections
+import multiprocessing
 import os
 import shutil
 import subprocess
@@ -81,6 +83,15 @@ def _CatFiles(file_list, output_file):
         output_file.write(input_file.read())
       os.remove(filename)
 
+def _ParseWorkersOption(workers):
+  """Interpret Nx syntax as N * cpu_count. Int value is left as is."""
+  base = float(workers.rstrip('x'))
+  if workers.endswith('x'):
+    result = int(base * multiprocessing.cpu_count())
+  else:
+    result = int(base)
+  return max(result, 1)  # Sanitize when using e.g. '0.5x'.
+
 
 class ReconstructibleArgumentGroup(object):
   """An argument group that can be converted back into a command line.
@@ -117,13 +128,15 @@ def ParseArgs(argv=None):
   gtest_group.AddArgument('-d', '--output_dir')
   gtest_group.AddArgument('-r', '--repeat')
   gtest_group.AddArgument('--retry_failed')
-  gtest_group.AddArgument('-w', '--workers')
   gtest_group.AddArgument('--gtest_color')
   gtest_group.AddArgument('--gtest_filter')
   gtest_group.AddArgument('--gtest_also_run_disabled_tests',
                           action='store_true', default=None)
   gtest_group.AddArgument('--timeout')
 
+  # Syntax 'Nx' will be interpreted as N * number of cpu cores.
+  gtest_group.AddArgument('-w', '--workers', type=_ParseWorkersOption)
+
   # --isolated-script-test-output is used to upload results to the flakiness
   # dashboard. This translation is made because gtest-parallel expects the flag
   # to be called --dump_json_test_results instead.

tools_webrtc/gtest_parallel_wrapper_test.py

@@ -10,11 +10,13 @@
 
 from contextlib import contextmanager
 
+import multiprocessing
 import os
 import tempfile
 import unittest
 
-script = __import__('gtest-parallel-wrapper')  # pylint: disable=invalid-name
+# pylint: disable=invalid-name
+gtest_parallel_wrapper = __import__('gtest-parallel-wrapper')
 
 
 @contextmanager
@@ -24,105 +26,151 @@ def TemporaryDirectory():
   os.rmdir(tmp_dir)
 
 
+class GtestParallelWrapperHelpersTest(unittest.TestCase):
+
+  def testGetWorkersAsIs(self):
+    # pylint: disable=protected-access
+    self.assertEqual(gtest_parallel_wrapper._ParseWorkersOption('12'), 12)
+
+  def testGetTwiceWorkers(self):
+    expected = 2 * multiprocessing.cpu_count()
+    # pylint: disable=protected-access
+    self.assertEqual(gtest_parallel_wrapper._ParseWorkersOption('2x'), expected)
+
+  def testGetHalfWorkers(self):
+    expected = max(multiprocessing.cpu_count() // 2, 1)
+    # pylint: disable=protected-access
+    self.assertEqual(
+        gtest_parallel_wrapper._ParseWorkersOption('0.5x'), expected)
+
+
 class GtestParallelWrapperTest(unittest.TestCase):
+
   @classmethod
   def _Expected(cls, gtest_parallel_args):
     return ['--shard_index=0', '--shard_count=1'] + gtest_parallel_args
 
   def testOverwrite(self):
-    result = script.ParseArgs(['--timeout=123', 'exec', '--timeout', '124'])
+    result = gtest_parallel_wrapper.ParseArgs(
+        ['--timeout=123', 'exec', '--timeout', '124'])
     expected = self._Expected(['--timeout=124', 'exec'])
     self.assertEqual(result.gtest_parallel_args, expected)
 
   def testMixing(self):
-    result = script.ParseArgs(
+    result = gtest_parallel_wrapper.ParseArgs(
         ['--timeout=123', '--param1', 'exec', '--param2', '--timeout', '124'])
     expected = self._Expected(
         ['--timeout=124', 'exec', '--', '--param1', '--param2'])
     self.assertEqual(result.gtest_parallel_args, expected)
 
   def testMixingPositional(self):
-    result = script.ParseArgs(['--timeout=123', 'exec', '--foo1', 'bar1',
+    result = gtest_parallel_wrapper.ParseArgs([
-                               '--timeout', '124', '--foo2', 'bar2'])
+        '--timeout=123', 'exec', '--foo1', 'bar1', '--timeout', '124', '--foo2',
-    expected = self._Expected(['--timeout=124', 'exec', '--', '--foo1', 'bar1',
+        'bar2'
-                               '--foo2', 'bar2'])
+    ])
+    expected = self._Expected(
+        ['--timeout=124', 'exec', '--', '--foo1', 'bar1', '--foo2', 'bar2'])
     self.assertEqual(result.gtest_parallel_args, expected)
 
   def testDoubleDash1(self):
-    result = script.ParseArgs(
+    result = gtest_parallel_wrapper.ParseArgs(
         ['--timeout', '123', 'exec', '--', '--timeout', '124'])
     expected = self._Expected(
         ['--timeout=123', 'exec', '--', '--timeout', '124'])
     self.assertEqual(result.gtest_parallel_args, expected)
 
   def testDoubleDash2(self):
-    result = script.ParseArgs(['--timeout=123', '--', 'exec', '--timeout=124'])
+    result = gtest_parallel_wrapper.ParseArgs(
+        ['--timeout=123', '--', 'exec', '--timeout=124'])
     expected = self._Expected(['--timeout=123', 'exec', '--', '--timeout=124'])
     self.assertEqual(result.gtest_parallel_args, expected)
 
   def testArtifacts(self):
     with TemporaryDirectory() as tmp_dir:
       output_dir = os.path.join(tmp_dir, 'foo')
-      result = script.ParseArgs(['exec', '--store-test-artifacts',
+      result = gtest_parallel_wrapper.ParseArgs(
-                                 '--output_dir', output_dir])
+          ['exec', '--store-test-artifacts', '--output_dir', output_dir])
       exp_artifacts_dir = os.path.join(output_dir, 'test_artifacts')
-      exp = self._Expected(['--output_dir=' + output_dir, 'exec', '--',
+      exp = self._Expected([
-                            '--test_artifacts_dir=' + exp_artifacts_dir])
+          '--output_dir=' + output_dir, 'exec', '--',
+          '--test_artifacts_dir=' + exp_artifacts_dir
+      ])
       self.assertEqual(result.gtest_parallel_args, exp)
       self.assertEqual(result.output_dir, output_dir)
       self.assertEqual(result.test_artifacts_dir, exp_artifacts_dir)
 
   def testNoDirsSpecified(self):
-    result = script.ParseArgs(['exec'])
+    result = gtest_parallel_wrapper.ParseArgs(['exec'])
     self.assertEqual(result.output_dir, None)
     self.assertEqual(result.test_artifacts_dir, None)
 
   def testOutputDirSpecified(self):
-    result = script.ParseArgs(['exec', '--output_dir', '/tmp/foo'])
+    result = gtest_parallel_wrapper.ParseArgs(
+        ['exec', '--output_dir', '/tmp/foo'])
     self.assertEqual(result.output_dir, '/tmp/foo')
     self.assertEqual(result.test_artifacts_dir, None)
 
   def testJsonTestResults(self):
-    result = script.ParseArgs(['--isolated-script-test-output', '/tmp/foo',
+    result = gtest_parallel_wrapper.ParseArgs(
-                               'exec'])
+        ['--isolated-script-test-output', '/tmp/foo', 'exec'])
     expected = self._Expected(['--dump_json_test_results=/tmp/foo', 'exec'])
     self.assertEqual(result.gtest_parallel_args, expected)
 
   def testShortArg(self):
-    result = script.ParseArgs(['-d', '/tmp/foo', 'exec'])
+    result = gtest_parallel_wrapper.ParseArgs(['-d', '/tmp/foo', 'exec'])
     expected = self._Expected(['--output_dir=/tmp/foo', 'exec'])
     self.assertEqual(result.gtest_parallel_args, expected)
     self.assertEqual(result.output_dir, '/tmp/foo')
 
   def testBoolArg(self):
-    result = script.ParseArgs(['--gtest_also_run_disabled_tests', 'exec'])
+    result = gtest_parallel_wrapper.ParseArgs(
+        ['--gtest_also_run_disabled_tests', 'exec'])
     expected = self._Expected(['--gtest_also_run_disabled_tests', 'exec'])
     self.assertEqual(result.gtest_parallel_args, expected)
 
   def testNoArgs(self):
-    result = script.ParseArgs(['exec'])
+    result = gtest_parallel_wrapper.ParseArgs(['exec'])
     expected = self._Expected(['exec'])
     self.assertEqual(result.gtest_parallel_args, expected)
 
   def testDocExample(self):
     with TemporaryDirectory() as tmp_dir:
       output_dir = os.path.join(tmp_dir, 'foo')
-      result = script.ParseArgs([
+      result = gtest_parallel_wrapper.ParseArgs([
           'some_test', '--some_flag=some_value', '--another_flag',
           '--output_dir=' + output_dir, '--store-test-artifacts',
           '--isolated-script-test-output=SOME_DIR',
-          '--isolated-script-test-perf-output=SOME_OTHER_DIR',
+          '--isolated-script-test-perf-output=SOME_OTHER_DIR', '--foo=bar',
-          '--foo=bar', '--baz'])
+          '--baz'
+      ])
       expected_artifacts_dir = os.path.join(output_dir, 'test_artifacts')
       expected = self._Expected([
           '--output_dir=' + output_dir, '--dump_json_test_results=SOME_DIR',
-          'some_test', '--',
+          'some_test', '--', '--test_artifacts_dir=' + expected_artifacts_dir,
-          '--test_artifacts_dir=' + expected_artifacts_dir,
           '--some_flag=some_value', '--another_flag',
-          '--isolated-script-test-perf-output=SOME_OTHER_DIR',
+          '--isolated-script-test-perf-output=SOME_OTHER_DIR', '--foo=bar',
-          '--foo=bar', '--baz'])
+          '--baz'
+      ])
       self.assertEqual(result.gtest_parallel_args, expected)
 
+  def testStandardWorkers(self):
+    """Check integer value is passed as-is."""
+    result = gtest_parallel_wrapper.ParseArgs(['--workers', '17', 'exec'])
+    expected = self._Expected(['--workers=17', 'exec'])
+    self.assertEqual(result.gtest_parallel_args, expected)
+
+  def testTwoWorkersPerCpuCore(self):
+    result = gtest_parallel_wrapper.ParseArgs(['--workers', '2x', 'exec'])
+    workers = 2 * multiprocessing.cpu_count()
+    expected = self._Expected(['--workers=%s' % workers, 'exec'])
+    self.assertEqual(result.gtest_parallel_args, expected)
+
+  def testUseHalfTheCpuCores(self):
+    result = gtest_parallel_wrapper.ParseArgs(['--workers', '0.5x', 'exec'])
+    workers = max(multiprocessing.cpu_count() // 2, 1)
+    expected = self._Expected(['--workers=%s' % workers, 'exec'])
+    self.assertEqual(result.gtest_parallel_args, expected)
+
 
 if __name__ == '__main__':
   unittest.main()