Update isolate.gypi to support Swarming + move .isolate files

This updates the isolate.gypi copies we have to maintain in our
code repo to Chromium's revision 310ea93.
The changes about generating .isolated.gen.json files are needed
to support running with Swarming (https://www.chromium.org/developers/testing/isolated-testing)

Since isolated testing is now using a new launch script
in tools: isolate_driver.py, that's added to our links
script.

In order to use isolate_driver.py, the .isolate files must be in the
same directory as the test_name_run target is defined, which meant
I had to move around some of the isolate files and targets below
webrtc/modules.

BUG=497757
R=maruel@chromium.org
TBR=henrik.lundin@webrtc.org, mflodman@webrtc.org, niklas.enbom@webrtc.org
TESTED=Clobbered trybots:
git cl try -c --bot=linux_compile_rel --bot=mac_compile_rel --bot=win_compile_rel --bot=android_compile_rel --bot=ios_rel -m tryserver.webrtc

Review URL: https://codereview.webrtc.org/1373513002 .

Cr-Commit-Position: refs/heads/master@{#10081}

.gitignore

@@ -144,6 +144,7 @@
 /tools/gn
 /tools/grit
 /tools/gyp
+/tools/isolate_driver.py
 /tools/memory
 /tools/protoc_wrapper
 /tools/python

PRESUBMIT.py

@@ -273,6 +273,7 @@ def _CommonChecks(input_api, output_api):
                   r'^tools[\\\/]generate_library_loader[\\\/].*\.py$',
                   r'^tools[\\\/]gn[\\\/].*\.py$',
                   r'^tools[\\\/]gyp[\\\/].*\.py$',
+                  r'^tools[\\\/]isolate_driver.py$',
                   r'^tools[\\\/]protoc_wrapper[\\\/].*\.py$',
                   r'^tools[\\\/]python[\\\/].*\.py$',
                   r'^tools[\\\/]python_charts[\\\/]data[\\\/].*\.py$',

setup_links.py

@@ -105,6 +105,7 @@ if 'ios' in target_os:
 
 FILES = {
   'tools/find_depot_tools.py': None,
+  'tools/isolate_driver.py': None,
   'third_party/BUILD.gn': None,
 }
 

talk/build/isolate.gypi

@@ -23,18 +23,17 @@
 # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
 
 # Copied from Chromium's src/build/isolate.gypi
 #
-# It was necessary to copy this file to libjingle , because the path to
+# It was necessary to copy this file because the path to build/common.gypi is
-# build/common.gypi is different for the standalone and Chromium builds. Gyp
+# different for the standalone and Chromium builds. Gyp doesn't permit
-# doesn't permit conditional inclusion or variable expansion in include paths.
+# conditional inclusion or variable expansion in include paths.
 # http://code.google.com/p/gyp/wiki/InputFormatReference#Including_Other_Files
 #
 # Local modifications:
 # * Removed include of '../chrome/version.gypi'.
-# * Removal passing of version_full variable created in version.gypi:
+# * Removed passing of version_full variable created in version.gypi:
 #   '--extra-variable', 'version_full=<(version_full)',
 
 # This file is meant to be included into a target to provide a rule
@@ -79,48 +78,81 @@
       'extension': 'isolate',
       'inputs': [
         # Files that are known to be involved in this step.
+        '<(DEPTH)/tools/isolate_driver.py',
         '<(DEPTH)/tools/swarming_client/isolate.py',
         '<(DEPTH)/tools/swarming_client/run_isolated.py',
       ],
-      'outputs': [
+      'outputs': [],
-        '<(PRODUCT_DIR)/<(RULE_INPUT_ROOT).isolated',
-      ],
       'action': [
         'python',
-        '<(DEPTH)/tools/swarming_client/isolate.py',
+        '<(DEPTH)/tools/isolate_driver.py',
         '<(test_isolation_mode)',
-        '--result', '<@(_outputs)',
+        '--isolated', '<(PRODUCT_DIR)/<(RULE_INPUT_ROOT).isolated',
         '--isolate', '<(RULE_INPUT_PATH)',
 
         # Variables should use the -V FOO=<(FOO) form so frequent values,
         # like '0' or '1', aren't stripped out by GYP. Run 'isolate.py help' for
         # more details.
-        #
-        # This list needs to be kept in sync with the cmd line options
-        # in src/build/android/pylib/gtest/setup.py.
 
         # Path variables are used to replace file paths when loading a .isolate
         # file
         '--path-variable', 'DEPTH', '<(DEPTH)',
         '--path-variable', 'PRODUCT_DIR', '<(PRODUCT_DIR) ',
 
+        # Note: This list must match DefaultConfigVariables()
+        # in build/android/pylib/utils/isolator.py
+        '--config-variable', 'CONFIGURATION_NAME=<(CONFIGURATION_NAME)',
         '--config-variable', 'OS=<(OS)',
+        '--config-variable', 'asan=<(asan)',
+        '--config-variable', 'branding=<(branding)',
         '--config-variable', 'chromeos=<(chromeos)',
         '--config-variable', 'component=<(component)',
+        '--config-variable', 'disable_nacl=<(disable_nacl)',
+        '--config-variable', 'enable_pepper_cdms=<(enable_pepper_cdms)',
+        '--config-variable', 'enable_plugins=<(enable_plugins)',
+        '--config-variable', 'fastbuild=<(fastbuild)',
+        '--config-variable', 'icu_use_data_file_flag=<(icu_use_data_file_flag)',
         # TODO(kbr): move this to chrome_tests.gypi:gles2_conform_tests_run
         # once support for user-defined config variables is added.
         '--config-variable',
           'internal_gles2_conform_tests=<(internal_gles2_conform_tests)',
-        '--config-variable', 'icu_use_data_file_flag=<(icu_use_data_file_flag)',
+        '--config-variable', 'kasko=<(kasko)',
+        '--config-variable', 'lsan=<(lsan)',
+        '--config-variable', 'msan=<(msan)',
+        '--config-variable', 'target_arch=<(target_arch)',
+        '--config-variable', 'tsan=<(tsan)',
+        '--config-variable', 'use_custom_libcxx=<(use_custom_libcxx)',
+        '--config-variable', 'use_instrumented_libraries=<(use_instrumented_libraries)',
+        '--config-variable',
+        'use_prebuilt_instrumented_libraries=<(use_prebuilt_instrumented_libraries)',
         '--config-variable', 'use_openssl=<(use_openssl)',
+        '--config-variable', 'use_ozone=<(use_ozone)',
+        '--config-variable', 'use_x11=<(use_x11)',
+        '--config-variable', 'v8_use_external_startup_data=<(v8_use_external_startup_data)',
       ],
       'conditions': [
         # Note: When gyp merges lists, it appends them to the old value.
         ['OS=="mac"', {
-          # <(mac_product_name) can contain a space, so don't use FOO=<(FOO)
-          # form.
           'action': [
-            '--extra-variable', 'mac_product_name', '<(mac_product_name)',
+            '--extra-variable', 'mac_product_name=<(mac_product_name)',
+          ],
+        }],
+        ["test_isolation_mode == 'prepare'", {
+          'outputs': [
+            '<(PRODUCT_DIR)/<(RULE_INPUT_ROOT).isolated.gen.json',
+          ],
+        }, {
+          'outputs': [
+            '<(PRODUCT_DIR)/<(RULE_INPUT_ROOT).isolated',
+          ],
+        }],
+        ['OS=="win"', {
+          'action': [
+            '--config-variable', 'msvs_version=<(MSVS_VERSION)',
+          ],
+        }, {
+          'action': [
+            '--config-variable', 'msvs_version=0',
           ],
         }],
       ],

webrtc/build/android/test_runner.py

@@ -33,7 +33,7 @@ def main():
   # pylint: disable=protected-access
   gtest_test_instance._DEFAULT_ISOLATE_FILE_PATHS.update({
     'audio_decoder_unittests':
-        'webrtc/modules/audio_coding/neteq/audio_decoder_unittests.isolate',
+        'webrtc/modules/audio_decoder_unittests.isolate',
     'common_audio_unittests':
         'webrtc/common_audio/common_audio_unittests.isolate',
     'common_video_unittests':

webrtc/build/isolate.gypi

@@ -8,14 +8,14 @@
 
 # Copied from Chromium's src/build/isolate.gypi
 #
-# It was necessary to copy this file to WebRTC, because the path to
+# It was necessary to copy this file because the path to build/common.gypi is
-# build/common.gypi is different for the standalone and Chromium builds. Gyp
+# different for the standalone and Chromium builds. Gyp doesn't permit
-# doesn't permit conditional inclusion or variable expansion in include paths.
+# conditional inclusion or variable expansion in include paths.
 # http://code.google.com/p/gyp/wiki/InputFormatReference#Including_Other_Files
 #
 # Local modifications:
 # * Removed include of '../chrome/version.gypi'.
-# * Removal passing of version_full variable created in version.gypi:
+# * Removed passing of version_full variable created in version.gypi:
 #   '--extra-variable', 'version_full=<(version_full)',
 
 # This file is meant to be included into a target to provide a rule
@@ -60,48 +60,81 @@
       'extension': 'isolate',
       'inputs': [
         # Files that are known to be involved in this step.
+        '<(DEPTH)/tools/isolate_driver.py',
         '<(DEPTH)/tools/swarming_client/isolate.py',
         '<(DEPTH)/tools/swarming_client/run_isolated.py',
       ],
-      'outputs': [
+      'outputs': [],
-        '<(PRODUCT_DIR)/<(RULE_INPUT_ROOT).isolated',
-      ],
       'action': [
         'python',
-        '<(DEPTH)/tools/swarming_client/isolate.py',
+        '<(DEPTH)/tools/isolate_driver.py',
         '<(test_isolation_mode)',
-        '--result', '<@(_outputs)',
+        '--isolated', '<(PRODUCT_DIR)/<(RULE_INPUT_ROOT).isolated',
         '--isolate', '<(RULE_INPUT_PATH)',
 
         # Variables should use the -V FOO=<(FOO) form so frequent values,
         # like '0' or '1', aren't stripped out by GYP. Run 'isolate.py help' for
         # more details.
-        #
-        # This list needs to be kept in sync with the cmd line options
-        # in src/build/android/pylib/gtest/setup.py.
 
         # Path variables are used to replace file paths when loading a .isolate
         # file
         '--path-variable', 'DEPTH', '<(DEPTH)',
         '--path-variable', 'PRODUCT_DIR', '<(PRODUCT_DIR) ',
 
+        # Note: This list must match DefaultConfigVariables()
+        # in build/android/pylib/utils/isolator.py
+        '--config-variable', 'CONFIGURATION_NAME=<(CONFIGURATION_NAME)',
         '--config-variable', 'OS=<(OS)',
+        '--config-variable', 'asan=<(asan)',
+        '--config-variable', 'branding=<(branding)',
         '--config-variable', 'chromeos=<(chromeos)',
         '--config-variable', 'component=<(component)',
+        '--config-variable', 'disable_nacl=<(disable_nacl)',
+        '--config-variable', 'enable_pepper_cdms=<(enable_pepper_cdms)',
+        '--config-variable', 'enable_plugins=<(enable_plugins)',
+        '--config-variable', 'fastbuild=<(fastbuild)',
+        '--config-variable', 'icu_use_data_file_flag=<(icu_use_data_file_flag)',
         # TODO(kbr): move this to chrome_tests.gypi:gles2_conform_tests_run
         # once support for user-defined config variables is added.
         '--config-variable',
           'internal_gles2_conform_tests=<(internal_gles2_conform_tests)',
-        '--config-variable', 'icu_use_data_file_flag=<(icu_use_data_file_flag)',
+        '--config-variable', 'kasko=<(kasko)',
+        '--config-variable', 'lsan=<(lsan)',
+        '--config-variable', 'msan=<(msan)',
+        '--config-variable', 'target_arch=<(target_arch)',
+        '--config-variable', 'tsan=<(tsan)',
+        '--config-variable', 'use_custom_libcxx=<(use_custom_libcxx)',
+        '--config-variable', 'use_instrumented_libraries=<(use_instrumented_libraries)',
+        '--config-variable',
+        'use_prebuilt_instrumented_libraries=<(use_prebuilt_instrumented_libraries)',
         '--config-variable', 'use_openssl=<(use_openssl)',
+        '--config-variable', 'use_ozone=<(use_ozone)',
+        '--config-variable', 'use_x11=<(use_x11)',
+        '--config-variable', 'v8_use_external_startup_data=<(v8_use_external_startup_data)',
       ],
       'conditions': [
         # Note: When gyp merges lists, it appends them to the old value.
         ['OS=="mac"', {
-          # <(mac_product_name) can contain a space, so don't use FOO=<(FOO)
-          # form.
           'action': [
-            '--extra-variable', 'mac_product_name', '<(mac_product_name)',
+            '--extra-variable', 'mac_product_name=<(mac_product_name)',
+          ],
+        }],
+        ["test_isolation_mode == 'prepare'", {
+          'outputs': [
+            '<(PRODUCT_DIR)/<(RULE_INPUT_ROOT).isolated.gen.json',
+          ],
+        }, {
+          'outputs': [
+            '<(PRODUCT_DIR)/<(RULE_INPUT_ROOT).isolated',
+          ],
+        }],
+        ['OS=="win"', {
+          'action': [
+            '--config-variable', 'msvs_version=<(MSVS_VERSION)',
+          ],
+        }, {
+          'action': [
+            '--config-variable', 'msvs_version=0',
           ],
         }],
       ],

webrtc/modules/audio_coding/audio_coding_tests.gypi

@@ -51,22 +51,5 @@
         },
       ],
     }],
-    ['test_isolation_mode != "noop"', {
-      'targets': [
-        {
-          'target_name': 'audio_codec_speed_tests_run',
-          'type': 'none',
-          'dependencies': [
-            'audio_codec_speed_tests',
-          ],
-          'includes': [
-            '../../build/isolate.gypi',
-          ],
-          'sources': [
-            'audio_codec_speed_tests.isolate',
-          ],
-        },
-      ],
-    }],
   ],
 }

webrtc/modules/audio_coding/neteq/neteq.gypi

@@ -199,23 +199,6 @@
             },
           ],
         }],
-        ['test_isolation_mode != "noop"', {
-          'targets': [
-            {
-              'target_name': 'audio_decoder_unittests_run',
-              'type': 'none',
-              'dependencies': [
-                'audio_decoder_unittests',
-              ],
-              'includes': [
-                '../../../build/isolate.gypi',
-              ],
-              'sources': [
-                'audio_decoder_unittests.isolate',
-              ],
-            },
-          ],
-        }],
       ],
     }], # include_tests
   ], # conditions

webrtc/modules/audio_device/audio_device.gypi

@@ -236,25 +236,6 @@
           ],
         },
       ], # targets
-      'conditions': [
-        ['test_isolation_mode != "noop"', {
-          'targets': [
-            {
-              'target_name': 'audio_device_tests_run',
-              'type': 'none',
-              'dependencies': [
-                'audio_device_tests',
-              ],
-              'includes': [
-                '../../build/isolate.gypi',
-              ],
-              'sources': [
-                'audio_device_tests.isolate',
-              ],
-            },
-          ],
-        }],
-      ],
     }], # include_tests
   ],
 }

webrtc/modules/modules.gyp

@@ -468,6 +468,45 @@
         }],
         ['test_isolation_mode != "noop"', {
           'targets': [
+            {
+              'target_name': 'audio_codec_speed_tests_run',
+              'type': 'none',
+              'dependencies': [
+                'audio_codec_speed_tests',
+              ],
+              'includes': [
+                '../build/isolate.gypi',
+              ],
+              'sources': [
+                'audio_codec_speed_tests.isolate',
+              ],
+            },
+            {
+              'target_name': 'audio_decoder_unittests_run',
+              'type': 'none',
+              'dependencies': [
+                'audio_decoder_unittests',
+              ],
+              'includes': [
+                '../build/isolate.gypi',
+              ],
+              'sources': [
+                'audio_decoder_unittests.isolate',
+              ],
+            },
+            {
+              'target_name': 'audio_device_tests_run',
+              'type': 'none',
+              'dependencies': [
+                'audio_device_tests',
+              ],
+              'includes': [
+                '../build/isolate.gypi',
+              ],
+              'sources': [
+                'audio_device_tests.isolate',
+              ],
+            },
             {
               'target_name': 'modules_tests_run',
               'type': 'none',
@@ -494,6 +533,19 @@
                 'modules_unittests.isolate',
               ],
             },
+            {
+              'target_name': 'video_render_tests_run',
+              'type': 'none',
+              'dependencies': [
+                'video_render_tests',
+              ],
+              'includes': [
+                '../build/isolate.gypi',
+              ],
+              'sources': [
+                'video_render_tests.isolate',
+              ],
+            },
           ],
         }],
       ],

webrtc/modules/video_render/video_render.gypi

@@ -197,25 +197,6 @@
           ] # conditions
         }, # video_render_module_test
       ], # targets
-      'conditions': [
-        ['test_isolation_mode != "noop"', {
-          'targets': [
-            {
-              'target_name': 'video_render_tests_run',
-              'type': 'none',
-              'dependencies': [
-                'video_render_tests',
-              ],
-              'includes': [
-                '../../build/isolate.gypi',
-              ],
-              'sources': [
-                'video_render_tests.isolate',
-              ],
-            },
-          ],
-        }],
-      ],
     }], # include_tests==1
   ], # conditions
 }