zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch WebRTC's MB to RBE-CAS.

Browse Source 
This CL updates WebRTC's MB to support RBE-CAS with the "mb run" command.

Bug: chromium:1166990, webrtc:12072
Change-Id: Id51fbe002714679f59ad46e0eee271358c8e119e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/202029
Reviewed-by: Andrey Logvin <landrey@webrtc.org>
Commit-Queue: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33120}
This commit is contained in:
Mirko Bonadei
2021-01-29 14:34:52 +01:00
committed by Commit Bot
parent b853d72250
commit 989e6e7d22
2 changed files with 149 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
tools_webrtc/mb/mb.py
View File

@ -323,11 +323,14 @@ class MetaBuildWrapper(object):
return ret return ret
if self.args.swarmed: if self.args.swarmed:
return self._RunUnderSwarming(build_dir, target) cmd, _ = self.GetSwarmingCommand(self.args.target[0], vals)
return self._RunUnderSwarming(build_dir, target, cmd)
else: else:
return self._RunLocallyIsolated(build_dir, target) return self._RunLocallyIsolated(build_dir, target)
def _RunUnderSwarming(self, build_dir, target): def _RunUnderSwarming(self, build_dir, target, isolate_cmd):
cas_instance = 'chromium-swarm'
swarming_server = 'chromium-swarm.appspot.com'
# TODO(dpranke): Look up the information for the target in # TODO(dpranke): Look up the information for the target in
# the //testing/buildbot.json file, if possible, so that we # the //testing/buildbot.json file, if possible, so that we
# can determine the isolate target, command line, and additional # can determine the isolate target, command line, and additional
@ -336,7 +339,7 @@ class MetaBuildWrapper(object):
# TODO(dpranke): Also, add support for sharding and merging results. # TODO(dpranke): Also, add support for sharding and merging results.
dimensions = [] dimensions = []
for k, v in self.args.dimensions: for k, v in self.args.dimensions:
dimensions += ['-d', k, v] dimensions += ['-d', '%s=%s' % (k, v)]
archive_json_path = self.ToSrcRelPath( archive_json_path = self.ToSrcRelPath(
'%s/%s.archive.json' % (build_dir, target)) '%s/%s.archive.json' % (build_dir, target))
@ -345,13 +348,29 @@ class MetaBuildWrapper(object):
'archive', 'archive',
'-i', '-i',
self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
'-s', '-cas-instance',
self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)), cas_instance,
'-I', 'isolateserver.appspot.com', '-dump-json',
'-dump-json', archive_json_path, archive_json_path,
] ]
ret, _, _ = self.Run(cmd, force_verbose=False)
# Talking to the isolateserver may fail because we're not logged in.
# We trap the command explicitly and rewrite the error output so that
# the error message is actually correct for a Chromium check out.
self.PrintCmd(cmd, env=None)
ret, out, err = self.Run(cmd, force_verbose=False)
if ret: if ret:
self.Print(' -> returned %d' % ret)
if out:
self.Print(out, end='')
if err:
# The swarming client will return an exit code of 2 (via
# argparse.ArgumentParser.error()) and print a message to indicate
# that auth failed, so we have to parse the message to check.
if (ret == 2 and 'Please login to' in err):
err = err.replace(' auth.py', ' tools/swarming_client/auth.py')
self.Print(err, end='', file=sys.stderr)
return ret return ret
try: try:
@ -361,7 +380,7 @@ class MetaBuildWrapper(object):
'Failed to read JSON file "%s"' % archive_json_path, file=sys.stderr) 'Failed to read JSON file "%s"' % archive_json_path, file=sys.stderr)
return 1 return 1
try: try:
isolated_hash = archive_hashes[target] cas_digest = archive_hashes[target]
except Exception: except Exception:
self.Print( self.Print(
'Cannot find hash for "%s" in "%s", file content: %s' % 'Cannot find hash for "%s" in "%s", file content: %s' %
@ -369,16 +388,44 @@ class MetaBuildWrapper(object):
file=sys.stderr) file=sys.stderr)
return 1 return 1
try:
json_dir = self.TempDir()
json_file = self.PathJoin(json_dir, 'task.json')
cmd = [ cmd = [
self.executable, self.PathJoin('tools', 'luci-go', 'swarming'),
self.PathJoin('tools', 'swarming_client', 'swarming.py'), 'trigger',
'run', '-digest',
'-s', isolated_hash, cas_digest,
'-I', 'isolateserver.appspot.com', '-server',
'-S', 'chromium-swarm.appspot.com', swarming_server,
] + dimensions '-tag=purpose:user-debug-mb',
'-relative-cwd',
self.ToSrcRelPath(build_dir),
'-dump-json',
json_file,
] + dimensions + ['--'] + list(isolate_cmd)
if self.args.extra_args: if self.args.extra_args:
cmd += ['--'] + self.args.extra_args cmd += ['--'] + self.args.extra_args
self.Print('')
ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False)
if ret:
return ret
task_json = self.ReadFile(json_file)
task_id = json.loads(task_json)["tasks"][0]['task_id']
finally:
if json_dir:
self.RemoveDirectory(json_dir)
cmd = [
self.PathJoin('tools', 'luci-go', 'swarming'),
'collect',
'-server',
swarming_server,
'-task-output-stdout=console',
task_id,
]
ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False)
return ret return ret
@ -683,7 +730,7 @@ class MetaBuildWrapper(object):
raise MBErr('did not generate any of %s' % raise MBErr('did not generate any of %s' %
', '.join(runtime_deps_targets)) ', '.join(runtime_deps_targets))
command, extra_files = self.GetIsolateCommand(target, vals) command, extra_files = self.GetSwarmingCommand(target, vals)
runtime_deps = self.ReadFile(runtime_deps_path).splitlines() runtime_deps = self.ReadFile(runtime_deps_path).splitlines()
@ -701,7 +748,7 @@ class MetaBuildWrapper(object):
label = labels[0] label = labels[0]
build_dir = self.args.path[0] build_dir = self.args.path[0]
command, extra_files = self.GetIsolateCommand(target, vals) command, extra_files = self.GetSwarmingCommand(target, vals)
cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps') cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps')
ret, out, _ = self.Call(cmd) ret, out, _ = self.Call(cmd)
@ -824,7 +871,7 @@ class MetaBuildWrapper(object):
gn_args = ('import("%s")\n' % vals['args_file']) + gn_args gn_args = ('import("%s")\n' % vals['args_file']) + gn_args
return gn_args return gn_args
def GetIsolateCommand(self, target, vals): def GetSwarmingCommand(self, target, vals):
isolate_map = self.ReadIsolateMap() isolate_map = self.ReadIsolateMap()
test_type = isolate_map[target]['type'] test_type = isolate_map[target]['type']
@ -1188,6 +1235,10 @@ class MetaBuildWrapper(object):
else: else:
shutil.rmtree(abs_path, ignore_errors=True) shutil.rmtree(abs_path, ignore_errors=True)
def TempDir(self):
# This function largely exists so it can be overriden for testing.
return tempfile.mkdtemp(prefix='mb_')
def TempFile(self, mode='w'): def TempFile(self, mode='w'):
# This function largely exists so it can be overriden for testing. # This function largely exists so it can be overriden for testing.
return tempfile.NamedTemporaryFile(mode=mode, delete=False) return tempfile.NamedTemporaryFile(mode=mode, delete=False)

99
tools_webrtc/mb/mb_unittest.py
View File

@ -13,7 +13,9 @@ import ast
import json import json
import StringIO import StringIO
import os import os
import re
import sys import sys
import tempfile
import unittest import unittest
import mb import mb
@ -32,6 +34,7 @@ class FakeMBW(mb.MetaBuildWrapper):
self.platform = 'win32' self.platform = 'win32'
self.executable = 'c:\\python\\python.exe' self.executable = 'c:\\python\\python.exe'
self.sep = '\\' self.sep = '\\'
self.cwd = 'c:\\fake_src\\out\\Default'
else: else:
self.src_dir = '/fake_src' self.src_dir = '/fake_src'
self.default_config = '/fake_src/tools_webrtc/mb/mb_config.pyl' self.default_config = '/fake_src/tools_webrtc/mb/mb_config.pyl'
@ -39,8 +42,10 @@ class FakeMBW(mb.MetaBuildWrapper):
self.executable = '/usr/bin/python' self.executable = '/usr/bin/python'
self.platform = 'linux2' self.platform = 'linux2'
self.sep = '/' self.sep = '/'
self.cwd = '/fake_src/out/Default'
self.files = {} self.files = {}
self.dirs = set()
self.calls = [] self.calls = []
self.cmds = [] self.cmds = []
self.cross_compile = None self.cross_compile = None
@ -52,21 +57,24 @@ class FakeMBW(mb.MetaBuildWrapper):
return '$HOME/%s' % path return '$HOME/%s' % path
def Exists(self, path): def Exists(self, path):
return self.files.get(path) is not None abs_path = self._AbsPath(path)
return (self.files.get(abs_path) is not None or abs_path in self.dirs)
def MaybeMakeDirectory(self, path): def MaybeMakeDirectory(self, path):
self.files[path] = True abpath = self._AbsPath(path)
self.dirs.add(abpath)
def PathJoin(self, *comps): def PathJoin(self, *comps):
return self.sep.join(comps) return self.sep.join(comps)
def ReadFile(self, path): def ReadFile(self, path):
return self.files[path] return self.files[self._AbsPath(path)]
def WriteFile(self, path, contents, force_verbose=False): def WriteFile(self, path, contents, force_verbose=False):
if self.args.dryrun or self.args.verbose or force_verbose: if self.args.dryrun or self.args.verbose or force_verbose:
self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path)) self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
self.files[path] = contents abpath = self._AbsPath(path)
self.files[abpath] = contents
def Call(self, cmd, env=None, buffer_output=True): def Call(self, cmd, env=None, buffer_output=True):
self.calls.append(cmd) self.calls.append(cmd)
@ -83,18 +91,34 @@ class FakeMBW(mb.MetaBuildWrapper):
else: else:
self.out += sep.join(args) + end self.out += sep.join(args) + end
def TempDir(self):
tmp_dir = os.path.join(tempfile.gettempdir(), 'mb_test')
self.dirs.add(tmp_dir)
return tmp_dir
def TempFile(self, mode='w'): def TempFile(self, mode='w'):
return FakeFile(self.files) return FakeFile(self.files)
def RemoveFile(self, path): def RemoveFile(self, path):
del self.files[path] abpath = self._AbsPath(path)
self.files[abpath] = None
def RemoveDirectory(self, path): def RemoveDirectory(self, path):
self.rmdirs.append(path) abpath = self._AbsPath(path)
files_to_delete = [f for f in self.files if f.startswith(path)] self.rmdirs.append(abpath)
files_to_delete = [f for f in self.files if f.startswith(abpath)]
for f in files_to_delete: for f in files_to_delete:
self.files[f] = None self.files[f] = None
def _AbsPath(self, path):
if not ((self.platform == 'win32' and path.startswith('c:')) or
(self.platform != 'win32' and path.startswith('/'))):
path = self.PathJoin(self.cwd, path)
if self.sep == '\\':
return re.sub(r'\\+', r'\\', path)
else:
return re.sub('/+', '/', path)
class FakeFile(object): class FakeFile(object):
def __init__(self, files): def __init__(self, files):
@ -176,13 +200,20 @@ class UnitTest(unittest.TestCase):
mbw.files[path] = contents mbw.files[path] = contents
return mbw return mbw
def check(self, args, mbw=None, files=None, out=None, err=None, ret=None): def check(self, args, mbw=None, files=None, out=None, err=None, ret=None,
env=None):
if not mbw: if not mbw:
mbw = self.fake_mbw(files) mbw = self.fake_mbw(files)
try:
prev_env = os.environ.copy()
os.environ = env if env else prev_env
actual_ret = mbw.Main(args) actual_ret = mbw.Main(args)
finally:
self.assertEqual(actual_ret, ret) os.environ = prev_env
self.assertEqual(
actual_ret, ret,
"ret: %s, out: %s, err: %s" % (actual_ret, mbw.out, mbw.err))
if out is not None: if out is not None:
self.assertEqual(mbw.out, out) self.assertEqual(mbw.out, out)
if err is not None: if err is not None:
@ -564,8 +595,8 @@ class UnitTest(unittest.TestCase):
def test_gen_windowed_test_launcher_win(self): def test_gen_windowed_test_launcher_win(self):
files = { files = {
'/tmp/swarming_targets': 'unittests\n', 'c:\\fake_src\\out\\Default\\tmp\\swarming_targets': 'unittests\n',
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( 'c:\\fake_src\\testing\\buildbot\\gn_isolate_map.pyl': (
"{'unittests': {" "{'unittests': {"
" 'label': '//somewhere:unittests'," " 'label': '//somewhere:unittests',"
" 'type': 'windowed_test_launcher'," " 'type': 'windowed_test_launcher',"
@ -579,9 +610,10 @@ class UnitTest(unittest.TestCase):
mbw = self.fake_mbw(files=files, win32=True) mbw = self.fake_mbw(files=files, win32=True)
self.check(['gen', self.check(['gen',
'-c', 'debug_goma', '-c', 'debug_goma',
'--swarming-targets-file', '/tmp/swarming_targets', '--swarming-targets-file',
'c:\\fake_src\\out\\Default\\tmp\\swarming_targets',
'--isolate-map-file', '--isolate-map-file',
'/fake_src/testing/buildbot/gn_isolate_map.pyl', 'c:\\fake_src\\testing\\buildbot\\gn_isolate_map.pyl',
'//out/Default'], mbw=mbw, ret=0) '//out/Default'], mbw=mbw, ret=0)
isolate_file = mbw.files['c:\\fake_src\\out\\Default\\unittests.isolate'] isolate_file = mbw.files['c:\\fake_src\\out\\Default\\unittests.isolate']
@ -750,23 +782,40 @@ class UnitTest(unittest.TestCase):
def test_run_swarmed(self): def test_run_swarmed(self):
files = { files = {
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl':
"{'base_unittests': {" ("{'base_unittests': {"
" 'label': '//base:base_unittests'," " 'label': '//base:base_unittests',"
" 'type': 'raw'," " 'type': 'console_test_launcher',"
" 'args': []," "}}\n"),
"}}\n" '/fake_src/out/Default/base_unittests.runtime_deps':
), ("base_unittests\n"),
'/fake_src/out/Default/base_unittests.runtime_deps': ( '/fake_src/out/Default/base_unittests.archive.json':
"base_unittests\n" ("{\"base_unittests\":\"fake_hash\"}"),
), '/fake_src/third_party/depot_tools/cipd_manifest.txt':
'out/Default/base_unittests.archive.json': ( ("# vpython\n"
"{\"base_unittests\":\"fake_hash\"}"), "/some/vpython/pkg git_revision:deadbeef\n"),
} }
task_json = json.dumps({'tasks': [{'task_id': '00000'}]})
collect_json = json.dumps({'00000': {'results': {}}})
mbw = self.fake_mbw(files=files) mbw = self.fake_mbw(files=files)
mbw.files[mbw.PathJoin(mbw.TempDir(), 'task.json')] = task_json
mbw.files[mbw.PathJoin(mbw.TempDir(), 'collect_output.json')] = collect_json
original_impl = mbw.ToSrcRelPath
def to_src_rel_path_stub(path):
if path.endswith('base_unittests.archive.json'):
return 'base_unittests.archive.json'
return original_impl(path)
mbw.ToSrcRelPath = to_src_rel_path_stub
self.check(['run', '-s', '-c', 'debug_goma', '//out/Default', self.check(['run', '-s', '-c', 'debug_goma', '//out/Default',
'base_unittests'], mbw=mbw, ret=0) 'base_unittests'], mbw=mbw, ret=0)
mbw = self.fake_mbw(files=files)
mbw.files[mbw.PathJoin(mbw.TempDir(), 'task.json')] = task_json
mbw.files[mbw.PathJoin(mbw.TempDir(), 'collect_output.json')] = collect_json
mbw.ToSrcRelPath = to_src_rel_path_stub
self.check(['run', '-s', '-c', 'debug_goma', '-d', 'os', 'Win7', self.check(['run', '-s', '-c', 'debug_goma', '-d', 'os', 'Win7',
'//out/Default', 'base_unittests'], mbw=mbw, ret=0) '//out/Default', 'base_unittests'], mbw=mbw, ret=0)