diff --git a/tools_webrtc/mb/mb.py b/tools_webrtc/mb/mb.py index 8a44a9bcc0..846653ff44 100755 --- a/tools_webrtc/mb/mb.py +++ b/tools_webrtc/mb/mb.py @@ -14,885 +14,31 @@ MB is a wrapper script for GN that can be used to generate build files for sets of canned configurations and analyze them. """ -import argparse -import ast -import errno -import json import os -import pipes -import pprint -import re -import shutil import sys -import subprocess -import tempfile -import traceback -from urllib.request import urlopen -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -SRC_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR)) -sys.path = [os.path.join(SRC_DIR, 'build')] + sys.path +_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +_SRC_DIR = os.path.dirname(os.path.dirname(_SCRIPT_DIR)) +sys.path.insert(0, _SRC_DIR) -import gn_helpers +from tools.mb import mb def main(args): - mbw = MetaBuildWrapper() + mbw = WebRTCMetaBuildWrapper() return mbw.Main(args) -class MetaBuildWrapper: +class WebRTCMetaBuildWrapper(mb.MetaBuildWrapper): def __init__(self): - self.src_dir = SRC_DIR - self.default_config = os.path.join(SCRIPT_DIR, 'mb_config.pyl') - self.default_isolate_map = os.path.join(SCRIPT_DIR, 'gn_isolate_map.pyl') - self.executable = sys.executable - self.platform = sys.platform - self.sep = os.sep - self.args = argparse.Namespace() - self.configs = {} - self.builder_groups = {} - self.mixins = {} - self.isolate_exe = 'isolate.exe' if self.platform.startswith( - 'win') else 'isolate' - - def Main(self, args): - self.ParseArgs(args) - try: - ret = self.args.func() - if ret: - self.DumpInputFiles() - return ret - except KeyboardInterrupt: - self.Print('interrupted, exiting') - return 130 - except Exception: - self.DumpInputFiles() - s = traceback.format_exc() - for l in s.splitlines(): - self.Print(l) - return 1 - - def ParseArgs(self, argv): - def AddCommonOptions(subp): - subp.add_argument('-b', - '--builder', - help='builder name to look up config from') - subp.add_argument('-m', - '--builder-group', - help='builder group name to look up config from') - subp.add_argument('-c', '--config', help='configuration to analyze') - subp.add_argument('--phase', - help='optional phase name (used when builders ' - 'do multiple compiles with different ' - 'arguments in a single build)') - subp.add_argument('-f', - '--config-file', - metavar='PATH', - default=self.default_config, - help='path to config file ' - '(default is %(default)s)') - subp.add_argument('-i', - '--isolate-map-file', - metavar='PATH', - default=self.default_isolate_map, - help='path to isolate map file ' - '(default is %(default)s)') - subp.add_argument('-r', - '--realm', - default='webrtc:try', - help='optional LUCI realm to use (for example ' - 'when triggering tasks on Swarming)') - subp.add_argument('-g', '--goma-dir', help='path to goma directory') - subp.add_argument('--android-version-code', - help='Sets GN arg android_default_version_code') - subp.add_argument('--android-version-name', - help='Sets GN arg android_default_version_name') - subp.add_argument('-n', - '--dryrun', - action='store_true', - help='Do a dry run (i.e., do nothing, just ' - 'print the commands that will run)') - subp.add_argument('-v', - '--verbose', - action='store_true', - help='verbose logging') - - parser = argparse.ArgumentParser(prog='mb') - subps = parser.add_subparsers() - - subp = subps.add_parser('analyze', - help='analyze whether changes to a set of ' - 'files will cause a set of binaries ' - 'to be rebuilt.') - AddCommonOptions(subp) - subp.add_argument('path', nargs=1, help='path build was generated into.') - subp.add_argument('input_path', - nargs=1, - help='path to a file containing the input ' - 'arguments as a JSON object.') - subp.add_argument('output_path', - nargs=1, - help='path to a file containing the output ' - 'arguments as a JSON object.') - subp.add_argument('--json-output', help='Write errors to json.output') - subp.set_defaults(func=self.CmdAnalyze) - - subp = subps.add_parser('export', - help='print out the expanded configuration for' - 'each builder as a JSON object') - subp.add_argument('-f', - '--config-file', - metavar='PATH', - default=self.default_config, - help='path to config file (default is %(default)s)') - subp.add_argument('-g', '--goma-dir', help='path to goma directory') - subp.set_defaults(func=self.CmdExport) - - subp = subps.add_parser('gen', help='generate a new set of build files') - AddCommonOptions(subp) - subp.add_argument('--swarming-targets-file', - help='save runtime dependencies for targets listed ' - 'in file.') - subp.add_argument('--json-output', help='Write errors to json.output') - subp.add_argument('path', nargs=1, help='path to generate build into') - subp.set_defaults(func=self.CmdGen) - - subp = subps.add_parser('isolate', - help='generate the .isolate files for a given' - 'binary') - AddCommonOptions(subp) - subp.add_argument('path', nargs=1, help='path build was generated into') - subp.add_argument('target', - nargs=1, - help='ninja target to generate the isolate for') - subp.set_defaults(func=self.CmdIsolate) - - subp = subps.add_parser('lookup', - help='look up the command for a given config ' - 'or builder') - AddCommonOptions(subp) - subp.add_argument('--quiet', - default=False, - action='store_true', - help='Print out just the arguments, do ' - 'not emulate the output of the gen subcommand.') - subp.set_defaults(func=self.CmdLookup) - - subp = subps.add_parser( - 'run', - help='build and run the isolated version of a ' - 'binary', - formatter_class=argparse.RawDescriptionHelpFormatter) - subp.description = ( - 'Build, isolate, and run the given binary with the command line\n' - 'listed in the isolate. You may pass extra arguments after the\n' - 'target; use "--" if the extra arguments need to include switches.' - '\n\n' - 'Examples:\n' - '\n' - ' % tools/mb/mb.py run -m chromium.linux -b "Linux Builder" \\\n' - ' //out/Default content_browsertests\n' - '\n' - ' % tools/mb/mb.py run out/Default content_browsertests\n' - '\n' - ' % tools/mb/mb.py run out/Default content_browsertests -- \\\n' - ' --test-launcher-retry-limit=0' - '\n') - AddCommonOptions(subp) - subp.add_argument('-j', - '--jobs', - dest='jobs', - type=int, - help='Number of jobs to pass to ninja') - subp.add_argument('--no-build', - dest='build', - default=True, - action='store_false', - help='Do not build, just isolate and run') - subp.add_argument('path', - nargs=1, - help=('path to generate build into (or use).' - ' This can be either a regular path or a ' - 'GN-style source-relative path like ' - '//out/Default.')) - subp.add_argument('-s', - '--swarmed', - action='store_true', - help='Run under swarming') - subp.add_argument('-d', - '--dimension', - default=[], - action='append', - nargs=2, - dest='dimensions', - metavar='FOO bar', - help='dimension to filter on') - subp.add_argument('target', nargs=1, help='ninja target to build and run') - subp.add_argument('extra_args', - nargs='*', - help=('extra args to pass to the isolate to run. ' - 'Use "--" as the first arg if you need to ' - 'pass switches')) - subp.set_defaults(func=self.CmdRun) - - subp = subps.add_parser('validate', help='validate the config file') - subp.add_argument('-f', - '--config-file', - metavar='PATH', - default=self.default_config, - help='path to config file (default is %(default)s)') - subp.set_defaults(func=self.CmdValidate) - - subp = subps.add_parser('help', help='Get help on a subcommand.') - subp.add_argument(nargs='?', - action='store', - dest='subcommand', - help='The command to get help for.') - subp.set_defaults(func=self.CmdHelp) - - self.args = parser.parse_args(argv) - - def DumpInputFiles(self): - def DumpContentsOfFilePassedTo(arg_name, path): - if path and self.Exists(path): - self.Print("\n# To recreate the file passed to %s:" % arg_name) - self.Print("%% cat > %s < returned %d' % ret) - if out: - self.Print(out, end='') - if err: - self.Print(err, end='', file=sys.stderr) - - return ret - - try: - archive_hashes = json.loads(self.ReadFile(archive_json_path)) - except Exception: - self.Print('Failed to read JSON file "%s"' % archive_json_path, - file=sys.stderr) - return 1 - try: - cas_digest = archive_hashes[target] - except Exception: - self.Print('Cannot find hash for "%s" in "%s", file content: %s' % - (target, archive_json_path, archive_hashes), - file=sys.stderr) - return 1 - - try: - json_dir = self.TempDir() - json_file = self.PathJoin(json_dir, 'task.json') - - cmd = [ - self.PathJoin('tools', 'luci-go', 'swarming'), - 'trigger', - '-realm', - self.args.realm, - '-digest', - cas_digest, - '-server', - swarming_server, - '-tag=purpose:user-debug-mb', - '-relative-cwd', - self.ToSrcRelPath(build_dir), - '-dump-json', - json_file, - ] + dimensions + ['--'] + list(isolate_cmd) - - if self.args.extra_args: - cmd += ['--'] + self.args.extra_args - self.Print('') - ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) - if ret: - return ret - task_json = self.ReadFile(json_file) - task_id = json.loads(task_json)["tasks"][0]['task_id'] - finally: - if json_dir: - self.RemoveDirectory(json_dir) - - cmd = [ - self.PathJoin('tools', 'luci-go', 'swarming'), - 'collect', - '-server', - swarming_server, - '-task-output-stdout=console', - task_id, - ] - ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) - return ret - - def _RunLocallyIsolated(self, build_dir, target): - cmd = [ - self.PathJoin(self.src_dir, 'tools', 'luci-go', self.isolate_exe), - 'run', - '-i', - self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), - ] - if self.args.extra_args: - cmd += ['--'] + self.args.extra_args - ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) - return ret - - def CmdValidate(self, print_ok=True): - errs = [] - - # Read the file to make sure it parses. - self.ReadConfigFile() - - # Build a list of all of the configs referenced by builders. - all_configs = {} - for builder_group in self.builder_groups: - for config in list(self.builder_groups[builder_group].values()): - if isinstance(config, dict): - for c in list(config.values()): - all_configs[c] = builder_group - else: - all_configs[config] = builder_group - - # Check that every referenced args file or config actually exists. - for config, loc in list(all_configs.items()): - if config.startswith('//'): - if not self.Exists(self.ToAbsPath(config)): - errs.append('Unknown args file "%s" referenced from "%s".' % - (config, loc)) - elif not config in self.configs: - errs.append('Unknown config "%s" referenced from "%s".' % (config, loc)) - - # Check that every actual config is actually referenced. - for config in self.configs: - if not config in all_configs: - errs.append('Unused config "%s".' % config) - - # Figure out the whole list of mixins, and check that every mixin - # listed by a config or another mixin actually exists. - referenced_mixins = set() - for config, mixins in list(self.configs.items()): - for mixin in mixins: - if not mixin in self.mixins: - errs.append('Unknown mixin "%s" referenced by config "%s".' % - (mixin, config)) - referenced_mixins.add(mixin) - - for mixin in self.mixins: - for sub_mixin in self.mixins[mixin].get('mixins', []): - if not sub_mixin in self.mixins: - errs.append('Unknown mixin "%s" referenced by mixin "%s".' % - (sub_mixin, mixin)) - referenced_mixins.add(sub_mixin) - - # Check that every mixin defined is actually referenced somewhere. - for mixin in self.mixins: - if not mixin in referenced_mixins: - errs.append('Unreferenced mixin "%s".' % mixin) - - if errs: - raise MBErr(('mb config file %s has problems:' % self.args.config_file) + - '\n ' + '\n '.join(errs)) - - if print_ok: - self.Print('mb config file %s looks ok.' % self.args.config_file) - return 0 - - def GetConfig(self): - build_dir = self.args.path[0] - - vals = self.DefaultVals() - if self.args.builder or self.args.builder_group or self.args.config: - vals = self.Lookup() - # Re-run gn gen in order to ensure the config is consistent with - # the build dir. - self.RunGNGen(vals) - return vals - - toolchain_path = self.PathJoin(self.ToAbsPath(build_dir), 'toolchain.ninja') - if not self.Exists(toolchain_path): - self.Print('Must either specify a path to an existing GN build ' - 'dir or pass in a -m/-b pair or a -c flag to specify ' - 'the configuration') - return {} - - vals['gn_args'] = self.GNArgsFromDir(build_dir) - return vals - - def GNArgsFromDir(self, build_dir): - args_contents = "" - gn_args_path = self.PathJoin(self.ToAbsPath(build_dir), 'args.gn') - if self.Exists(gn_args_path): - args_contents = self.ReadFile(gn_args_path) - gn_args = [] - for l in args_contents.splitlines(): - fields = l.split(' ') - name = fields[0] - val = ' '.join(fields[2:]) - gn_args.append('%s=%s' % (name, val)) - - return ' '.join(gn_args) - - def Lookup(self): - self.ReadConfigFile() - config = self.ConfigFromArgs() - if config.startswith('//'): - if not self.Exists(self.ToAbsPath(config)): - raise MBErr('args file "%s" not found' % config) - vals = self.DefaultVals() - vals['args_file'] = config - else: - if not config in self.configs: - raise MBErr('Config "%s" not found in %s' % - (config, self.args.config_file)) - vals = self.FlattenConfig(config) - return vals - - def ReadConfigFile(self): - if not self.Exists(self.args.config_file): - raise MBErr('config file not found at %s' % self.args.config_file) - - try: - contents = ast.literal_eval(self.ReadFile(self.args.config_file)) - except SyntaxError as e: - raise MBErr('Failed to parse config file "%s"' % - self.args.config_file) from e - - self.configs = contents['configs'] - self.builder_groups = contents['builder_groups'] - self.mixins = contents['mixins'] - - def ReadIsolateMap(self): - isolate_map = self.args.isolate_map_file - if not self.Exists(isolate_map): - raise MBErr('isolate map file not found at %s' % isolate_map) - try: - return ast.literal_eval(self.ReadFile(isolate_map)) - except SyntaxError as e: - raise MBErr('Failed to parse isolate map file "%s"' % isolate_map) from e - - def ConfigFromArgs(self): - if self.args.config: - if self.args.builder_group or self.args.builder: - raise MBErr('Can not specific both -c/--config and ' - '-m/--builder-group or -b/--builder') - - return self.args.config - - if not self.args.builder_group or not self.args.builder: - raise MBErr('Must specify either -c/--config or ' - '(-m/--builder-group and -b/--builder)') - - if not self.args.builder_group in self.builder_groups: - raise MBErr('Master name "%s" not found in "%s"' % - (self.args.builder_group, self.args.config_file)) - - if not self.args.builder in self.builder_groups[self.args.builder_group]: - raise MBErr( - 'Builder name "%s" not found under builder_groups[%s] in "%s"' % - (self.args.builder, self.args.builder_group, self.args.config_file)) - - config = (self.builder_groups[self.args.builder_group][self.args.builder]) - if isinstance(config, dict): - if self.args.phase is None: - raise MBErr('Must specify a build --phase for %s on %s' % - (self.args.builder, self.args.builder_group)) - phase = str(self.args.phase) - if phase not in config: - raise MBErr('Phase %s doesn\'t exist for %s on %s' % - (phase, self.args.builder, self.args.builder_group)) - return config[phase] - - if self.args.phase is not None: - raise MBErr('Must not specify a build --phase for %s on %s' % - (self.args.builder, self.args.builder_group)) - return config - - def FlattenConfig(self, config): - mixins = self.configs[config] - vals = self.DefaultVals() - - visited = [] - self.FlattenMixins(mixins, vals, visited) - return vals - - @staticmethod - def DefaultVals(): - return { - 'args_file': '', - 'cros_passthrough': False, - 'gn_args': '', - } - - def FlattenMixins(self, mixins, vals, visited): - for m in mixins: - if m not in self.mixins: - raise MBErr('Unknown mixin "%s"' % m) - - visited.append(m) - - mixin_vals = self.mixins[m] - - if 'cros_passthrough' in mixin_vals: - vals['cros_passthrough'] = mixin_vals['cros_passthrough'] - if 'gn_args' in mixin_vals: - if vals['gn_args']: - vals['gn_args'] += ' ' + mixin_vals['gn_args'] - else: - vals['gn_args'] = mixin_vals['gn_args'] - - if 'mixins' in mixin_vals: - self.FlattenMixins(mixin_vals['mixins'], vals, visited) - return vals - - def RunGNGen(self, vals): - build_dir = self.args.path[0] - - cmd = self.GNCmd('gen', build_dir, '--check') - gn_args = self.GNArgs(vals) - - # Since GN hasn't run yet, the build directory may not even exist. - self.MaybeMakeDirectory(self.ToAbsPath(build_dir)) - - gn_args_path = self.ToAbsPath(build_dir, 'args.gn') - self.WriteFile(gn_args_path, gn_args, force_verbose=True) - - swarming_targets = set() - if getattr(self.args, 'swarming_targets_file', None): - # We need GN to generate the list of runtime dependencies for - # the compile targets listed (one per line) in the file so - # we can run them via swarming. We use gn_isolate_map.pyl to - # convert the compile targets to the matching GN labels. - path = self.args.swarming_targets_file - if not self.Exists(path): - self.WriteFailureAndRaise('"%s" does not exist' % path, - output_path=None) - contents = self.ReadFile(path) - swarming_targets = set(contents.splitlines()) - - isolate_map = self.ReadIsolateMap() - err, labels = self.MapTargetsToLabels(isolate_map, swarming_targets) - if err: - raise MBErr(err) - - gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps') - self.WriteFile(gn_runtime_deps_path, '\n'.join(labels) + '\n') - cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path) - - ret, output, _ = self.Run(cmd) - if ret: - if self.args.json_output: - # write errors to json.output - self.WriteJSON({'output': output}, self.args.json_output) - # If `gn gen` failed, we should exit early rather than trying to - # generate isolates. Run() will have already logged any error - # output. - self.Print('GN gen failed: %d' % ret) - return ret - - android = 'target_os="android"' in vals['gn_args'] - for target in swarming_targets: - if android: - # Android targets may be either android_apk or executable. The - # former will result in runtime_deps associated with the stamp - # file, while the latter will result in runtime_deps associated - # with the executable. - label = isolate_map[target]['label'] - runtime_deps_targets = [ - target + '.runtime_deps', - 'obj/%s.stamp.runtime_deps' % label.replace(':', '/') - ] - elif isolate_map[target]['type'] == 'gpu_browser_test': - if self.platform == 'win32': - runtime_deps_targets = ['browser_tests.exe.runtime_deps'] - else: - runtime_deps_targets = ['browser_tests.runtime_deps'] - elif isolate_map[target]['type'] == 'script': - label = isolate_map[target]['label'].split(':')[1] - runtime_deps_targets = ['%s.runtime_deps' % label] - if self.platform == 'win32': - runtime_deps_targets += [label + '.exe.runtime_deps'] - else: - runtime_deps_targets += [label + '.runtime_deps'] - elif self.platform == 'win32': - runtime_deps_targets = [target + '.exe.runtime_deps'] - else: - runtime_deps_targets = [target + '.runtime_deps'] - - for r in runtime_deps_targets: - runtime_deps_path = self.ToAbsPath(build_dir, r) - if self.Exists(runtime_deps_path): - break - else: - raise MBErr('did not generate any of %s' % - ', '.join(runtime_deps_targets)) - - command, extra_files = self.GetSwarmingCommand(target, vals) - - runtime_deps = self.ReadFile(runtime_deps_path).splitlines() - - self.WriteIsolateFiles(build_dir, command, target, runtime_deps, - extra_files) - - return 0 - - def RunGNIsolate(self, vals): - target = self.args.target[0] - isolate_map = self.ReadIsolateMap() - err, labels = self.MapTargetsToLabels(isolate_map, [target]) - if err: - raise MBErr(err) - label = labels[0] - - build_dir = self.args.path[0] - command, extra_files = self.GetSwarmingCommand(target, vals) - - cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps') - ret, out, _ = self.Call(cmd) - if ret: - if out: - self.Print(out) - return ret - - runtime_deps = out.splitlines() - - self.WriteIsolateFiles(build_dir, command, target, runtime_deps, - extra_files) - - ret, _, _ = self.Run([ - self.PathJoin(self.src_dir, 'tools', 'luci-go', self.isolate_exe), - 'check', '-i', - self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)) - ], - buffer_output=False) - - return ret - - def WriteIsolateFiles(self, build_dir, command, target, runtime_deps, - extra_files): - isolate_path = self.ToAbsPath(build_dir, target + '.isolate') - self.WriteFile( - isolate_path, - pprint.pformat({ - 'variables': { - 'command': command, - 'files': sorted(runtime_deps + extra_files), - } - }) + '\n') - - self.WriteJSON( - { - 'args': [ - '--isolate', - self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), - ], - 'dir': - self.src_dir, - 'version': - 1, - }, - isolate_path + 'd.gen.json', - ) - - @staticmethod - def MapTargetsToLabels(isolate_map, targets): - labels = [] - err = '' - - def StripTestSuffixes(target): - for suffix in ('_apk_run', '_apk', '_run'): - if target.endswith(suffix): - return target[:-len(suffix)], suffix - return None, None - - for target in targets: - if target == 'all': - labels.append(target) - elif target.startswith('//'): - labels.append(target) - else: - if target in isolate_map: - stripped_target, suffix = target, '' - else: - stripped_target, suffix = StripTestSuffixes(target) - if stripped_target in isolate_map: - if isolate_map[stripped_target]['type'] == 'unknown': - err += ('test target "%s" type is unknown\n' % target) - else: - labels.append(isolate_map[stripped_target]['label'] + suffix) - else: - err += ('target "%s" not found in ' - '//testing/buildbot/gn_isolate_map.pyl\n' % target) - - return err, labels - - def GNCmd(self, subcommand, path, *args): - if self.platform.startswith('linux'): - subdir, exe = 'linux64', 'gn' - elif self.platform == 'darwin': - subdir, exe = 'mac', 'gn' - else: - subdir, exe = 'win', 'gn.exe' - - gn_path = self.PathJoin(self.src_dir, 'buildtools', subdir, exe) - return [gn_path, subcommand, path] + list(args) - - def GNArgs(self, vals): - if vals['cros_passthrough']: - if not 'GN_ARGS' in os.environ: - raise MBErr('MB is expecting GN_ARGS to be in the environment') - gn_args = os.environ['GN_ARGS'] - if not re.search('target_os.*=.*"chromeos"', gn_args): - raise MBErr('GN_ARGS is missing target_os = "chromeos": ' - '(GN_ARGS=%s)' % gn_args) - else: - gn_args = vals['gn_args'] - - if self.args.goma_dir: - gn_args += ' goma_dir="%s"' % self.args.goma_dir - - android_version_code = self.args.android_version_code - if android_version_code: - gn_args += (' android_default_version_code="%s"' % android_version_code) - - android_version_name = self.args.android_version_name - if android_version_name: - gn_args += (' android_default_version_name="%s"' % android_version_name) - - # Canonicalize the arg string into a sorted, newline-separated list - # of key-value pairs, and de-dup the keys if need be so that only - # the last instance of each arg is listed. - gn_args = gn_helpers.ToGNString(gn_helpers.FromGNArgs(gn_args)) - - args_file = vals.get('args_file', None) - if args_file: - gn_args = ('import("%s")\n' % vals['args_file']) + gn_args - return gn_args + super(WebRTCMetaBuildWrapper, self).__init__() + # Make sure default_config and default_isolate_map are attributes of the + # parent class before changing their values. + # pylint: disable=access-member-before-definition + assert self.default_config + assert self.default_isolate_map + self.default_config = os.path.join(_SCRIPT_DIR, 'mb_config.pyl') + self.default_isolate_map = os.path.join(_SCRIPT_DIR, 'gn_isolate_map.pyl') def GetSwarmingCommand(self, target, vals): isolate_map = self.ReadIsolateMap() @@ -1012,329 +158,5 @@ class MetaBuildWrapper: return cmdline, extra_files - def ToAbsPath(self, build_path, *comps): - return self.PathJoin(self.src_dir, self.ToSrcRelPath(build_path), *comps) - - def ToSrcRelPath(self, path): - """Returns a relative path from the top of the repo.""" - if path.startswith('//'): - return path[2:].replace('/', self.sep) - return self.RelPath(path, self.src_dir) - - def RunGNAnalyze(self, vals): - # Analyze runs before 'gn gen' now, so we need to run gn gen - # in order to ensure that we have a build directory. - ret = self.RunGNGen(vals) - if ret: - return ret - - build_path = self.args.path[0] - input_path = self.args.input_path[0] - gn_input_path = input_path + '.gn' - output_path = self.args.output_path[0] - gn_output_path = output_path + '.gn' - - inp = self.ReadInputJSON( - ['files', 'test_targets', 'additional_compile_targets']) - if self.args.verbose: - self.Print() - self.Print('analyze input:') - self.PrintJSON(inp) - self.Print() - - # This shouldn't normally happen, but could due to unusual race - # conditions, like a try job that gets scheduled before a patch - # lands but runs after the patch has landed. - if not inp['files']: - self.Print('Warning: No files modified in patch, bailing out early.') - self.WriteJSON( - { - 'status': 'No dependency', - 'compile_targets': [], - 'test_targets': [], - }, output_path) - return 0 - - gn_inp = {} - gn_inp['files'] = ['//' + f for f in inp['files'] if not f.startswith('//')] - - isolate_map = self.ReadIsolateMap() - err, gn_inp['additional_compile_targets'] = self.MapTargetsToLabels( - isolate_map, inp['additional_compile_targets']) - if err: - raise MBErr(err) - - err, gn_inp['test_targets'] = self.MapTargetsToLabels( - isolate_map, inp['test_targets']) - if err: - raise MBErr(err) - labels_to_targets = {} - for i, label in enumerate(gn_inp['test_targets']): - labels_to_targets[label] = inp['test_targets'][i] - - try: - self.WriteJSON(gn_inp, gn_input_path) - cmd = self.GNCmd('analyze', build_path, gn_input_path, gn_output_path) - ret, output, _ = self.Run(cmd, force_verbose=True) - if ret: - if self.args.json_output: - # write errors to json.output - self.WriteJSON({'output': output}, self.args.json_output) - return ret - - gn_outp_str = self.ReadFile(gn_output_path) - try: - gn_outp = json.loads(gn_outp_str) - except Exception as e: - self.Print("Failed to parse the JSON string GN " - "returned: %s\n%s" % (repr(gn_outp_str), str(e))) - raise - - outp = {} - if 'status' in gn_outp: - outp['status'] = gn_outp['status'] - if 'error' in gn_outp: - outp['error'] = gn_outp['error'] - if 'invalid_targets' in gn_outp: - outp['invalid_targets'] = gn_outp['invalid_targets'] - if 'compile_targets' in gn_outp: - if 'all' in gn_outp['compile_targets']: - outp['compile_targets'] = ['all'] - else: - outp['compile_targets'] = [ - label.replace('//', '') for label in gn_outp['compile_targets'] - ] - if 'test_targets' in gn_outp: - outp['test_targets'] = [ - labels_to_targets[label] for label in gn_outp['test_targets'] - ] - - if self.args.verbose: - self.Print() - self.Print('analyze output:') - self.PrintJSON(outp) - self.Print() - - self.WriteJSON(outp, output_path) - - finally: - if self.Exists(gn_input_path): - self.RemoveFile(gn_input_path) - if self.Exists(gn_output_path): - self.RemoveFile(gn_output_path) - - return 0 - - def ReadInputJSON(self, required_keys): - path = self.args.input_path[0] - output_path = self.args.output_path[0] - if not self.Exists(path): - self.WriteFailureAndRaise('"%s" does not exist' % path, output_path) - - try: - inp = json.loads(self.ReadFile(path)) - except Exception as e: - self.WriteFailureAndRaise( - 'Failed to read JSON input from "%s": %s' % (path, e), output_path) - - for k in required_keys: - if not k in inp: - self.WriteFailureAndRaise('input file is missing a "%s" key' % k, - output_path) - - return inp - - def WriteFailureAndRaise(self, msg, output_path): - if output_path: - self.WriteJSON({'error': msg}, output_path, force_verbose=True) - raise MBErr(msg) - - def WriteJSON(self, obj, path, force_verbose=False): - try: - self.WriteFile(path, - json.dumps(obj, indent=2, sort_keys=True) + '\n', - force_verbose=force_verbose) - except Exception as e: - raise MBErr('Error writing to the output path "%s"' % path) from e - - def PrintCmd(self, cmd, env): - if self.platform == 'win32': - env_prefix = 'set ' - env_quoter = QuoteForSet - shell_quoter = QuoteForCmd - else: - env_prefix = '' - env_quoter = pipes.quote - shell_quoter = pipes.quote - - var = 'LLVM_FORCE_HEAD_REVISION' - if env and var in env: - self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var]))) - - if cmd[0] == self.executable: - cmd = ['vpython3'] + cmd[1:] - self.Print(*[shell_quoter(arg) for arg in cmd]) - - def PrintJSON(self, obj): - self.Print(json.dumps(obj, indent=2, sort_keys=True)) - - def Build(self, target): - build_dir = self.ToSrcRelPath(self.args.path[0]) - ninja_cmd = ['ninja', '-C', build_dir] - if self.args.jobs: - ninja_cmd.extend(['-j', '%d' % self.args.jobs]) - ninja_cmd.append(target) - ret, _, _ = self.Run(ninja_cmd, force_verbose=False, buffer_output=False) - return ret - - def Run(self, cmd, env=None, force_verbose=True, buffer_output=True): - # This function largely exists so it can be overridden for testing. - if self.args.dryrun or self.args.verbose or force_verbose: - self.PrintCmd(cmd, env) - if self.args.dryrun: - return 0, '', '' - - ret, out, err = self.Call(cmd, env=env, buffer_output=buffer_output) - if self.args.verbose or force_verbose: - if ret: - self.Print(' -> returned %d' % ret) - if out: - self.Print(out, end='') - if err: - self.Print(err, end='', file=sys.stderr) - return ret, out, err - - def Call(self, cmd, env=None, buffer_output=True): - if buffer_output: - p = subprocess.Popen(cmd, - shell=False, - cwd=self.src_dir, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env) - out, err = p.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') - else: - p = subprocess.Popen(cmd, shell=False, cwd=self.src_dir, env=env) - p.wait() - out = err = '' - return p.returncode, out, err - - @staticmethod - def ExpandUser(path): - # This function largely exists so it can be overridden for testing. - return os.path.expanduser(path) - - @staticmethod - def Exists(path): - # This function largely exists so it can be overridden for testing. - return os.path.exists(path) - - @staticmethod - def Fetch(url): - # This function largely exists so it can be overridden for testing. - f = urlopen(url) - contents = f.read() - f.close() - return contents - - @staticmethod - def MaybeMakeDirectory(path): - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - @staticmethod - def PathJoin(*comps): - # This function largely exists so it can be overriden for testing. - return os.path.join(*comps) - - @staticmethod - def Print(*args, **kwargs): - # This function largely exists so it can be overridden for testing. - print(*args, **kwargs) - if kwargs.get('stream', sys.stdout) == sys.stdout: - sys.stdout.flush() - - @staticmethod - def ReadFile(path): - # This function largely exists so it can be overriden for testing. - with open(path) as fp: - return fp.read() - - @staticmethod - def RelPath(path, start='.'): - # This function largely exists so it can be overriden for testing. - return os.path.relpath(path, start) - - @staticmethod - def RemoveFile(path): - # This function largely exists so it can be overriden for testing. - os.remove(path) - - def RemoveDirectory(self, abs_path): - if self.platform == 'win32': - # In other places in chromium, we often have to retry this command - # because we're worried about other processes still holding on to - # file handles, but when MB is invoked, it will be early enough in - # the build that their should be no other processes to interfere. - # We can change this if need be. - self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path]) - else: - shutil.rmtree(abs_path, ignore_errors=True) - - @staticmethod - def TempDir(): - # This function largely exists so it can be overriden for testing. - return tempfile.mkdtemp(prefix='mb_') - - @staticmethod - def TempFile(mode='w'): - # This function largely exists so it can be overriden for testing. - return tempfile.NamedTemporaryFile(mode=mode, delete=False) - - def WriteFile(self, path, contents, force_verbose=False): - # This function largely exists so it can be overriden for testing. - if self.args.dryrun or self.args.verbose or force_verbose: - self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path)) - with open(path, 'w') as fp: - return fp.write(contents) - - -class MBErr(Exception): - pass - - -# See http://goo.gl/l5NPDW and http://goo.gl/4Diozm for the painful -# details of this next section, which handles escaping command lines -# so that they can be copied and pasted into a cmd window. -UNSAFE_FOR_SET = set('^<>&|') -UNSAFE_FOR_CMD = UNSAFE_FOR_SET.union(set('()%')) -ALL_META_CHARS = UNSAFE_FOR_CMD.union(set('"')) - - -def QuoteForSet(arg): - if any(a in UNSAFE_FOR_SET for a in arg): - arg = ''.join('^' + a if a in UNSAFE_FOR_SET else a for a in arg) - return arg - - -def QuoteForCmd(arg): - # First, escape the arg so that CommandLineToArgvW will parse it properly. - if arg == '' or ' ' in arg or '"' in arg: - quote_re = re.compile(r'(\\*)"') - arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg)) - - # Then check to see if the arg contains any metacharacters other than - # double quotes; if it does, quote everything (including the double - # quotes) for safety. - if any(a in UNSAFE_FOR_CMD for a in arg): - arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg) - return arg - - if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/tools_webrtc/mb/mb_config.pyl b/tools_webrtc/mb/mb_config.pyl index 89dd294f87..0080ae23c5 100644 --- a/tools_webrtc/mb/mb_config.pyl +++ b/tools_webrtc/mb/mb_config.pyl @@ -18,6 +18,9 @@ # The builders should be sorted by the order they appear in the /builders # page on the buildbots, *not* alphabetically. 'builder_groups': { + # This is required because WebRTC mb.py overwrites the default configs + # and Chromium's mb.py checks the default config contains 'chromium'. + 'chromium': {}, 'client.webrtc': { # iOS 'iOS64 Debug': 'ios_debug_bot_arm64', diff --git a/tools_webrtc/mb/mb_unittest.py b/tools_webrtc/mb/mb_unittest.py index 6f4b59a9ed..18e276a2d4 100755 --- a/tools_webrtc/mb/mb_unittest.py +++ b/tools_webrtc/mb/mb_unittest.py @@ -19,16 +19,20 @@ import sys import tempfile import unittest -import mb +_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +_SRC_DIR = os.path.dirname(os.path.dirname(_SCRIPT_DIR)) +sys.path.insert(0, _SRC_DIR) + +from tools_webrtc.mb import mb -class FakeMBW(mb.MetaBuildWrapper): +class FakeMBW(mb.WebRTCMetaBuildWrapper): def __init__(self, win32=False): super(FakeMBW, self).__init__() # Override vars for test portability. if win32: - self.src_dir = 'c:\\fake_src' + self.chromium_src_dir = 'c:\\fake_src' self.default_config = 'c:\\fake_src\\tools_webrtc\\mb\\mb_config.pyl' self.default_isolate_map = ('c:\\fake_src\\testing\\buildbot\\' 'gn_isolate_map.pyl') @@ -37,7 +41,7 @@ class FakeMBW(mb.MetaBuildWrapper): self.sep = '\\' self.cwd = 'c:\\fake_src\\out\\Default' else: - self.src_dir = '/fake_src' + self.chromium_src_dir = '/fake_src' self.default_config = '/fake_src/tools_webrtc/mb/mb_config.pyl' self.default_isolate_map = '/fake_src/testing/buildbot/gn_isolate_map.pyl' self.executable = '/usr/bin/vpython3' @@ -59,7 +63,15 @@ class FakeMBW(mb.MetaBuildWrapper): def Exists(self, path): abs_path = self._AbsPath(path) - return self.files.get(abs_path) is not None or abs_path in self.dirs + return (self.files.get(abs_path) is not None or abs_path in self.dirs) + + def ListDir(self, path): + dir_contents = [] + for f in list(self.files.keys()) + list(self.dirs): + head, _ = os.path.split(f) + if head == path: + dir_contents.append(f) + return dir_contents def MaybeMakeDirectory(self, path): abpath = self._AbsPath(path) @@ -69,7 +81,10 @@ class FakeMBW(mb.MetaBuildWrapper): return self.sep.join(comps) def ReadFile(self, path): - return self.files[self._AbsPath(path)] + try: + return self.files[self._AbsPath(path)] + except KeyError: + raise IOError('%s not found' % path) def WriteFile(self, path, contents, force_verbose=False): if self.args.dryrun or self.args.verbose or force_verbose: @@ -77,7 +92,10 @@ class FakeMBW(mb.MetaBuildWrapper): abpath = self._AbsPath(path) self.files[abpath] = contents - def Call(self, cmd, env=None, buffer_output=True): + def Call(self, cmd, env=None, buffer_output=True, stdin=None): + del env + del buffer_output + del stdin self.calls.append(cmd) if self.cmds: return self.cmds.pop(0) @@ -97,17 +115,20 @@ class FakeMBW(mb.MetaBuildWrapper): self.dirs.add(tmp_dir) return tmp_dir - def TempFile(self): + def TempFile(self, mode='w'): + del mode return FakeFile(self.files) def RemoveFile(self, path): abpath = self._AbsPath(path) self.files[abpath] = None - def RemoveDirectory(self, path): - abpath = self._AbsPath(path) - self.rmdirs.append(abpath) - files_to_delete = [f for f in self.files if f.startswith(abpath)] + def RemoveDirectory(self, abs_path): + # Normalize the passed-in path to handle different working directories + # used during unit testing. + abs_path = self._AbsPath(abs_path) + self.rmdirs.append(abs_path) + files_to_delete = [f for f in self.files if f.startswith(abs_path)] for f in files_to_delete: self.files[f] = None @@ -140,19 +161,30 @@ TEST_CONFIG = """\ 'fake_group': { 'fake_builder': 'rel_bot', 'fake_debug_builder': 'debug_goma', - 'fake_args_bot': '//build/args/bots/fake_group/fake_args_bot.gn', + 'fake_args_bot': 'fake_args_bot', 'fake_multi_phase': { 'phase_1': 'phase_1', 'phase_2': 'phase_2'}, 'fake_android_bot': 'android_bot', + 'fake_args_file': 'args_file_goma', + 'fake_ios_error': 'ios_error', }, }, 'configs': { + 'args_file_goma': ['fake_args_bot', 'goma'], + 'fake_args_bot': ['fake_args_bot'], 'rel_bot': ['rel', 'goma', 'fake_feature1'], 'debug_goma': ['debug', 'goma'], - 'phase_1': ['phase_1'], - 'phase_2': ['phase_2'], + 'phase_1': ['rel', 'phase_1'], + 'phase_2': ['rel', 'phase_2'], 'android_bot': ['android'], + 'ios_error': ['error'], }, 'mixins': { + 'error': { + 'gn_args': 'error', + }, + 'fake_args_bot': { + 'args_file': '//build/args/bots/fake_group/fake_args_bot.gn', + }, 'fake_feature1': { 'gn_args': 'enable_doom_melon=true', }, @@ -166,13 +198,13 @@ TEST_CONFIG = """\ 'gn_args': 'phase=2', }, 'rel': { - 'gn_args': 'is_debug=false', + 'gn_args': 'is_debug=false dcheck_always_on=false', }, 'debug': { 'gn_args': 'is_debug=true', }, 'android': { - 'gn_args': 'target_os="android"', + 'gn_args': 'target_os="android" dcheck_always_on=false', } }, } @@ -193,10 +225,23 @@ class UnitTest(unittest.TestCase): }''') mbw.files.setdefault( mbw.ToAbsPath('//build/args/bots/fake_group/fake_args_bot.gn'), - 'is_debug = false\n') + 'is_debug = false\ndcheck_always_on=false\n') + mbw.files.setdefault(mbw.ToAbsPath('//tools/mb/rts_banned_suites.json'), + '{}') if files: for path, contents in list(files.items()): mbw.files[path] = contents + if path.endswith('.runtime_deps'): + + def fake_call(cmd, env=None, buffer_output=True, stdin=None): + del cmd + del env + del buffer_output + del stdin + mbw.files[path] = contents + return 0, '', '' + + mbw.Call = fake_call return mbw def check(self, @@ -330,7 +375,7 @@ class UnitTest(unittest.TestCase): '/fake_src/testing/buildbot/gn_isolate_map.pyl': ("{'base_unittests': {" " 'label': '//base:base_unittests'," - " 'type': 'additional_compile_target'," + " 'type': 'console_test_launcher'," "}}\n"), '/fake_src/out/Default/base_unittests.runtime_deps': ("base_unittests\n"), @@ -461,7 +506,7 @@ class UnitTest(unittest.TestCase): " 'type': 'script'," " 'script': '//base/base_unittests_script.py'," "}}\n"), - '/fake_src/out/Default/base_unittests.runtime_deps': + '/fake_src/out/Default/base_unittests_script.runtime_deps': ("base_unittests\n" "base_unittests_script.py\n"), } @@ -803,7 +848,7 @@ class UnitTest(unittest.TestCase): ret=0) # test running isolate on an existing build_dir - files['/fake_src/out/Default/args.gn'] = 'is_debug = True\n' + files['/fake_src/out/Default/args.gn'] = 'is_debug = true\n' self.check(['isolate', '//out/Default', 'base_unittests'], files=files, ret=0) @@ -827,6 +872,7 @@ class UnitTest(unittest.TestCase): ret=0) def test_run_swarmed(self): + # pylint: disable=attribute-defined-outside-init files = { '/fake_src/testing/buildbot/gn_isolate_map.pyl': ("{'base_unittests': {" @@ -885,6 +931,7 @@ class UnitTest(unittest.TestCase): ret=0, out=('\n' 'Writing """\\\n' + 'dcheck_always_on = false\n' 'enable_doom_melon = true\n' 'goma_dir = "/foo"\n' 'is_debug = false\n'