diff --git a/tools_webrtc/mb/mb.py b/tools_webrtc/mb/mb.py index b0266ac5e2..e8d2d63245 100755 --- a/tools_webrtc/mb/mb.py +++ b/tools_webrtc/mb/mb.py @@ -40,1218 +40,1248 @@ import gn_helpers def main(args): - mbw = MetaBuildWrapper() - return mbw.Main(args) + mbw = MetaBuildWrapper() + return mbw.Main(args) class MetaBuildWrapper(object): - def __init__(self): - self.src_dir = SRC_DIR - self.default_config = os.path.join(SCRIPT_DIR, 'mb_config.pyl') - self.default_isolate_map = os.path.join(SCRIPT_DIR, 'gn_isolate_map.pyl') - self.executable = sys.executable - self.platform = sys.platform - self.sep = os.sep - self.args = argparse.Namespace() - self.configs = {} - self.builder_groups = {} - self.mixins = {} - self.isolate_exe = 'isolate.exe' if self.platform.startswith( - 'win') else 'isolate' + def __init__(self): + self.src_dir = SRC_DIR + self.default_config = os.path.join(SCRIPT_DIR, 'mb_config.pyl') + self.default_isolate_map = os.path.join( + SCRIPT_DIR, 'gn_isolate_map.pyl') + self.executable = sys.executable + self.platform = sys.platform + self.sep = os.sep + self.args = argparse.Namespace() + self.configs = {} + self.builder_groups = {} + self.mixins = {} + self.isolate_exe = 'isolate.exe' if self.platform.startswith( + 'win') else 'isolate' - def Main(self, args): - self.ParseArgs(args) - try: - ret = self.args.func() - if ret: - self.DumpInputFiles() - return ret - except KeyboardInterrupt: - self.Print('interrupted, exiting') - return 130 - except Exception: - self.DumpInputFiles() - s = traceback.format_exc() - for l in s.splitlines(): - self.Print(l) - return 1 + def Main(self, args): + self.ParseArgs(args) + try: + ret = self.args.func() + if ret: + self.DumpInputFiles() + return ret + except KeyboardInterrupt: + self.Print('interrupted, exiting') + return 130 + except Exception: + self.DumpInputFiles() + s = traceback.format_exc() + for l in s.splitlines(): + self.Print(l) + return 1 - def ParseArgs(self, argv): - def AddCommonOptions(subp): - subp.add_argument('-b', '--builder', - help='builder name to look up config from') - subp.add_argument('-m', '--builder-group', - help='builder group name to look up config from') - subp.add_argument('-c', '--config', - help='configuration to analyze') - subp.add_argument('--phase', - help='optional phase name (used when builders ' - 'do multiple compiles with different ' - 'arguments in a single build)') - subp.add_argument('-f', '--config-file', metavar='PATH', - default=self.default_config, - help='path to config file ' - '(default is %(default)s)') - subp.add_argument('-i', '--isolate-map-file', metavar='PATH', - default=self.default_isolate_map, - help='path to isolate map file ' - '(default is %(default)s)') - subp.add_argument('-r', '--realm', default='webrtc:try', - help='optional LUCI realm to use (for ' - 'example when triggering tasks on Swarming)') - subp.add_argument('-g', '--goma-dir', - help='path to goma directory') - subp.add_argument('--android-version-code', - help='Sets GN arg android_default_version_code') - subp.add_argument('--android-version-name', - help='Sets GN arg android_default_version_name') - subp.add_argument('-n', '--dryrun', action='store_true', - help='Do a dry run (i.e., do nothing, just print ' - 'the commands that will run)') - subp.add_argument('-v', '--verbose', action='store_true', - help='verbose logging') + def ParseArgs(self, argv): + def AddCommonOptions(subp): + subp.add_argument('-b', '--builder', + help='builder name to look up config from') + subp.add_argument('-m', '--builder-group', + help='builder group name to look up config from') + subp.add_argument('-c', '--config', + help='configuration to analyze') + subp.add_argument('--phase', + help='optional phase name (used when builders ' + 'do multiple compiles with different ' + 'arguments in a single build)') + subp.add_argument('-f', '--config-file', metavar='PATH', + default=self.default_config, + help='path to config file ' + '(default is %(default)s)') + subp.add_argument('-i', '--isolate-map-file', metavar='PATH', + default=self.default_isolate_map, + help='path to isolate map file ' + '(default is %(default)s)') + subp.add_argument('-r', '--realm', default='webrtc:try', + help='optional LUCI realm to use (for example ' + 'when triggering tasks on Swarming)') + subp.add_argument('-g', '--goma-dir', + help='path to goma directory') + subp.add_argument('--android-version-code', + help='Sets GN arg android_default_version_code') + subp.add_argument('--android-version-name', + help='Sets GN arg android_default_version_name') + subp.add_argument('-n', '--dryrun', action='store_true', + help='Do a dry run (i.e., do nothing, just ' + 'print the commands that will run)') + subp.add_argument('-v', '--verbose', action='store_true', + help='verbose logging') - parser = argparse.ArgumentParser(prog='mb') - subps = parser.add_subparsers() + parser = argparse.ArgumentParser(prog='mb') + subps = parser.add_subparsers() - subp = subps.add_parser('analyze', - help='analyze whether changes to a set of files ' - 'will cause a set of binaries to be rebuilt.') - AddCommonOptions(subp) - subp.add_argument('path', nargs=1, - help='path build was generated into.') - subp.add_argument('input_path', nargs=1, - help='path to a file containing the input arguments ' - 'as a JSON object.') - subp.add_argument('output_path', nargs=1, - help='path to a file containing the output arguments ' - 'as a JSON object.') - subp.add_argument('--json-output', - help='Write errors to json.output') - subp.set_defaults(func=self.CmdAnalyze) + subp = subps.add_parser('analyze', + help='analyze whether changes to a set of ' + 'files will cause a set of binaries ' + 'to be rebuilt.') + AddCommonOptions(subp) + subp.add_argument('path', nargs=1, + help='path build was generated into.') + subp.add_argument('input_path', nargs=1, + help='path to a file containing the input ' + 'arguments as a JSON object.') + subp.add_argument('output_path', nargs=1, + help='path to a file containing the output ' + 'arguments as a JSON object.') + subp.add_argument('--json-output', + help='Write errors to json.output') + subp.set_defaults(func=self.CmdAnalyze) - subp = subps.add_parser('export', - help='print out the expanded configuration for' - 'each builder as a JSON object') - subp.add_argument('-f', '--config-file', metavar='PATH', - default=self.default_config, - help='path to config file (default is %(default)s)') - subp.add_argument('-g', '--goma-dir', - help='path to goma directory') - subp.set_defaults(func=self.CmdExport) + subp = subps.add_parser('export', + help='print out the expanded configuration for' + 'each builder as a JSON object') + subp.add_argument('-f', '--config-file', metavar='PATH', + default=self.default_config, + help='path to config file (default is %(default)s)') + subp.add_argument('-g', '--goma-dir', + help='path to goma directory') + subp.set_defaults(func=self.CmdExport) - subp = subps.add_parser('gen', - help='generate a new set of build files') - AddCommonOptions(subp) - subp.add_argument('--swarming-targets-file', - help='save runtime dependencies for targets listed ' - 'in file.') - subp.add_argument('--json-output', - help='Write errors to json.output') - subp.add_argument('path', nargs=1, - help='path to generate build into') - subp.set_defaults(func=self.CmdGen) + subp = subps.add_parser('gen', + help='generate a new set of build files') + AddCommonOptions(subp) + subp.add_argument('--swarming-targets-file', + help='save runtime dependencies for targets listed ' + 'in file.') + subp.add_argument('--json-output', + help='Write errors to json.output') + subp.add_argument('path', nargs=1, + help='path to generate build into') + subp.set_defaults(func=self.CmdGen) - subp = subps.add_parser('isolate', - help='generate the .isolate files for a given' - 'binary') - AddCommonOptions(subp) - subp.add_argument('path', nargs=1, - help='path build was generated into') - subp.add_argument('target', nargs=1, - help='ninja target to generate the isolate for') - subp.set_defaults(func=self.CmdIsolate) + subp = subps.add_parser('isolate', + help='generate the .isolate files for a given' + 'binary') + AddCommonOptions(subp) + subp.add_argument('path', nargs=1, + help='path build was generated into') + subp.add_argument('target', nargs=1, + help='ninja target to generate the isolate for') + subp.set_defaults(func=self.CmdIsolate) - subp = subps.add_parser('lookup', - help='look up the command for a given config or ' - 'builder') - AddCommonOptions(subp) - subp.add_argument('--quiet', default=False, action='store_true', - help='Print out just the arguments, ' - 'do not emulate the output of the gen subcommand.') - subp.set_defaults(func=self.CmdLookup) + subp = subps.add_parser('lookup', + help='look up the command for a given config ' + 'or builder') + AddCommonOptions(subp) + subp.add_argument('--quiet', default=False, action='store_true', + help='Print out just the arguments, do ' + 'not emulate the output of the gen subcommand.') + subp.set_defaults(func=self.CmdLookup) - subp = subps.add_parser( - 'run', - help='build and run the isolated version of a ' - 'binary', - formatter_class=argparse.RawDescriptionHelpFormatter) - subp.description = ( - 'Build, isolate, and run the given binary with the command line\n' - 'listed in the isolate. You may pass extra arguments after the\n' - 'target; use "--" if the extra arguments need to include switches.\n' - '\n' - 'Examples:\n' - '\n' - ' % tools/mb/mb.py run -m chromium.linux -b "Linux Builder" \\\n' - ' //out/Default content_browsertests\n' - '\n' - ' % tools/mb/mb.py run out/Default content_browsertests\n' - '\n' - ' % tools/mb/mb.py run out/Default content_browsertests -- \\\n' - ' --test-launcher-retry-limit=0' - '\n' - ) - AddCommonOptions(subp) - subp.add_argument('-j', '--jobs', dest='jobs', type=int, - help='Number of jobs to pass to ninja') - subp.add_argument('--no-build', dest='build', default=True, - action='store_false', - help='Do not build, just isolate and run') - subp.add_argument('path', nargs=1, - help=('path to generate build into (or use).' - ' This can be either a regular path or a ' - 'GN-style source-relative path like ' - '//out/Default.')) - subp.add_argument('-s', '--swarmed', action='store_true', - help='Run under swarming') - subp.add_argument('-d', '--dimension', default=[], action='append', nargs=2, - dest='dimensions', metavar='FOO bar', - help='dimension to filter on') - subp.add_argument('target', nargs=1, - help='ninja target to build and run') - subp.add_argument('extra_args', nargs='*', - help=('extra args to pass to the isolate to run. Use ' - '"--" as the first arg if you need to pass ' - 'switches')) - subp.set_defaults(func=self.CmdRun) + subp = subps.add_parser( + 'run', + help='build and run the isolated version of a ' + 'binary', + formatter_class=argparse.RawDescriptionHelpFormatter) + subp.description = ( + 'Build, isolate, and run the given binary with the command line\n' + 'listed in the isolate. You may pass extra arguments after the\n' + 'target; use "--" if the extra arguments need to include switches.' + '\n\n' + 'Examples:\n' + '\n' + ' % tools/mb/mb.py run -m chromium.linux -b "Linux Builder" \\\n' + ' //out/Default content_browsertests\n' + '\n' + ' % tools/mb/mb.py run out/Default content_browsertests\n' + '\n' + ' % tools/mb/mb.py run out/Default content_browsertests -- \\\n' + ' --test-launcher-retry-limit=0' + '\n' + ) + AddCommonOptions(subp) + subp.add_argument('-j', '--jobs', dest='jobs', type=int, + help='Number of jobs to pass to ninja') + subp.add_argument('--no-build', dest='build', default=True, + action='store_false', + help='Do not build, just isolate and run') + subp.add_argument('path', nargs=1, + help=('path to generate build into (or use).' + ' This can be either a regular path or a ' + 'GN-style source-relative path like ' + '//out/Default.')) + subp.add_argument('-s', '--swarmed', action='store_true', + help='Run under swarming') + subp.add_argument('-d', '--dimension', default=[], action='append', + nargs=2, dest='dimensions', metavar='FOO bar', + help='dimension to filter on') + subp.add_argument('target', nargs=1, + help='ninja target to build and run') + subp.add_argument('extra_args', nargs='*', + help=('extra args to pass to the isolate to run. ' + 'Use "--" as the first arg if you need to ' + 'pass switches')) + subp.set_defaults(func=self.CmdRun) - subp = subps.add_parser('validate', - help='validate the config file') - subp.add_argument('-f', '--config-file', metavar='PATH', - default=self.default_config, - help='path to config file (default is %(default)s)') - subp.set_defaults(func=self.CmdValidate) + subp = subps.add_parser('validate', + help='validate the config file') + subp.add_argument('-f', '--config-file', metavar='PATH', + default=self.default_config, + help='path to config file (default is %(default)s)') + subp.set_defaults(func=self.CmdValidate) - subp = subps.add_parser('help', - help='Get help on a subcommand.') - subp.add_argument(nargs='?', action='store', dest='subcommand', - help='The command to get help for.') - subp.set_defaults(func=self.CmdHelp) + subp = subps.add_parser('help', + help='Get help on a subcommand.') + subp.add_argument(nargs='?', action='store', dest='subcommand', + help='The command to get help for.') + subp.set_defaults(func=self.CmdHelp) - self.args = parser.parse_args(argv) + self.args = parser.parse_args(argv) - def DumpInputFiles(self): + def DumpInputFiles(self): - def DumpContentsOfFilePassedTo(arg_name, path): - if path and self.Exists(path): - self.Print("\n# To recreate the file passed to %s:" % arg_name) - self.Print("%% cat > %s < %s < returned %d' % ret) + if out: + self.Print(out, end='') + if err: + self.Print(err, end='', file=sys.stderr) + + return ret + + try: + archive_hashes = json.loads(self.ReadFile(archive_json_path)) + except Exception: + self.Print( + 'Failed to read JSON file "%s"' % + archive_json_path, file=sys.stderr) + return 1 + try: + cas_digest = archive_hashes[target] + except Exception: + self.Print( + 'Cannot find hash for "%s" in "%s", file content: %s' % + (target, archive_json_path, archive_hashes), + file=sys.stderr) + return 1 + + try: + json_dir = self.TempDir() + json_file = self.PathJoin(json_dir, 'task.json') + + cmd = [ + self.PathJoin('tools', 'luci-go', 'swarming'), + 'trigger', + '-realm', + self.args.realm, + '-digest', + cas_digest, + '-server', + swarming_server, + '-tag=purpose:user-debug-mb', + '-relative-cwd', + self.ToSrcRelPath(build_dir), + '-dump-json', + json_file, + ] + dimensions + ['--'] + list(isolate_cmd) + + if self.args.extra_args: + cmd += ['--'] + self.args.extra_args + self.Print('') + ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) + if ret: + return ret + task_json = self.ReadFile(json_file) + task_id = json.loads(task_json)["tasks"][0]['task_id'] + finally: + if json_dir: + self.RemoveDirectory(json_dir) + + cmd = [ + self.PathJoin('tools', 'luci-go', 'swarming'), + 'collect', + '-server', + swarming_server, + '-task-output-stdout=console', + task_id, + ] + ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) + return ret + + def _RunLocallyIsolated(self, build_dir, target): + cmd = [ + self.PathJoin(self.src_dir, 'tools', 'luci-go', self.isolate_exe), + 'run', + '-i', + self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), + ] + if self.args.extra_args: + cmd += ['--'] + self.args.extra_args + ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) + return ret + + def CmdValidate(self, print_ok=True): + errs = [] + + # Read the file to make sure it parses. + self.ReadConfigFile() + + # Build a list of all of the configs referenced by builders. + all_configs = {} + for builder_group in self.builder_groups: + for config in self.builder_groups[builder_group].values(): + if isinstance(config, dict): + for c in config.values(): + all_configs[c] = builder_group + else: + all_configs[config] = builder_group + + # Check that every referenced args file or config actually exists. + for config, loc in all_configs.items(): + if config.startswith('//'): + if not self.Exists(self.ToAbsPath(config)): + errs.append('Unknown args file "%s" referenced from "%s".' % + (config, loc)) + elif not config in self.configs: + errs.append('Unknown config "%s" referenced from "%s".' % + (config, loc)) + + # Check that every actual config is actually referenced. + for config in self.configs: + if not config in all_configs: + errs.append('Unused config "%s".' % config) + + # Figure out the whole list of mixins, and check that every mixin + # listed by a config or another mixin actually exists. + referenced_mixins = set() + for config, mixins in self.configs.items(): + for mixin in mixins: + if not mixin in self.mixins: + errs.append('Unknown mixin "%s" referenced by config "%s".' + % (mixin, config)) + referenced_mixins.add(mixin) + + for mixin in self.mixins: + for sub_mixin in self.mixins[mixin].get('mixins', []): + if not sub_mixin in self.mixins: + errs.append('Unknown mixin "%s" referenced by mixin "%s".' + % (sub_mixin, mixin)) + referenced_mixins.add(sub_mixin) + + # Check that every mixin defined is actually referenced somewhere. + for mixin in self.mixins: + if not mixin in referenced_mixins: + errs.append('Unreferenced mixin "%s".' % mixin) + + if errs: + raise MBErr(('mb config file %s has problems:' % + self.args.config_file) + '\n ' + '\n '.join(errs)) + + if print_ok: + self.Print('mb config file %s looks ok.' % self.args.config_file) + return 0 + + def GetConfig(self): + build_dir = self.args.path[0] + + vals = self.DefaultVals() + if self.args.builder or self.args.builder_group or self.args.config: + vals = self.Lookup() + # Re-run gn gen in order to ensure the config is consistent with + # the build dir. + self.RunGNGen(vals) + return vals + + toolchain_path = self.PathJoin(self.ToAbsPath(build_dir), + 'toolchain.ninja') + if not self.Exists(toolchain_path): + self.Print('Must either specify a path to an existing GN build ' + 'dir or pass in a -m/-b pair or a -c flag to specify ' + 'the configuration') + return {} + + vals['gn_args'] = self.GNArgsFromDir(build_dir) + return vals + + def GNArgsFromDir(self, build_dir): + args_contents = "" + gn_args_path = self.PathJoin(self.ToAbsPath(build_dir), 'args.gn') + if self.Exists(gn_args_path): + args_contents = self.ReadFile(gn_args_path) + gn_args = [] + for l in args_contents.splitlines(): + fields = l.split(' ') + name = fields[0] + val = ' '.join(fields[2:]) + gn_args.append('%s=%s' % (name, val)) + + return ' '.join(gn_args) + + def Lookup(self): + self.ReadConfigFile() + config = self.ConfigFromArgs() + if config.startswith('//'): + if not self.Exists(self.ToAbsPath(config)): + raise MBErr('args file "%s" not found' % config) + vals = self.DefaultVals() + vals['args_file'] = config + else: + if not config in self.configs: + raise MBErr('Config "%s" not found in %s' % + (config, self.args.config_file)) + vals = self.FlattenConfig(config) + return vals + + def ReadConfigFile(self): + if not self.Exists(self.args.config_file): + raise MBErr('config file not found at %s' % self.args.config_file) + + try: + contents = ast.literal_eval(self.ReadFile(self.args.config_file)) + except SyntaxError as e: + raise MBErr('Failed to parse config file "%s": %s' % + (self.args.config_file, e)) + + self.configs = contents['configs'] + self.builder_groups = contents['builder_groups'] + self.mixins = contents['mixins'] + + def ReadIsolateMap(self): + isolate_map = self.args.isolate_map_file + if not self.Exists(isolate_map): + raise MBErr('isolate map file not found at %s' % isolate_map) + try: + return ast.literal_eval(self.ReadFile(isolate_map)) + except SyntaxError as e: + raise MBErr( + 'Failed to parse isolate map file "%s": %s' % (isolate_map, e)) + + def ConfigFromArgs(self): + if self.args.config: + if self.args.builder_group or self.args.builder: + raise MBErr('Can not specific both -c/--config and ' + '-m/--builder-group or -b/--builder') + + return self.args.config + + if not self.args.builder_group or not self.args.builder: + raise MBErr('Must specify either -c/--config or ' + '(-m/--builder-group and -b/--builder)') + + if not self.args.builder_group in self.builder_groups: + raise MBErr('Master name "%s" not found in "%s"' % + (self.args.builder_group, self.args.config_file)) + + if (not self.args.builder in + self.builder_groups[self.args.builder_group]): + raise MBErr( + 'Builder name "%s" not found under builder_groups[%s] in "%s"' + % (self.args.builder, self.args.builder_group, + self.args.config_file)) + + config = ( + self.builder_groups[self.args.builder_group][self.args.builder] + ) if isinstance(config, dict): - args = {k: self.FlattenConfig(v)['gn_args'] - for k, v in config.items()} - elif config.startswith('//'): - args = config - else: - args = self.FlattenConfig(config)['gn_args'] - if 'error' in args: - continue - - obj[builder_group][builder] = args - - # Dump object and trim trailing whitespace. - s = '\n'.join(l.rstrip() for l in - json.dumps(obj, sort_keys=True, indent=2).splitlines()) - self.Print(s) - return 0 - - def CmdGen(self): - vals = self.Lookup() - return self.RunGNGen(vals) - - def CmdHelp(self): - if self.args.subcommand: - self.ParseArgs([self.args.subcommand, '--help']) - else: - self.ParseArgs(['--help']) - - def CmdIsolate(self): - vals = self.GetConfig() - if not vals: - return 1 - return self.RunGNIsolate(vals) - - def CmdLookup(self): - vals = self.Lookup() - gn_args = self.GNArgs(vals) - if self.args.quiet: - self.Print(gn_args, end='') - else: - cmd = self.GNCmd('gen', '_path_') - self.Print('\nWriting """\\\n%s""" to _path_/args.gn.\n' % gn_args) - env = None - - self.PrintCmd(cmd, env) - return 0 - - def CmdRun(self): - vals = self.GetConfig() - if not vals: - return 1 - - build_dir = self.args.path[0] - target = self.args.target[0] - - if self.args.build: - ret = self.Build(target) - if ret: - return ret - ret = self.RunGNIsolate(vals) - if ret: - return ret - - if self.args.swarmed: - cmd, _ = self.GetSwarmingCommand(self.args.target[0], vals) - return self._RunUnderSwarming(build_dir, target, cmd) - else: - return self._RunLocallyIsolated(build_dir, target) - - def _RunUnderSwarming(self, build_dir, target, isolate_cmd): - cas_instance = 'chromium-swarm' - swarming_server = 'chromium-swarm.appspot.com' - # TODO(dpranke): Look up the information for the target in - # the //testing/buildbot.json file, if possible, so that we - # can determine the isolate target, command line, and additional - # swarming parameters, if possible. - # - # TODO(dpranke): Also, add support for sharding and merging results. - dimensions = [] - for k, v in self.args.dimensions: - dimensions += ['-d', '%s=%s' % (k, v)] - - archive_json_path = self.ToSrcRelPath( - '%s/%s.archive.json' % (build_dir, target)) - cmd = [ - self.PathJoin(self.src_dir, 'tools', 'luci-go', self.isolate_exe), - 'archive', - '-i', - self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), - '-cas-instance', - cas_instance, - '-dump-json', - archive_json_path, - ] - - # Talking to the isolateserver may fail because we're not logged in. - # We trap the command explicitly and rewrite the error output so that - # the error message is actually correct for a Chromium check out. - self.PrintCmd(cmd, env=None) - ret, out, err = self.Run(cmd, force_verbose=False) - if ret: - self.Print(' -> returned %d' % ret) - if out: - self.Print(out, end='') - if err: - self.Print(err, end='', file=sys.stderr) - - return ret - - try: - archive_hashes = json.loads(self.ReadFile(archive_json_path)) - except Exception: - self.Print( - 'Failed to read JSON file "%s"' % archive_json_path, file=sys.stderr) - return 1 - try: - cas_digest = archive_hashes[target] - except Exception: - self.Print( - 'Cannot find hash for "%s" in "%s", file content: %s' % - (target, archive_json_path, archive_hashes), - file=sys.stderr) - return 1 - - try: - json_dir = self.TempDir() - json_file = self.PathJoin(json_dir, 'task.json') - - cmd = [ - self.PathJoin('tools', 'luci-go', 'swarming'), - 'trigger', - '-realm', - self.args.realm, - '-digest', - cas_digest, - '-server', - swarming_server, - '-tag=purpose:user-debug-mb', - '-relative-cwd', - self.ToSrcRelPath(build_dir), - '-dump-json', - json_file, - ] + dimensions + ['--'] + list(isolate_cmd) - - if self.args.extra_args: - cmd += ['--'] + self.args.extra_args - self.Print('') - ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) - if ret: - return ret - task_json = self.ReadFile(json_file) - task_id = json.loads(task_json)["tasks"][0]['task_id'] - finally: - if json_dir: - self.RemoveDirectory(json_dir) - - cmd = [ - self.PathJoin('tools', 'luci-go', 'swarming'), - 'collect', - '-server', - swarming_server, - '-task-output-stdout=console', - task_id, - ] - ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) - return ret - - def _RunLocallyIsolated(self, build_dir, target): - cmd = [ - self.PathJoin(self.src_dir, 'tools', 'luci-go', self.isolate_exe), - 'run', - '-i', - self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), - ] - if self.args.extra_args: - cmd += ['--'] + self.args.extra_args - ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) - return ret - - def CmdValidate(self, print_ok=True): - errs = [] - - # Read the file to make sure it parses. - self.ReadConfigFile() - - # Build a list of all of the configs referenced by builders. - all_configs = {} - for builder_group in self.builder_groups: - for config in self.builder_groups[builder_group].values(): - if isinstance(config, dict): - for c in config.values(): - all_configs[c] = builder_group - else: - all_configs[config] = builder_group - - # Check that every referenced args file or config actually exists. - for config, loc in all_configs.items(): - if config.startswith('//'): - if not self.Exists(self.ToAbsPath(config)): - errs.append('Unknown args file "%s" referenced from "%s".' % - (config, loc)) - elif not config in self.configs: - errs.append('Unknown config "%s" referenced from "%s".' % - (config, loc)) - - # Check that every actual config is actually referenced. - for config in self.configs: - if not config in all_configs: - errs.append('Unused config "%s".' % config) - - # Figure out the whole list of mixins, and check that every mixin - # listed by a config or another mixin actually exists. - referenced_mixins = set() - for config, mixins in self.configs.items(): - for mixin in mixins: - if not mixin in self.mixins: - errs.append('Unknown mixin "%s" referenced by config "%s".' % - (mixin, config)) - referenced_mixins.add(mixin) - - for mixin in self.mixins: - for sub_mixin in self.mixins[mixin].get('mixins', []): - if not sub_mixin in self.mixins: - errs.append('Unknown mixin "%s" referenced by mixin "%s".' % - (sub_mixin, mixin)) - referenced_mixins.add(sub_mixin) - - # Check that every mixin defined is actually referenced somewhere. - for mixin in self.mixins: - if not mixin in referenced_mixins: - errs.append('Unreferenced mixin "%s".' % mixin) - - if errs: - raise MBErr(('mb config file %s has problems:' % self.args.config_file) + - '\n ' + '\n '.join(errs)) - - if print_ok: - self.Print('mb config file %s looks ok.' % self.args.config_file) - return 0 - - def GetConfig(self): - build_dir = self.args.path[0] - - vals = self.DefaultVals() - if self.args.builder or self.args.builder_group or self.args.config: - vals = self.Lookup() - # Re-run gn gen in order to ensure the config is consistent with the - # build dir. - self.RunGNGen(vals) - return vals - - toolchain_path = self.PathJoin(self.ToAbsPath(build_dir), - 'toolchain.ninja') - if not self.Exists(toolchain_path): - self.Print('Must either specify a path to an existing GN build dir ' - 'or pass in a -m/-b pair or a -c flag to specify the ' - 'configuration') - return {} - - vals['gn_args'] = self.GNArgsFromDir(build_dir) - return vals - - def GNArgsFromDir(self, build_dir): - args_contents = "" - gn_args_path = self.PathJoin(self.ToAbsPath(build_dir), 'args.gn') - if self.Exists(gn_args_path): - args_contents = self.ReadFile(gn_args_path) - gn_args = [] - for l in args_contents.splitlines(): - fields = l.split(' ') - name = fields[0] - val = ' '.join(fields[2:]) - gn_args.append('%s=%s' % (name, val)) - - return ' '.join(gn_args) - - def Lookup(self): - self.ReadConfigFile() - config = self.ConfigFromArgs() - if config.startswith('//'): - if not self.Exists(self.ToAbsPath(config)): - raise MBErr('args file "%s" not found' % config) - vals = self.DefaultVals() - vals['args_file'] = config - else: - if not config in self.configs: - raise MBErr('Config "%s" not found in %s' % - (config, self.args.config_file)) - vals = self.FlattenConfig(config) - return vals - - def ReadConfigFile(self): - if not self.Exists(self.args.config_file): - raise MBErr('config file not found at %s' % self.args.config_file) - - try: - contents = ast.literal_eval(self.ReadFile(self.args.config_file)) - except SyntaxError as e: - raise MBErr('Failed to parse config file "%s": %s' % - (self.args.config_file, e)) - - self.configs = contents['configs'] - self.builder_groups = contents['builder_groups'] - self.mixins = contents['mixins'] - - def ReadIsolateMap(self): - isolate_map = self.args.isolate_map_file - if not self.Exists(isolate_map): - raise MBErr('isolate map file not found at %s' % isolate_map) - try: - return ast.literal_eval(self.ReadFile(isolate_map)) - except SyntaxError as e: - raise MBErr( - 'Failed to parse isolate map file "%s": %s' % (isolate_map, e)) - - def ConfigFromArgs(self): - if self.args.config: - if self.args.builder_group or self.args.builder: - raise MBErr('Can not specific both -c/--config and -m/--builder-group ' - 'or -b/--builder') - - return self.args.config - - if not self.args.builder_group or not self.args.builder: - raise MBErr('Must specify either -c/--config or ' - '(-m/--builder-group and -b/--builder)') - - if not self.args.builder_group in self.builder_groups: - raise MBErr('Master name "%s" not found in "%s"' % - (self.args.builder_group, self.args.config_file)) - - if not self.args.builder in self.builder_groups[self.args.builder_group]: - raise MBErr( - 'Builder name "%s" not found under builder_groups[%s] in "%s"' % - (self.args.builder, self.args.builder_group, self.args.config_file)) - - config = self.builder_groups[self.args.builder_group][self.args.builder] - if isinstance(config, dict): - if self.args.phase is None: - raise MBErr('Must specify a build --phase for %s on %s' % - (self.args.builder, self.args.builder_group)) - phase = str(self.args.phase) - if phase not in config: - raise MBErr('Phase %s doesn\'t exist for %s on %s' % - (phase, self.args.builder, self.args.builder_group)) - return config[phase] - - if self.args.phase is not None: - raise MBErr('Must not specify a build --phase for %s on %s' % - (self.args.builder, self.args.builder_group)) - return config - - def FlattenConfig(self, config): - mixins = self.configs[config] - vals = self.DefaultVals() - - visited = [] - self.FlattenMixins(mixins, vals, visited) - return vals - - def DefaultVals(self): - return { - 'args_file': '', - 'cros_passthrough': False, - 'gn_args': '', - } - - def FlattenMixins(self, mixins, vals, visited): - for m in mixins: - if m not in self.mixins: - raise MBErr('Unknown mixin "%s"' % m) - - visited.append(m) - - mixin_vals = self.mixins[m] - - if 'cros_passthrough' in mixin_vals: - vals['cros_passthrough'] = mixin_vals['cros_passthrough'] - if 'gn_args' in mixin_vals: - if vals['gn_args']: - vals['gn_args'] += ' ' + mixin_vals['gn_args'] - else: - vals['gn_args'] = mixin_vals['gn_args'] - - if 'mixins' in mixin_vals: - self.FlattenMixins(mixin_vals['mixins'], vals, visited) - return vals - - def RunGNGen(self, vals): - build_dir = self.args.path[0] - - cmd = self.GNCmd('gen', build_dir, '--check') - gn_args = self.GNArgs(vals) - - # Since GN hasn't run yet, the build directory may not even exist. - self.MaybeMakeDirectory(self.ToAbsPath(build_dir)) - - gn_args_path = self.ToAbsPath(build_dir, 'args.gn') - self.WriteFile(gn_args_path, gn_args, force_verbose=True) - - swarming_targets = [] - if getattr(self.args, 'swarming_targets_file', None): - # We need GN to generate the list of runtime dependencies for - # the compile targets listed (one per line) in the file so - # we can run them via swarming. We use gn_isolate_map.pyl to convert - # the compile targets to the matching GN labels. - path = self.args.swarming_targets_file - if not self.Exists(path): - self.WriteFailureAndRaise('"%s" does not exist' % path, - output_path=None) - contents = self.ReadFile(path) - swarming_targets = set(contents.splitlines()) - - isolate_map = self.ReadIsolateMap() - err, labels = self.MapTargetsToLabels(isolate_map, swarming_targets) - if err: - raise MBErr(err) - - gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps') - self.WriteFile(gn_runtime_deps_path, '\n'.join(labels) + '\n') - cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path) - - ret, output, _ = self.Run(cmd) - if ret: - if self.args.json_output: - # write errors to json.output - self.WriteJSON({'output': output}, self.args.json_output) - # If `gn gen` failed, we should exit early rather than trying to - # generate isolates. Run() will have already logged any error output. - self.Print('GN gen failed: %d' % ret) - return ret - - android = 'target_os="android"' in vals['gn_args'] - for target in swarming_targets: - if android: - # Android targets may be either android_apk or executable. The former - # will result in runtime_deps associated with the stamp file, while the - # latter will result in runtime_deps associated with the executable. - label = isolate_map[target]['label'] - runtime_deps_targets = [ - target + '.runtime_deps', - 'obj/%s.stamp.runtime_deps' % label.replace(':', '/')] - elif isolate_map[target]['type'] == 'gpu_browser_test': - if self.platform == 'win32': - runtime_deps_targets = ['browser_tests.exe.runtime_deps'] - else: - runtime_deps_targets = ['browser_tests.runtime_deps'] - elif isolate_map[target]['type'] == 'script': - label = isolate_map[target]['label'].split(':')[1] - runtime_deps_targets = [ - '%s.runtime_deps' % label] - if self.platform == 'win32': - runtime_deps_targets += [ label + '.exe.runtime_deps' ] - else: - runtime_deps_targets += [ label + '.runtime_deps' ] - elif self.platform == 'win32': - runtime_deps_targets = [target + '.exe.runtime_deps'] - else: - runtime_deps_targets = [target + '.runtime_deps'] - - for r in runtime_deps_targets: - runtime_deps_path = self.ToAbsPath(build_dir, r) - if self.Exists(runtime_deps_path): - break - else: - raise MBErr('did not generate any of %s' % - ', '.join(runtime_deps_targets)) - - command, extra_files = self.GetSwarmingCommand(target, vals) - - runtime_deps = self.ReadFile(runtime_deps_path).splitlines() - - self.WriteIsolateFiles(build_dir, command, target, runtime_deps, - extra_files) - - return 0 - - def RunGNIsolate(self, vals): - target = self.args.target[0] - isolate_map = self.ReadIsolateMap() - err, labels = self.MapTargetsToLabels(isolate_map, [target]) - if err: - raise MBErr(err) - label = labels[0] - - build_dir = self.args.path[0] - command, extra_files = self.GetSwarmingCommand(target, vals) - - cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps') - ret, out, _ = self.Call(cmd) - if ret: - if out: - self.Print(out) - return ret - - runtime_deps = out.splitlines() - - self.WriteIsolateFiles(build_dir, command, target, runtime_deps, - extra_files) - - ret, _, _ = self.Run([ - self.PathJoin(self.src_dir, 'tools', 'luci-go', self.isolate_exe), - 'check', - '-i', - self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target))], - buffer_output=False) - - return ret - - def WriteIsolateFiles(self, build_dir, command, target, runtime_deps, - extra_files): - isolate_path = self.ToAbsPath(build_dir, target + '.isolate') - self.WriteFile(isolate_path, - pprint.pformat({ - 'variables': { - 'command': command, - 'files': sorted(runtime_deps + extra_files), + if self.args.phase is None: + raise MBErr('Must specify a build --phase for %s on %s' % + (self.args.builder, self.args.builder_group)) + phase = str(self.args.phase) + if phase not in config: + raise MBErr('Phase %s doesn\'t exist for %s on %s' % + (phase, self.args.builder, + self.args.builder_group)) + return config[phase] + + if self.args.phase is not None: + raise MBErr('Must not specify a build --phase for %s on %s' % + (self.args.builder, self.args.builder_group)) + return config + + def FlattenConfig(self, config): + mixins = self.configs[config] + vals = self.DefaultVals() + + visited = [] + self.FlattenMixins(mixins, vals, visited) + return vals + + def DefaultVals(self): + return { + 'args_file': '', + 'cros_passthrough': False, + 'gn_args': '', } - }) + '\n') - self.WriteJSON( - { - 'args': [ - '--isolated', - self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)), - '--isolate', - self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), - ], - 'dir': self.src_dir, - 'version': 1, - }, - isolate_path + 'd.gen.json', - ) + def FlattenMixins(self, mixins, vals, visited): + for m in mixins: + if m not in self.mixins: + raise MBErr('Unknown mixin "%s"' % m) - def MapTargetsToLabels(self, isolate_map, targets): - labels = [] - err = '' + visited.append(m) - def StripTestSuffixes(target): - for suffix in ('_apk_run', '_apk', '_run'): - if target.endswith(suffix): - return target[:-len(suffix)], suffix - return None, None + mixin_vals = self.mixins[m] - for target in targets: - if target == 'all': - labels.append(target) - elif target.startswith('//'): - labels.append(target) - else: - if target in isolate_map: - stripped_target, suffix = target, '' - else: - stripped_target, suffix = StripTestSuffixes(target) - if stripped_target in isolate_map: - if isolate_map[stripped_target]['type'] == 'unknown': - err += ('test target "%s" type is unknown\n' % target) - else: - labels.append(isolate_map[stripped_target]['label'] + suffix) - else: - err += ('target "%s" not found in ' - '//testing/buildbot/gn_isolate_map.pyl\n' % target) + if 'cros_passthrough' in mixin_vals: + vals['cros_passthrough'] = mixin_vals['cros_passthrough'] + if 'gn_args' in mixin_vals: + if vals['gn_args']: + vals['gn_args'] += ' ' + mixin_vals['gn_args'] + else: + vals['gn_args'] = mixin_vals['gn_args'] - return err, labels + if 'mixins' in mixin_vals: + self.FlattenMixins(mixin_vals['mixins'], vals, visited) + return vals - def GNCmd(self, subcommand, path, *args): - if self.platform.startswith('linux'): - subdir, exe = 'linux64', 'gn' - elif self.platform == 'darwin': - subdir, exe = 'mac', 'gn' - else: - subdir, exe = 'win', 'gn.exe' + def RunGNGen(self, vals): + build_dir = self.args.path[0] - gn_path = self.PathJoin(self.src_dir, 'buildtools', subdir, exe) - return [gn_path, subcommand, path] + list(args) + cmd = self.GNCmd('gen', build_dir, '--check') + gn_args = self.GNArgs(vals) + # Since GN hasn't run yet, the build directory may not even exist. + self.MaybeMakeDirectory(self.ToAbsPath(build_dir)) - def GNArgs(self, vals): - if vals['cros_passthrough']: - if not 'GN_ARGS' in os.environ: - raise MBErr('MB is expecting GN_ARGS to be in the environment') - gn_args = os.environ['GN_ARGS'] - if not re.search('target_os.*=.*"chromeos"', gn_args): - raise MBErr('GN_ARGS is missing target_os = "chromeos": (GN_ARGS=%s)' % - gn_args) - else: - gn_args = vals['gn_args'] + gn_args_path = self.ToAbsPath(build_dir, 'args.gn') + self.WriteFile(gn_args_path, gn_args, force_verbose=True) - if self.args.goma_dir: - gn_args += ' goma_dir="%s"' % self.args.goma_dir + swarming_targets = [] + if getattr(self.args, 'swarming_targets_file', None): + # We need GN to generate the list of runtime dependencies for + # the compile targets listed (one per line) in the file so + # we can run them via swarming. We use gn_isolate_map.pyl to + # convert the compile targets to the matching GN labels. + path = self.args.swarming_targets_file + if not self.Exists(path): + self.WriteFailureAndRaise('"%s" does not exist' % path, + output_path=None) + contents = self.ReadFile(path) + swarming_targets = set(contents.splitlines()) - android_version_code = self.args.android_version_code - if android_version_code: - gn_args += ' android_default_version_code="%s"' % android_version_code + isolate_map = self.ReadIsolateMap() + err, labels = self.MapTargetsToLabels( + isolate_map, swarming_targets) + if err: + raise MBErr(err) - android_version_name = self.args.android_version_name - if android_version_name: - gn_args += ' android_default_version_name="%s"' % android_version_name + gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps') + self.WriteFile(gn_runtime_deps_path, '\n'.join(labels) + '\n') + cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path) - # Canonicalize the arg string into a sorted, newline-separated list - # of key-value pairs, and de-dup the keys if need be so that only - # the last instance of each arg is listed. - gn_args = gn_helpers.ToGNString(gn_helpers.FromGNArgs(gn_args)) + ret, output, _ = self.Run(cmd) + if ret: + if self.args.json_output: + # write errors to json.output + self.WriteJSON({'output': output}, self.args.json_output) + # If `gn gen` failed, we should exit early rather than trying to + # generate isolates. Run() will have already logged any error + # output. + self.Print('GN gen failed: %d' % ret) + return ret - args_file = vals.get('args_file', None) - if args_file: - gn_args = ('import("%s")\n' % vals['args_file']) + gn_args - return gn_args + android = 'target_os="android"' in vals['gn_args'] + for target in swarming_targets: + if android: + # Android targets may be either android_apk or executable. The + # former will result in runtime_deps associated with the stamp + # file, while the latter will result in runtime_deps associated + # with the executable. + label = isolate_map[target]['label'] + runtime_deps_targets = [ + target + '.runtime_deps', + 'obj/%s.stamp.runtime_deps' % label.replace(':', '/')] + elif isolate_map[target]['type'] == 'gpu_browser_test': + if self.platform == 'win32': + runtime_deps_targets = ['browser_tests.exe.runtime_deps'] + else: + runtime_deps_targets = ['browser_tests.runtime_deps'] + elif isolate_map[target]['type'] == 'script': + label = isolate_map[target]['label'].split(':')[1] + runtime_deps_targets = [ + '%s.runtime_deps' % label] + if self.platform == 'win32': + runtime_deps_targets += [label + '.exe.runtime_deps'] + else: + runtime_deps_targets += [label + '.runtime_deps'] + elif self.platform == 'win32': + runtime_deps_targets = [target + '.exe.runtime_deps'] + else: + runtime_deps_targets = [target + '.runtime_deps'] - def GetSwarmingCommand(self, target, vals): - isolate_map = self.ReadIsolateMap() - test_type = isolate_map[target]['type'] + for r in runtime_deps_targets: + runtime_deps_path = self.ToAbsPath(build_dir, r) + if self.Exists(runtime_deps_path): + break + else: + raise MBErr('did not generate any of %s' % + ', '.join(runtime_deps_targets)) - is_android = 'target_os="android"' in vals['gn_args'] - is_linux = self.platform.startswith('linux') and not is_android + command, extra_files = self.GetSwarmingCommand(target, vals) - if test_type == 'nontest': - self.WriteFailureAndRaise('We should not be isolating %s.' % target, - output_path=None) - if test_type not in ('console_test_launcher', 'windowed_test_launcher', - 'non_parallel_console_test_launcher', 'raw', - 'additional_compile_target', 'junit_test', 'script'): - self.WriteFailureAndRaise('No command line for %s found (test type %s).' - % (target, test_type), output_path=None) + runtime_deps = self.ReadFile(runtime_deps_path).splitlines() - cmdline = [] - extra_files = [ - '../../.vpython', - '../../testing/test_env.py', - ] + self.WriteIsolateFiles(build_dir, command, target, runtime_deps, + extra_files) - must_retry = False - if test_type == 'script': - cmdline += ['../../' + self.ToSrcRelPath(isolate_map[target]['script'])] - elif is_android: - cmdline += ['../../build/android/test_wrapper/logdog_wrapper.py', - '--target', target, - '--logdog-bin-cmd', '../../bin/logdog_butler', - '--logcat-output-file', '${ISOLATED_OUTDIR}/logcats', - '--store-tombstones'] - else: - if test_type == 'raw': - cmdline.append('../../tools_webrtc/flags_compatibility.py') - extra_files.append('../../tools_webrtc/flags_compatibility.py') + return 0 - if isolate_map[target].get('use_webcam', False): - cmdline.append('../../tools_webrtc/ensure_webcam_is_running.py') - extra_files.append('../../tools_webrtc/ensure_webcam_is_running.py') + def RunGNIsolate(self, vals): + target = self.args.target[0] + isolate_map = self.ReadIsolateMap() + err, labels = self.MapTargetsToLabels(isolate_map, [target]) + if err: + raise MBErr(err) + label = labels[0] - # is_linux uses use_ozone and x11 by default. - use_x11 = is_linux + build_dir = self.args.path[0] + command, extra_files = self.GetSwarmingCommand(target, vals) - xvfb = use_x11 and test_type == 'windowed_test_launcher' - if xvfb: - cmdline.append('../../testing/xvfb.py') - extra_files.append('../../testing/xvfb.py') - else: - cmdline.append('../../testing/test_env.py') + cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps') + ret, out, _ = self.Call(cmd) + if ret: + if out: + self.Print(out) + return ret - if test_type != 'raw': - extra_files += [ - '../../third_party/gtest-parallel/gtest-parallel', - '../../third_party/gtest-parallel/gtest_parallel.py', - '../../tools_webrtc/gtest-parallel-wrapper.py', - ] - sep = '\\' if self.platform == 'win32' else '/' - output_dir = '${ISOLATED_OUTDIR}' + sep + 'test_logs' - timeout = isolate_map[target].get('timeout', 900) - cmdline += [ - '../../tools_webrtc/gtest-parallel-wrapper.py', - '--output_dir=%s' % output_dir, - '--gtest_color=no', - # We tell gtest-parallel to interrupt the test after 900 seconds, - # so it can exit cleanly and report results, instead of being - # interrupted by swarming and not reporting anything. - '--timeout=%s' % timeout, - ] - if test_type == 'non_parallel_console_test_launcher': - # Still use the gtest-parallel-wrapper.py script since we need it to - # run tests on swarming, but don't execute tests in parallel. - cmdline.append('--workers=1') - must_retry = True + runtime_deps = out.splitlines() - asan = 'is_asan=true' in vals['gn_args'] - lsan = 'is_lsan=true' in vals['gn_args'] - msan = 'is_msan=true' in vals['gn_args'] - tsan = 'is_tsan=true' in vals['gn_args'] - sanitizer = asan or lsan or msan or tsan - if must_retry and not sanitizer: - # Retry would hide most sanitizers detections. - cmdline.append('--retry_failed=3') + self.WriteIsolateFiles(build_dir, command, target, runtime_deps, + extra_files) - executable_prefix = '.\\' if self.platform == 'win32' else './' - executable_suffix = '.exe' if self.platform == 'win32' else '' - executable = executable_prefix + target + executable_suffix + ret, _, _ = self.Run([ + self.PathJoin(self.src_dir, 'tools', 'luci-go', self.isolate_exe), + 'check', + '-i', + self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target))], + buffer_output=False) - cmdline.append(executable) - - cmdline.extend([ - '--asan=%d' % asan, - '--lsan=%d' % lsan, - '--msan=%d' % msan, - '--tsan=%d' % tsan, - ]) - - cmdline += isolate_map[target].get('args', []) - - return cmdline, extra_files - - def ToAbsPath(self, build_path, *comps): - return self.PathJoin(self.src_dir, - self.ToSrcRelPath(build_path), - *comps) - - def ToSrcRelPath(self, path): - """Returns a relative path from the top of the repo.""" - if path.startswith('//'): - return path[2:].replace('/', self.sep) - return self.RelPath(path, self.src_dir) - - def RunGNAnalyze(self, vals): - # Analyze runs before 'gn gen' now, so we need to run gn gen - # in order to ensure that we have a build directory. - ret = self.RunGNGen(vals) - if ret: - return ret - - build_path = self.args.path[0] - input_path = self.args.input_path[0] - gn_input_path = input_path + '.gn' - output_path = self.args.output_path[0] - gn_output_path = output_path + '.gn' - - inp = self.ReadInputJSON(['files', 'test_targets', - 'additional_compile_targets']) - if self.args.verbose: - self.Print() - self.Print('analyze input:') - self.PrintJSON(inp) - self.Print() - - - # This shouldn't normally happen, but could due to unusual race conditions, - # like a try job that gets scheduled before a patch lands but runs after - # the patch has landed. - if not inp['files']: - self.Print('Warning: No files modified in patch, bailing out early.') - self.WriteJSON({ - 'status': 'No dependency', - 'compile_targets': [], - 'test_targets': [], - }, output_path) - return 0 - - gn_inp = {} - gn_inp['files'] = ['//' + f for f in inp['files'] if not f.startswith('//')] - - isolate_map = self.ReadIsolateMap() - err, gn_inp['additional_compile_targets'] = self.MapTargetsToLabels( - isolate_map, inp['additional_compile_targets']) - if err: - raise MBErr(err) - - err, gn_inp['test_targets'] = self.MapTargetsToLabels( - isolate_map, inp['test_targets']) - if err: - raise MBErr(err) - labels_to_targets = {} - for i, label in enumerate(gn_inp['test_targets']): - labels_to_targets[label] = inp['test_targets'][i] - - try: - self.WriteJSON(gn_inp, gn_input_path) - cmd = self.GNCmd('analyze', build_path, gn_input_path, gn_output_path) - ret, output, _ = self.Run(cmd, force_verbose=True) - if ret: - if self.args.json_output: - # write errors to json.output - self.WriteJSON({'output': output}, self.args.json_output) return ret - gn_outp_str = self.ReadFile(gn_output_path) - try: - gn_outp = json.loads(gn_outp_str) - except Exception as e: - self.Print("Failed to parse the JSON string GN returned: %s\n%s" - % (repr(gn_outp_str), str(e))) - raise + def WriteIsolateFiles(self, build_dir, command, target, runtime_deps, + extra_files): + isolate_path = self.ToAbsPath(build_dir, target + '.isolate') + self.WriteFile(isolate_path, + pprint.pformat({ + 'variables': { + 'command': command, + 'files': sorted(runtime_deps + extra_files), + } + }) + '\n') - outp = {} - if 'status' in gn_outp: - outp['status'] = gn_outp['status'] - if 'error' in gn_outp: - outp['error'] = gn_outp['error'] - if 'invalid_targets' in gn_outp: - outp['invalid_targets'] = gn_outp['invalid_targets'] - if 'compile_targets' in gn_outp: - if 'all' in gn_outp['compile_targets']: - outp['compile_targets'] = ['all'] + self.WriteJSON( + { + 'args': [ + '--isolated', + self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)), + '--isolate', + self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), + ], + 'dir': self.src_dir, + 'version': 1, + }, + isolate_path + 'd.gen.json', + ) + + def MapTargetsToLabels(self, isolate_map, targets): + labels = [] + err = '' + + def StripTestSuffixes(target): + for suffix in ('_apk_run', '_apk', '_run'): + if target.endswith(suffix): + return target[:-len(suffix)], suffix + return None, None + + for target in targets: + if target == 'all': + labels.append(target) + elif target.startswith('//'): + labels.append(target) + else: + if target in isolate_map: + stripped_target, suffix = target, '' + else: + stripped_target, suffix = StripTestSuffixes(target) + if stripped_target in isolate_map: + if isolate_map[stripped_target]['type'] == 'unknown': + err += ('test target "%s" type is unknown\n' % target) + else: + labels.append( + isolate_map[stripped_target]['label'] + suffix) + else: + err += ('target "%s" not found in ' + '//testing/buildbot/gn_isolate_map.pyl\n' % target) + + return err, labels + + def GNCmd(self, subcommand, path, *args): + if self.platform.startswith('linux'): + subdir, exe = 'linux64', 'gn' + elif self.platform == 'darwin': + subdir, exe = 'mac', 'gn' else: - outp['compile_targets'] = [ - label.replace('//', '') for label in gn_outp['compile_targets']] - if 'test_targets' in gn_outp: - outp['test_targets'] = [ - labels_to_targets[label] for label in gn_outp['test_targets']] + subdir, exe = 'win', 'gn.exe' - if self.args.verbose: - self.Print() - self.Print('analyze output:') - self.PrintJSON(outp) - self.Print() + gn_path = self.PathJoin(self.src_dir, 'buildtools', subdir, exe) + return [gn_path, subcommand, path] + list(args) - self.WriteJSON(outp, output_path) + def GNArgs(self, vals): + if vals['cros_passthrough']: + if not 'GN_ARGS' in os.environ: + raise MBErr('MB is expecting GN_ARGS to be in the environment') + gn_args = os.environ['GN_ARGS'] + if not re.search('target_os.*=.*"chromeos"', gn_args): + raise MBErr('GN_ARGS is missing target_os = "chromeos": ' + '(GN_ARGS=%s)' % gn_args) + else: + gn_args = vals['gn_args'] - finally: - if self.Exists(gn_input_path): - self.RemoveFile(gn_input_path) - if self.Exists(gn_output_path): - self.RemoveFile(gn_output_path) + if self.args.goma_dir: + gn_args += ' goma_dir="%s"' % self.args.goma_dir - return 0 + android_version_code = self.args.android_version_code + if android_version_code: + gn_args += (' android_default_version_code="%s"' % + android_version_code) - def ReadInputJSON(self, required_keys): - path = self.args.input_path[0] - output_path = self.args.output_path[0] - if not self.Exists(path): - self.WriteFailureAndRaise('"%s" does not exist' % path, output_path) + android_version_name = self.args.android_version_name + if android_version_name: + gn_args += (' android_default_version_name="%s"' % + android_version_name) - try: - inp = json.loads(self.ReadFile(path)) - except Exception as e: - self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' % - (path, e), output_path) + # Canonicalize the arg string into a sorted, newline-separated list + # of key-value pairs, and de-dup the keys if need be so that only + # the last instance of each arg is listed. + gn_args = gn_helpers.ToGNString(gn_helpers.FromGNArgs(gn_args)) - for k in required_keys: - if not k in inp: - self.WriteFailureAndRaise('input file is missing a "%s" key' % k, - output_path) + args_file = vals.get('args_file', None) + if args_file: + gn_args = ('import("%s")\n' % vals['args_file']) + gn_args + return gn_args - return inp + def GetSwarmingCommand(self, target, vals): + isolate_map = self.ReadIsolateMap() + test_type = isolate_map[target]['type'] - def WriteFailureAndRaise(self, msg, output_path): - if output_path: - self.WriteJSON({'error': msg}, output_path, force_verbose=True) - raise MBErr(msg) + is_android = 'target_os="android"' in vals['gn_args'] + is_linux = self.platform.startswith('linux') and not is_android - def WriteJSON(self, obj, path, force_verbose=False): - try: - self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n', - force_verbose=force_verbose) - except Exception as e: - raise MBErr('Error %s writing to the output path "%s"' % - (e, path)) + if test_type == 'nontest': + self.WriteFailureAndRaise('We should not be isolating %s.' % + target, output_path=None) + if test_type not in ('console_test_launcher', 'windowed_test_launcher', + 'non_parallel_console_test_launcher', 'raw', + 'additional_compile_target', 'junit_test', + 'script'): + self.WriteFailureAndRaise('No command line for ' + '%s found (test type %s).' + % (target, test_type), output_path=None) - def PrintCmd(self, cmd, env): - if self.platform == 'win32': - env_prefix = 'set ' - env_quoter = QuoteForSet - shell_quoter = QuoteForCmd - else: - env_prefix = '' - env_quoter = pipes.quote - shell_quoter = pipes.quote + cmdline = [] + extra_files = [ + '../../.vpython', + '../../testing/test_env.py', + ] - def print_env(var): - if env and var in env: - self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var]))) + must_retry = False + if test_type == 'script': + cmdline += ['../../' + + self.ToSrcRelPath(isolate_map[target]['script'])] + elif is_android: + cmdline += ['../../build/android/test_wrapper/logdog_wrapper.py', + '--target', target, + '--logdog-bin-cmd', '../../bin/logdog_butler', + '--logcat-output-file', '${ISOLATED_OUTDIR}/logcats', + '--store-tombstones'] + else: + if test_type == 'raw': + cmdline.append('../../tools_webrtc/flags_compatibility.py') + extra_files.append('../../tools_webrtc/flags_compatibility.py') - print_env('LLVM_FORCE_HEAD_REVISION') + if isolate_map[target].get('use_webcam', False): + cmdline.append( + '../../tools_webrtc/ensure_webcam_is_running.py') + extra_files.append( + '../../tools_webrtc/ensure_webcam_is_running.py') - if cmd[0] == self.executable: - cmd = ['python'] + cmd[1:] - self.Print(*[shell_quoter(arg) for arg in cmd]) + # is_linux uses use_ozone and x11 by default. + use_x11 = is_linux - def PrintJSON(self, obj): - self.Print(json.dumps(obj, indent=2, sort_keys=True)) + xvfb = use_x11 and test_type == 'windowed_test_launcher' + if xvfb: + cmdline.append('../../testing/xvfb.py') + extra_files.append('../../testing/xvfb.py') + else: + cmdline.append('../../testing/test_env.py') - def Build(self, target): - build_dir = self.ToSrcRelPath(self.args.path[0]) - ninja_cmd = ['ninja', '-C', build_dir] - if self.args.jobs: - ninja_cmd.extend(['-j', '%d' % self.args.jobs]) - ninja_cmd.append(target) - ret, _, _ = self.Run(ninja_cmd, force_verbose=False, buffer_output=False) - return ret + if test_type != 'raw': + extra_files += [ + '../../third_party/gtest-parallel/gtest-parallel', + '../../third_party/gtest-parallel/gtest_parallel.py', + '../../tools_webrtc/gtest-parallel-wrapper.py', + ] + sep = '\\' if self.platform == 'win32' else '/' + output_dir = '${ISOLATED_OUTDIR}' + sep + 'test_logs' + timeout = isolate_map[target].get('timeout', 900) + cmdline += [ + '../../tools_webrtc/gtest-parallel-wrapper.py', + '--output_dir=%s' % output_dir, + '--gtest_color=no', + # We tell gtest-parallel to interrupt the test after 900 + # seconds, so it can exit cleanly and report results, + # instead of being interrupted by swarming and not + # reporting anything. + '--timeout=%s' % timeout, + ] + if test_type == 'non_parallel_console_test_launcher': + # Still use the gtest-parallel-wrapper.py script since we + # need it to run tests on swarming, but don't execute tests + # in parallel. + cmdline.append('--workers=1') + must_retry = True - def Run(self, cmd, env=None, force_verbose=True, buffer_output=True): - # This function largely exists so it can be overridden for testing. - if self.args.dryrun or self.args.verbose or force_verbose: - self.PrintCmd(cmd, env) - if self.args.dryrun: - return 0, '', '' + asan = 'is_asan=true' in vals['gn_args'] + lsan = 'is_lsan=true' in vals['gn_args'] + msan = 'is_msan=true' in vals['gn_args'] + tsan = 'is_tsan=true' in vals['gn_args'] + sanitizer = asan or lsan or msan or tsan + if must_retry and not sanitizer: + # Retry would hide most sanitizers detections. + cmdline.append('--retry_failed=3') - ret, out, err = self.Call(cmd, env=env, buffer_output=buffer_output) - if self.args.verbose or force_verbose: - if ret: - self.Print(' -> returned %d' % ret) - if out: - self.Print(out, end='') - if err: - self.Print(err, end='', file=sys.stderr) - return ret, out, err + executable_prefix = '.\\' if self.platform == 'win32' else './' + executable_suffix = '.exe' if self.platform == 'win32' else '' + executable = executable_prefix + target + executable_suffix - def Call(self, cmd, env=None, buffer_output=True): - if buffer_output: - p = subprocess.Popen(cmd, shell=False, cwd=self.src_dir, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=env) - out, err = p.communicate() - else: - p = subprocess.Popen(cmd, shell=False, cwd=self.src_dir, - env=env) - p.wait() - out = err = '' - return p.returncode, out, err + cmdline.append(executable) - def ExpandUser(self, path): - # This function largely exists so it can be overridden for testing. - return os.path.expanduser(path) + cmdline.extend([ + '--asan=%d' % asan, + '--lsan=%d' % lsan, + '--msan=%d' % msan, + '--tsan=%d' % tsan, + ]) - def Exists(self, path): - # This function largely exists so it can be overridden for testing. - return os.path.exists(path) + cmdline += isolate_map[target].get('args', []) - def Fetch(self, url): - # This function largely exists so it can be overridden for testing. - f = urllib2.urlopen(url) - contents = f.read() - f.close() - return contents + return cmdline, extra_files - def MaybeMakeDirectory(self, path): - try: - os.makedirs(path) - except OSError, e: - if e.errno != errno.EEXIST: - raise + def ToAbsPath(self, build_path, *comps): + return self.PathJoin(self.src_dir, + self.ToSrcRelPath(build_path), + *comps) - def PathJoin(self, *comps): - # This function largely exists so it can be overriden for testing. - return os.path.join(*comps) + def ToSrcRelPath(self, path): + """Returns a relative path from the top of the repo.""" + if path.startswith('//'): + return path[2:].replace('/', self.sep) + return self.RelPath(path, self.src_dir) - def Print(self, *args, **kwargs): - # This function largely exists so it can be overridden for testing. - print(*args, **kwargs) - if kwargs.get('stream', sys.stdout) == sys.stdout: - sys.stdout.flush() + def RunGNAnalyze(self, vals): + # Analyze runs before 'gn gen' now, so we need to run gn gen + # in order to ensure that we have a build directory. + ret = self.RunGNGen(vals) + if ret: + return ret - def ReadFile(self, path): - # This function largely exists so it can be overriden for testing. - with open(path) as fp: - return fp.read() + build_path = self.args.path[0] + input_path = self.args.input_path[0] + gn_input_path = input_path + '.gn' + output_path = self.args.output_path[0] + gn_output_path = output_path + '.gn' - def RelPath(self, path, start='.'): - # This function largely exists so it can be overriden for testing. - return os.path.relpath(path, start) + inp = self.ReadInputJSON(['files', 'test_targets', + 'additional_compile_targets']) + if self.args.verbose: + self.Print() + self.Print('analyze input:') + self.PrintJSON(inp) + self.Print() - def RemoveFile(self, path): - # This function largely exists so it can be overriden for testing. - os.remove(path) + # This shouldn't normally happen, but could due to unusual race + # conditions, like a try job that gets scheduled before a patch + # lands but runs after the patch has landed. + if not inp['files']: + self.Print( + 'Warning: No files modified in patch, bailing out early.') + self.WriteJSON({ + 'status': 'No dependency', + 'compile_targets': [], + 'test_targets': [], + }, output_path) + return 0 - def RemoveDirectory(self, abs_path): - if self.platform == 'win32': - # In other places in chromium, we often have to retry this command - # because we're worried about other processes still holding on to - # file handles, but when MB is invoked, it will be early enough in the - # build that their should be no other processes to interfere. We - # can change this if need be. - self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path]) - else: - shutil.rmtree(abs_path, ignore_errors=True) + gn_inp = {} + gn_inp['files'] = ['//' + f for f in inp['files'] + if not f.startswith('//')] - def TempDir(self): - # This function largely exists so it can be overriden for testing. - return tempfile.mkdtemp(prefix='mb_') + isolate_map = self.ReadIsolateMap() + err, gn_inp['additional_compile_targets'] = self.MapTargetsToLabels( + isolate_map, inp['additional_compile_targets']) + if err: + raise MBErr(err) - def TempFile(self, mode='w'): - # This function largely exists so it can be overriden for testing. - return tempfile.NamedTemporaryFile(mode=mode, delete=False) + err, gn_inp['test_targets'] = self.MapTargetsToLabels( + isolate_map, inp['test_targets']) + if err: + raise MBErr(err) + labels_to_targets = {} + for i, label in enumerate(gn_inp['test_targets']): + labels_to_targets[label] = inp['test_targets'][i] - def WriteFile(self, path, contents, force_verbose=False): - # This function largely exists so it can be overriden for testing. - if self.args.dryrun or self.args.verbose or force_verbose: - self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path)) - with open(path, 'w') as fp: - return fp.write(contents) + try: + self.WriteJSON(gn_inp, gn_input_path) + cmd = self.GNCmd('analyze', build_path, + gn_input_path, gn_output_path) + ret, output, _ = self.Run(cmd, force_verbose=True) + if ret: + if self.args.json_output: + # write errors to json.output + self.WriteJSON({'output': output}, self.args.json_output) + return ret + + gn_outp_str = self.ReadFile(gn_output_path) + try: + gn_outp = json.loads(gn_outp_str) + except Exception as e: + self.Print("Failed to parse the JSON string GN " + "returned: %s\n%s" % (repr(gn_outp_str), str(e))) + raise + + outp = {} + if 'status' in gn_outp: + outp['status'] = gn_outp['status'] + if 'error' in gn_outp: + outp['error'] = gn_outp['error'] + if 'invalid_targets' in gn_outp: + outp['invalid_targets'] = gn_outp['invalid_targets'] + if 'compile_targets' in gn_outp: + if 'all' in gn_outp['compile_targets']: + outp['compile_targets'] = ['all'] + else: + outp['compile_targets'] = [ + label.replace('//', '') + for label in gn_outp['compile_targets'] + ] + if 'test_targets' in gn_outp: + outp['test_targets'] = [ + labels_to_targets[label] + for label in gn_outp['test_targets'] + ] + + if self.args.verbose: + self.Print() + self.Print('analyze output:') + self.PrintJSON(outp) + self.Print() + + self.WriteJSON(outp, output_path) + + finally: + if self.Exists(gn_input_path): + self.RemoveFile(gn_input_path) + if self.Exists(gn_output_path): + self.RemoveFile(gn_output_path) + + return 0 + + def ReadInputJSON(self, required_keys): + path = self.args.input_path[0] + output_path = self.args.output_path[0] + if not self.Exists(path): + self.WriteFailureAndRaise( + '"%s" does not exist' % path, output_path) + + try: + inp = json.loads(self.ReadFile(path)) + except Exception as e: + self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' + % (path, e), output_path) + + for k in required_keys: + if not k in inp: + self.WriteFailureAndRaise('input file is missing a "%s" key' % + k, output_path) + + return inp + + def WriteFailureAndRaise(self, msg, output_path): + if output_path: + self.WriteJSON({'error': msg}, output_path, force_verbose=True) + raise MBErr(msg) + + def WriteJSON(self, obj, path, force_verbose=False): + try: + self.WriteFile(path, + json.dumps(obj, indent=2, sort_keys=True) + '\n', + force_verbose=force_verbose) + except Exception as e: + raise MBErr('Error %s writing to the output path "%s"' % + (e, path)) + + def PrintCmd(self, cmd, env): + if self.platform == 'win32': + env_prefix = 'set ' + env_quoter = QuoteForSet + shell_quoter = QuoteForCmd + else: + env_prefix = '' + env_quoter = pipes.quote + shell_quoter = pipes.quote + + def print_env(var): + if env and var in env: + self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var]))) + + print_env('LLVM_FORCE_HEAD_REVISION') + + if cmd[0] == self.executable: + cmd = ['python'] + cmd[1:] + self.Print(*[shell_quoter(arg) for arg in cmd]) + + def PrintJSON(self, obj): + self.Print(json.dumps(obj, indent=2, sort_keys=True)) + + def Build(self, target): + build_dir = self.ToSrcRelPath(self.args.path[0]) + ninja_cmd = ['ninja', '-C', build_dir] + if self.args.jobs: + ninja_cmd.extend(['-j', '%d' % self.args.jobs]) + ninja_cmd.append(target) + ret, _, _ = self.Run( + ninja_cmd, force_verbose=False, buffer_output=False) + return ret + + def Run(self, cmd, env=None, force_verbose=True, buffer_output=True): + # This function largely exists so it can be overridden for testing. + if self.args.dryrun or self.args.verbose or force_verbose: + self.PrintCmd(cmd, env) + if self.args.dryrun: + return 0, '', '' + + ret, out, err = self.Call(cmd, env=env, buffer_output=buffer_output) + if self.args.verbose or force_verbose: + if ret: + self.Print(' -> returned %d' % ret) + if out: + self.Print(out, end='') + if err: + self.Print(err, end='', file=sys.stderr) + return ret, out, err + + def Call(self, cmd, env=None, buffer_output=True): + if buffer_output: + p = subprocess.Popen(cmd, shell=False, cwd=self.src_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env) + out, err = p.communicate() + else: + p = subprocess.Popen(cmd, shell=False, cwd=self.src_dir, + env=env) + p.wait() + out = err = '' + return p.returncode, out, err + + def ExpandUser(self, path): + # This function largely exists so it can be overridden for testing. + return os.path.expanduser(path) + + def Exists(self, path): + # This function largely exists so it can be overridden for testing. + return os.path.exists(path) + + def Fetch(self, url): + # This function largely exists so it can be overridden for testing. + f = urllib2.urlopen(url) + contents = f.read() + f.close() + return contents + + def MaybeMakeDirectory(self, path): + try: + os.makedirs(path) + except OSError, e: + if e.errno != errno.EEXIST: + raise + + def PathJoin(self, *comps): + # This function largely exists so it can be overriden for testing. + return os.path.join(*comps) + + def Print(self, *args, **kwargs): + # This function largely exists so it can be overridden for testing. + print(*args, **kwargs) + if kwargs.get('stream', sys.stdout) == sys.stdout: + sys.stdout.flush() + + def ReadFile(self, path): + # This function largely exists so it can be overriden for testing. + with open(path) as fp: + return fp.read() + + def RelPath(self, path, start='.'): + # This function largely exists so it can be overriden for testing. + return os.path.relpath(path, start) + + def RemoveFile(self, path): + # This function largely exists so it can be overriden for testing. + os.remove(path) + + def RemoveDirectory(self, abs_path): + if self.platform == 'win32': + # In other places in chromium, we often have to retry this command + # because we're worried about other processes still holding on to + # file handles, but when MB is invoked, it will be early enough in + # the build that their should be no other processes to interfere. + # We can change this if need be. + self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path]) + else: + shutil.rmtree(abs_path, ignore_errors=True) + + def TempDir(self): + # This function largely exists so it can be overriden for testing. + return tempfile.mkdtemp(prefix='mb_') + + def TempFile(self, mode='w'): + # This function largely exists so it can be overriden for testing. + return tempfile.NamedTemporaryFile(mode=mode, delete=False) + + def WriteFile(self, path, contents, force_verbose=False): + # This function largely exists so it can be overriden for testing. + if self.args.dryrun or self.args.verbose or force_verbose: + self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path)) + with open(path, 'w') as fp: + return fp.write(contents) class MBErr(Exception): - pass + pass # See http://goo.gl/l5NPDW and http://goo.gl/4Diozm for the painful @@ -1263,24 +1293,24 @@ ALL_META_CHARS = UNSAFE_FOR_CMD.union(set('"')) def QuoteForSet(arg): - if any(a in UNSAFE_FOR_SET for a in arg): - arg = ''.join('^' + a if a in UNSAFE_FOR_SET else a for a in arg) - return arg + if any(a in UNSAFE_FOR_SET for a in arg): + arg = ''.join('^' + a if a in UNSAFE_FOR_SET else a for a in arg) + return arg def QuoteForCmd(arg): - # First, escape the arg so that CommandLineToArgvW will parse it properly. - if arg == '' or ' ' in arg or '"' in arg: - quote_re = re.compile(r'(\\*)"') - arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg)) + # First, escape the arg so that CommandLineToArgvW will parse it properly. + if arg == '' or ' ' in arg or '"' in arg: + quote_re = re.compile(r'(\\*)"') + arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg)) - # Then check to see if the arg contains any metacharacters other than - # double quotes; if it does, quote everything (including the double - # quotes) for safety. - if any(a in UNSAFE_FOR_CMD for a in arg): - arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg) - return arg + # Then check to see if the arg contains any metacharacters other than + # double quotes; if it does, quote everything (including the double + # quotes) for safety. + if any(a in UNSAFE_FOR_CMD for a in arg): + arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg) + return arg if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + sys.exit(main(sys.argv[1:]))