zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bring mb up to date with Chromium's changes

Browse Source 
Bug: None
Change-Id: I87abffb0bba6ee945447f3e651a505d7602fa15d
Reviewed-on: https://webrtc-review.googlesource.com/62641
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Commit-Queue: Oleh Prypin <oprypin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22488}
This commit is contained in:
Oleh Prypin
2018-03-18 17:34:20 +01:00
committed by Commit Bot
parent 8d2c5a810f
commit b708e93ad0
2 changed files with 191 additions and 582 deletions
Download Patch File Download Diff File Expand all files Collapse all files

437
tools_webrtc/mb/mb.py
View File

@ -7,9 +7,9 @@
# in the file PATENTS. All contributing project authors may # in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree. # be found in the AUTHORS file in the root of the source tree.
"""MB - the Meta-Build wrapper around GYP and GN """MB - the Meta-Build wrapper around GN.
MB is a wrapper script for GYP and GN that can be used to generate build files MB is a wrapper script for GN that can be used to generate build files
for sets of canned configurations and analyze them. for sets of canned configurations and analyze them.
""" """
@ -36,7 +36,6 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
SRC_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR)) SRC_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR))
sys.path = [os.path.join(SRC_DIR, 'build')] + sys.path sys.path = [os.path.join(SRC_DIR, 'build')] + sys.path
import find_depot_tools
import gn_helpers import gn_helpers
@ -97,16 +96,10 @@ class MetaBuildWrapper(object):
'(default is %(default)s)') '(default is %(default)s)')
subp.add_argument('-g', '--goma-dir', subp.add_argument('-g', '--goma-dir',
help='path to goma directory') help='path to goma directory')
subp.add_argument('--gyp-script', metavar='PATH',
default=self.PathJoin('build', 'gyp_chromium'),
help='path to gyp script relative to project root '
'(default is %(default)s)')
subp.add_argument('--android-version-code', subp.add_argument('--android-version-code',
help='Sets GN arg android_default_version_code and ' help='Sets GN arg android_default_version_code')
'GYP_DEFINE app_manifest_version_code')
subp.add_argument('--android-version-name', subp.add_argument('--android-version-name',
help='Sets GN arg android_default_version_name and ' help='Sets GN arg android_default_version_name')
'GYP_DEFINE app_manifest_version_name')
subp.add_argument('-n', '--dryrun', action='store_true', subp.add_argument('-n', '--dryrun', action='store_true',
help='Do a dry run (i.e., do nothing, just print ' help='Do a dry run (i.e., do nothing, just print '
'the commands that will run)') 'the commands that will run)')
@ -187,7 +180,6 @@ class MetaBuildWrapper(object):
' --test-launcher-retry-limit=0' ' --test-launcher-retry-limit=0'
'\n' '\n'
) )
AddCommonOptions(subp) AddCommonOptions(subp)
subp.add_argument('-j', '--jobs', dest='jobs', type=int, subp.add_argument('-j', '--jobs', dest='jobs', type=int,
help='Number of jobs to pass to ninja') help='Number of jobs to pass to ninja')
@ -199,6 +191,11 @@ class MetaBuildWrapper(object):
' This can be either a regular path or a ' ' This can be either a regular path or a '
'GN-style source-relative path like ' 'GN-style source-relative path like '
'//out/Default.')) '//out/Default.'))
subp.add_argument('-s', '--swarmed', action='store_true',
help='Run under swarming')
subp.add_argument('-d', '--dimension', default=[], action='append', nargs=2,
dest='dimensions', metavar='FOO bar',
help='dimension to filter on')
subp.add_argument('target', nargs=1, subp.add_argument('target', nargs=1,
help='ninja target to build and run') help='ninja target to build and run')
subp.add_argument('extra_args', nargs='*', subp.add_argument('extra_args', nargs='*',
@ -214,26 +211,6 @@ class MetaBuildWrapper(object):
help='path to config file (default is %(default)s)') help='path to config file (default is %(default)s)')
subp.set_defaults(func=self.CmdValidate) subp.set_defaults(func=self.CmdValidate)
subp = subps.add_parser('audit',
help='Audit the config file to track progress')
subp.add_argument('-f', '--config-file', metavar='PATH',
default=self.default_config,
help='path to config file (default is %(default)s)')
subp.add_argument('-i', '--internal', action='store_true',
help='check internal masters also')
subp.add_argument('-m', '--master', action='append',
help='master to audit (default is all non-internal '
'masters in file)')
subp.add_argument('-u', '--url-template', action='store',
default='https://build.chromium.org/p/'
'{master}/json/builders',
help='URL scheme for JSON APIs to buildbot '
'(default: %(default)s) ')
subp.add_argument('-c', '--check-compile', action='store_true',
help='check whether tbd and master-only bots actually'
' do compiles')
subp.set_defaults(func=self.CmdAudit)
subp = subps.add_parser('help', subp = subps.add_parser('help',
help='Get help on a subcommand.') help='Get help on a subcommand.')
subp.add_argument(nargs='?', action='store', dest='subcommand', subp.add_argument(nargs='?', action='store', dest='subcommand',
@ -261,11 +238,7 @@ class MetaBuildWrapper(object):
def CmdAnalyze(self): def CmdAnalyze(self):
vals = self.Lookup() vals = self.Lookup()
self.ClobberIfNeeded(vals)
if vals['type'] == 'gn':
return self.RunGNAnalyze(vals) return self.RunGNAnalyze(vals)
else:
return self.RunGYPAnalyze(vals)
def CmdExport(self): def CmdExport(self):
self.ReadConfigFile() self.ReadConfigFile()
@ -297,11 +270,7 @@ class MetaBuildWrapper(object):
def CmdGen(self): def CmdGen(self):
vals = self.Lookup() vals = self.Lookup()
self.ClobberIfNeeded(vals)
if vals['type'] == 'gn':
return self.RunGNGen(vals) return self.RunGNGen(vals)
else:
return self.RunGYPGen(vals)
def CmdHelp(self): def CmdHelp(self):
if self.args.subcommand: if self.args.subcommand:
@ -313,21 +282,14 @@ class MetaBuildWrapper(object):
vals = self.GetConfig() vals = self.GetConfig()
if not vals: if not vals:
return 1 return 1
if vals['type'] == 'gn':
return self.RunGNIsolate(vals) return self.RunGNIsolate(vals)
else:
return self.Build('%s_run' % self.args.target[0])
def CmdLookup(self): def CmdLookup(self):
vals = self.Lookup() vals = self.Lookup()
if vals['type'] == 'gn':
cmd = self.GNCmd('gen', '_path_') cmd = self.GNCmd('gen', '_path_')
gn_args = self.GNArgs(vals) gn_args = self.GNArgs(vals)
self.Print('\nWriting """\\\n%s""" to _path_/args.gn.\n' % gn_args) self.Print('\nWriting """\\\n%s""" to _path_/args.gn.\n' % gn_args)
env = None env = None
else:
cmd, env = self.GYPCmd('_path_', vals)
self.PrintCmd(cmd, env) self.PrintCmd(cmd, env)
return 0 return 0
@ -340,7 +302,6 @@ class MetaBuildWrapper(object):
build_dir = self.args.path[0] build_dir = self.args.path[0]
target = self.args.target[0] target = self.args.target[0]
if vals['type'] == 'gn':
if self.args.build: if self.args.build:
ret = self.Build(target) ret = self.Build(target)
if ret: if ret:
@ -348,11 +309,50 @@ class MetaBuildWrapper(object):
ret = self.RunGNIsolate(vals) ret = self.RunGNIsolate(vals)
if ret: if ret:
return ret return ret
if self.args.swarmed:
return self._RunUnderSwarming(build_dir, target)
else: else:
ret = self.Build('%s_run' % target) return self._RunLocallyIsolated(build_dir, target)
def _RunUnderSwarming(self, build_dir, target):
# TODO(dpranke): Look up the information for the target in
# the //testing/buildbot.json file, if possible, so that we
# can determine the isolate target, command line, and additional
# swarming parameters, if possible.
#
# TODO(dpranke): Also, add support for sharding and merging results.
dimensions = []
for k, v in self.args.dimensions:
dimensions += ['-d', k, v]
cmd = [
self.executable,
self.PathJoin('tools', 'swarming_client', 'isolate.py'),
'archive',
'-s',
self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)),
'-I', 'isolateserver.appspot.com',
]
ret, out, _ = self.Run(cmd, force_verbose=False)
if ret: if ret:
return ret return ret
isolated_hash = out.splitlines()[0].split()[0]
cmd = [
self.executable,
self.PathJoin('tools', 'swarming_client', 'swarming.py'),
'run',
'-s', isolated_hash,
'-I', 'isolateserver.appspot.com',
'-S', 'chromium-swarm.appspot.com',
] + dimensions
if self.args.extra_args:
cmd += ['--'] + self.args.extra_args
ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False)
return ret
def _RunLocallyIsolated(self, build_dir, target):
cmd = [ cmd = [
self.executable, self.executable,
self.PathJoin('tools', 'swarming_client', 'isolate.py'), self.PathJoin('tools', 'swarming_client', 'isolate.py'),
@ -362,9 +362,7 @@ class MetaBuildWrapper(object):
] ]
if self.args.extra_args: if self.args.extra_args:
cmd += ['--'] + self.args.extra_args cmd += ['--'] + self.args.extra_args
ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False)
ret, _, _ = self.Run(cmd, force_verbose=False, buffer_output=False)
return ret return ret
def CmdValidate(self, print_ok=True): def CmdValidate(self, print_ok=True):
@ -428,138 +426,17 @@ class MetaBuildWrapper(object):
self.Print('mb config file %s looks ok.' % self.args.config_file) self.Print('mb config file %s looks ok.' % self.args.config_file)
return 0 return 0
def CmdAudit(self):
"""Track the progress of the GYP->GN migration on the bots."""
# First, make sure the config file is okay, but don't print anything
# if it is (it will throw an error if it isn't).
self.CmdValidate(print_ok=False)
stats = OrderedDict()
STAT_MASTER_ONLY = 'Master only'
STAT_CONFIG_ONLY = 'Config only'
STAT_TBD = 'Still TBD'
STAT_GYP = 'Still GYP'
STAT_DONE = 'Done (on GN)'
stats[STAT_MASTER_ONLY] = 0
stats[STAT_CONFIG_ONLY] = 0
stats[STAT_TBD] = 0
stats[STAT_GYP] = 0
stats[STAT_DONE] = 0
def PrintBuilders(heading, builders, notes):
stats.setdefault(heading, 0)
stats[heading] += len(builders)
if builders:
self.Print(' %s:' % heading)
for builder in sorted(builders):
self.Print(' %s%s' % (builder, notes[builder]))
self.ReadConfigFile()
masters = self.args.master or self.masters
for master in sorted(masters):
url = self.args.url_template.replace('{master}', master)
self.Print('Auditing %s' % master)
MASTERS_TO_SKIP = (
'client.skia',
'client.v8.fyi',
'tryserver.v8',
)
if master in MASTERS_TO_SKIP:
# Skip these bots because converting them is the responsibility of
# those teams and out of scope for the Chromium migration to GN.
self.Print(' Skipped (out of scope)')
self.Print('')
continue
INTERNAL_MASTERS = ('official.desktop', 'official.desktop.continuous',
'internal.client.kitchensync')
if master in INTERNAL_MASTERS and not self.args.internal:
# Skip these because the servers aren't accessible by default ...
self.Print(' Skipped (internal)')
self.Print('')
continue
try:
# Fetch the /builders contents from the buildbot master. The
# keys of the dict are the builder names themselves.
json_contents = self.Fetch(url)
d = json.loads(json_contents)
except Exception as e:
self.Print(str(e))
return 1
config_builders = set(self.masters[master])
master_builders = set(d.keys())
both = master_builders & config_builders
master_only = master_builders - config_builders
config_only = config_builders - master_builders
tbd = set()
gyp = set()
done = set()
notes = {builder: '' for builder in config_builders | master_builders}
for builder in both:
config = self.masters[master][builder]
if config == 'tbd':
tbd.add(builder)
elif isinstance(config, dict):
vals = self.FlattenConfig(config.values()[0])
if vals['type'] == 'gyp':
gyp.add(builder)
else:
done.add(builder)
elif config.startswith('//'):
done.add(builder)
else:
vals = self.FlattenConfig(config)
if vals['type'] == 'gyp':
gyp.add(builder)
else:
done.add(builder)
if self.args.check_compile and (tbd or master_only):
either = tbd | master_only
for builder in either:
notes[builder] = ' (' + self.CheckCompile(master, builder) +')'
if master_only or config_only or tbd or gyp:
PrintBuilders(STAT_MASTER_ONLY, master_only, notes)
PrintBuilders(STAT_CONFIG_ONLY, config_only, notes)
PrintBuilders(STAT_TBD, tbd, notes)
PrintBuilders(STAT_GYP, gyp, notes)
else:
self.Print(' All GN!')
stats[STAT_DONE] += len(done)
self.Print('')
fmt = '{:<27} {:>4}'
self.Print(fmt.format('Totals', str(sum(int(v) for v in stats.values()))))
self.Print(fmt.format('-' * 27, '----'))
for stat, count in stats.items():
self.Print(fmt.format(stat, str(count)))
return 0
def GetConfig(self): def GetConfig(self):
build_dir = self.args.path[0] build_dir = self.args.path[0]
vals = self.DefaultVals() vals = self.DefaultVals()
if self.args.builder or self.args.master or self.args.config: if self.args.builder or self.args.master or self.args.config:
vals = self.Lookup() vals = self.Lookup()
if vals['type'] == 'gn':
# Re-run gn gen in order to ensure the config is consistent with the # Re-run gn gen in order to ensure the config is consistent with the
# build dir. # build dir.
self.RunGNGen(vals) self.RunGNGen(vals)
return vals return vals
mb_type_path = self.PathJoin(self.ToAbsPath(build_dir), 'mb_type')
if not self.Exists(mb_type_path):
toolchain_path = self.PathJoin(self.ToAbsPath(build_dir), toolchain_path = self.PathJoin(self.ToAbsPath(build_dir),
'toolchain.ninja') 'toolchain.ninja')
if not self.Exists(toolchain_path): if not self.Exists(toolchain_path):
@ -567,15 +444,8 @@ class MetaBuildWrapper(object):
'or pass in a -m/-b pair or a -c flag to specify the ' 'or pass in a -m/-b pair or a -c flag to specify the '
'configuration') 'configuration')
return {} return {}
else:
mb_type = 'gn'
else:
mb_type = self.ReadFile(mb_type_path).strip()
if mb_type == 'gn':
vals['gn_args'] = self.GNArgsFromDir(build_dir) vals['gn_args'] = self.GNArgsFromDir(build_dir)
vals['type'] = mb_type
return vals return vals
def GNArgsFromDir(self, build_dir): def GNArgsFromDir(self, build_dir):
@ -607,37 +477,21 @@ class MetaBuildWrapper(object):
raise MBErr('Config "%s" not found in %s' % raise MBErr('Config "%s" not found in %s' %
(config, self.args.config_file)) (config, self.args.config_file))
vals = self.FlattenConfig(config) vals = self.FlattenConfig(config)
# Do some basic sanity checking on the config so that we
# don't have to do this in every caller.
if 'type' not in vals:
vals['type'] = 'gn'
assert vals['type'] in ('gn', 'gyp'), (
'Unknown meta-build type "%s"' % vals['gn_args'])
return vals return vals
def ReadIOSBotConfig(self): def ReadIOSBotConfig(self):
if not self.args.master or not self.args.builder: if not self.args.master or not self.args.builder:
return {} return {}
path = self.PathJoin(self.src_dir, 'tools_webrtc', 'ios', path = self.PathJoin(self.src_dir, 'tools_webrtc', 'ios', self.args.master,
self.args.master,
self.args.builder.replace(' ', '_') + '.json') self.args.builder.replace(' ', '_') + '.json')
if not self.Exists(path): if not self.Exists(path):
return {} return {}
contents = json.loads(self.ReadFile(path)) contents = json.loads(self.ReadFile(path))
gyp_vals = contents.get('GYP_DEFINES', {})
if isinstance(gyp_vals, dict):
gyp_defines = ' '.join('%s=%s' % (k, v) for k, v in gyp_vals.items())
else:
gyp_defines = ' '.join(gyp_vals)
gn_args = ' '.join(contents.get('gn_args', [])) gn_args = ' '.join(contents.get('gn_args', []))
vals = self.DefaultVals() vals = self.DefaultVals()
vals['gn_args'] = gn_args vals['gn_args'] = gn_args
vals['gyp_defines'] = gyp_defines
vals['type'] = contents.get('mb_type', 'gn')
return vals return vals
def ReadConfigFile(self): def ReadConfigFile(self):
@ -655,14 +509,14 @@ class MetaBuildWrapper(object):
self.mixins = contents['mixins'] self.mixins = contents['mixins']
def ReadIsolateMap(self): def ReadIsolateMap(self):
if not self.Exists(self.args.isolate_map_file): isolate_map = self.args.isolate_map_file
raise MBErr('isolate map file not found at %s' % if not self.Exists(isolate_map):
self.args.isolate_map_file) raise MBErr('isolate map file not found at %s' % isolate_map)
try: try:
return ast.literal_eval(self.ReadFile(self.args.isolate_map_file)) return ast.literal_eval(self.ReadFile(isolate_map))
except SyntaxError as e: except SyntaxError as e:
raise MBErr('Failed to parse isolate map file "%s": %s' % raise MBErr(
(self.args.isolate_map_file, e)) 'Failed to parse isolate map file "%s": %s' % (isolate_map, e))
def ConfigFromArgs(self): def ConfigFromArgs(self):
if self.args.config: if self.args.config:
@ -713,9 +567,6 @@ class MetaBuildWrapper(object):
'args_file': '', 'args_file': '',
'cros_passthrough': False, 'cros_passthrough': False,
'gn_args': '', 'gn_args': '',
'gyp_defines': '',
'gyp_crosscompile': False,
'type': 'gn',
} }
def FlattenMixins(self, mixins, vals, visited): def FlattenMixins(self, mixins, vals, visited):
@ -734,50 +585,11 @@ class MetaBuildWrapper(object):
vals['gn_args'] += ' ' + mixin_vals['gn_args'] vals['gn_args'] += ' ' + mixin_vals['gn_args']
else: else:
vals['gn_args'] = mixin_vals['gn_args'] vals['gn_args'] = mixin_vals['gn_args']
if 'gyp_crosscompile' in mixin_vals:
vals['gyp_crosscompile'] = mixin_vals['gyp_crosscompile']
if 'gyp_defines' in mixin_vals:
if vals['gyp_defines']:
vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
else:
vals['gyp_defines'] = mixin_vals['gyp_defines']
if 'type' in mixin_vals:
vals['type'] = mixin_vals['type']
if 'mixins' in mixin_vals: if 'mixins' in mixin_vals:
self.FlattenMixins(mixin_vals['mixins'], vals, visited) self.FlattenMixins(mixin_vals['mixins'], vals, visited)
return vals return vals
def ClobberIfNeeded(self, vals):
path = self.args.path[0]
build_dir = self.ToAbsPath(path)
mb_type_path = self.PathJoin(build_dir, 'mb_type')
needs_clobber = False
new_mb_type = vals['type']
if self.Exists(build_dir):
if self.Exists(mb_type_path):
old_mb_type = self.ReadFile(mb_type_path)
if old_mb_type != new_mb_type:
self.Print("Build type mismatch: was %s, will be %s, clobbering %s" %
(old_mb_type, new_mb_type, path))
needs_clobber = True
else:
# There is no 'mb_type' file in the build directory, so this probably
# means that the prior build(s) were not done through mb, and we
# have no idea if this was a GYP build or a GN build. Clobber it
# to be safe.
self.Print("%s/mb_type missing, clobbering to be safe" % path)
needs_clobber = True
if self.args.dryrun:
return
if needs_clobber:
self.RemoveDirectory(build_dir)
self.MaybeMakeDirectory(build_dir)
self.WriteFile(mb_type_path, new_mb_type)
def RunGNGen(self, vals): def RunGNGen(self, vals):
build_dir = self.args.path[0] build_dir = self.args.path[0]
@ -956,7 +768,7 @@ class MetaBuildWrapper(object):
return err, labels return err, labels
def GNCmd(self, subcommand, path, *args): def GNCmd(self, subcommand, path, *args):
if self.platform == 'linux2': if self.platform.startswith('linux'):
subdir, exe = 'linux64', 'gn' subdir, exe = 'linux64', 'gn'
elif self.platform == 'darwin': elif self.platform == 'darwin':
subdir, exe = 'mac', 'gn' subdir, exe = 'mac', 'gn'
@ -999,44 +811,12 @@ class MetaBuildWrapper(object):
gn_args = ('import("%s")\n' % vals['args_file']) + gn_args gn_args = ('import("%s")\n' % vals['args_file']) + gn_args
return gn_args return gn_args
def RunGYPGen(self, vals):
path = self.args.path[0]
output_dir = self.ParseGYPConfigPath(path)
cmd, env = self.GYPCmd(output_dir, vals)
ret, _, _ = self.Run(cmd, env=env)
return ret
def RunGYPAnalyze(self, vals):
output_dir = self.ParseGYPConfigPath(self.args.path[0])
if self.args.verbose:
inp = self.ReadInputJSON(['files', 'test_targets',
'additional_compile_targets'])
self.Print()
self.Print('analyze input:')
self.PrintJSON(inp)
self.Print()
cmd, env = self.GYPCmd(output_dir, vals)
cmd.extend(['-f', 'analyzer',
'-G', 'config_path=%s' % self.args.input_path[0],
'-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
ret, _, _ = self.Run(cmd, env=env)
if not ret and self.args.verbose:
outp = json.loads(self.ReadFile(self.args.output_path[0]))
self.Print()
self.Print('analyze output:')
self.PrintJSON(outp)
self.Print()
return ret
def GetIsolateCommand(self, target, vals): def GetIsolateCommand(self, target, vals):
isolate_map = self.ReadIsolateMap() isolate_map = self.ReadIsolateMap()
test_type = isolate_map[target]['type'] test_type = isolate_map[target]['type']
android = 'target_os="android"' in vals['gn_args'] is_android = 'target_os="android"' in vals['gn_args']
is_linux = self.platform == 'linux2' and not android is_linux = self.platform.startswith('linux') and not is_android
if test_type == 'nontest': if test_type == 'nontest':
self.WriteFailureAndRaise('We should not be isolating %s.' % target, self.WriteFailureAndRaise('We should not be isolating %s.' % target,
@ -1048,11 +828,14 @@ class MetaBuildWrapper(object):
% (target, test_type), output_path=None) % (target, test_type), output_path=None)
cmdline = [] cmdline = []
extra_files = ['../../.vpython'] extra_files = [
'../../.vpython',
'../../testing/test_env.py',
]
if test_type == 'script': if test_type == 'script':
cmdline = ['../../' + self.ToSrcRelPath(isolate_map[target]['script'])] cmdline = ['../../' + self.ToSrcRelPath(isolate_map[target]['script'])]
elif android: elif is_android:
cmdline = ['../../build/android/test_wrapper/logdog_wrapper.py', cmdline = ['../../build/android/test_wrapper/logdog_wrapper.py',
'--target', target, '--target', target,
'--logdog-bin-cmd', '../../bin/logdog_butler', '--logdog-bin-cmd', '../../bin/logdog_butler',
@ -1146,86 +929,6 @@ class MetaBuildWrapper(object):
return path[2:].replace('/', self.sep) return path[2:].replace('/', self.sep)
return self.RelPath(path, self.src_dir) return self.RelPath(path, self.src_dir)
def ParseGYPConfigPath(self, path):
rpath = self.ToSrcRelPath(path)
output_dir, _, _ = rpath.rpartition(self.sep)
return output_dir
def GYPCmd(self, output_dir, vals):
if vals['cros_passthrough']:
if not 'GYP_DEFINES' in os.environ:
raise MBErr('MB is expecting GYP_DEFINES to be in the environment')
gyp_defines = os.environ['GYP_DEFINES']
if not 'chromeos=1' in gyp_defines:
raise MBErr('GYP_DEFINES is missing chromeos=1: (GYP_DEFINES=%s)' %
gyp_defines)
else:
gyp_defines = vals['gyp_defines']
goma_dir = self.args.goma_dir
# GYP uses shlex.split() to split the gyp defines into separate arguments,
# so we can support backslashes and and spaces in arguments by quoting
# them, even on Windows, where this normally wouldn't work.
if goma_dir and ('\\' in goma_dir or ' ' in goma_dir):
goma_dir = "'%s'" % goma_dir
if goma_dir:
gyp_defines += ' gomadir=%s' % goma_dir
android_version_code = self.args.android_version_code
if android_version_code:
gyp_defines += ' app_manifest_version_code=%s' % android_version_code
android_version_name = self.args.android_version_name
if android_version_name:
gyp_defines += ' app_manifest_version_name=%s' % android_version_name
cmd = [
self.executable,
self.args.gyp_script,
'-G',
'output_dir=' + output_dir,
]
# Ensure that we have an environment that only contains
# the exact values of the GYP variables we need.
env = os.environ.copy()
# This is a terrible hack to work around the fact that
# //tools/clang/scripts/update.py is invoked by GYP and GN but
# currently relies on an environment variable to figure out
# what revision to embed in the command line #defines.
# For GN, we've made this work via a gn arg that will cause update.py
# to get an additional command line arg, but getting that to work
# via GYP_DEFINES has proven difficult, so we rewrite the GYP_DEFINES
# to get rid of the arg and add the old var in, instead.
# See crbug.com/582737 for more on this. This can hopefully all
# go away with GYP.
m = re.search('llvm_force_head_revision=1\s*', gyp_defines)
if m:
env['LLVM_FORCE_HEAD_REVISION'] = '1'
gyp_defines = gyp_defines.replace(m.group(0), '')
# This is another terrible hack to work around the fact that
# GYP sets the link concurrency to use via the GYP_LINK_CONCURRENCY
# environment variable, and not via a proper GYP_DEFINE. See
# crbug.com/611491 for more on this.
m = re.search('gyp_link_concurrency=(\d+)(\s*)', gyp_defines)
if m:
env['GYP_LINK_CONCURRENCY'] = m.group(1)
gyp_defines = gyp_defines.replace(m.group(0), '')
env['GYP_GENERATORS'] = 'ninja'
if 'GYP_CHROMIUM_NO_ACTION' in env:
del env['GYP_CHROMIUM_NO_ACTION']
if 'GYP_CROSSCOMPILE' in env:
del env['GYP_CROSSCOMPILE']
env['GYP_DEFINES'] = gyp_defines
if vals['gyp_crosscompile']:
env['GYP_CROSSCOMPILE'] = '1'
return cmd, env
def RunGNAnalyze(self, vals): def RunGNAnalyze(self, vals):
# Analyze runs before 'gn gen' now, so we need to run gn gen # Analyze runs before 'gn gen' now, so we need to run gn gen
# in order to ensure that we have a build directory. # in order to ensure that we have a build directory.
@ -1392,9 +1095,6 @@ class MetaBuildWrapper(object):
if env and var in env: if env and var in env:
self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var]))) self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var])))
print_env('GYP_CROSSCOMPILE')
print_env('GYP_DEFINES')
print_env('GYP_LINK_CONCURRENCY')
print_env('LLVM_FORCE_HEAD_REVISION') print_env('LLVM_FORCE_HEAD_REVISION')
if cmd[0] == self.executable: if cmd[0] == self.executable:
@ -1406,11 +1106,7 @@ class MetaBuildWrapper(object):
def Build(self, target): def Build(self, target):
build_dir = self.ToSrcRelPath(self.args.path[0]) build_dir = self.ToSrcRelPath(self.args.path[0])
ninja_cmd = [ ninja_cmd = ['ninja', '-C', build_dir]
os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'ninja'),
'-C',
build_dir,
]
if self.args.jobs: if self.args.jobs:
ninja_cmd.extend(['-j', '%d' % self.args.jobs]) ninja_cmd.extend(['-j', '%d' % self.args.jobs])
ninja_cmd.append(target) ninja_cmd.append(target)
@ -1535,7 +1231,6 @@ def QuoteForSet(arg):
def QuoteForCmd(arg): def QuoteForCmd(arg):
# First, escape the arg so that CommandLineToArgvW will parse it properly. # First, escape the arg so that CommandLineToArgvW will parse it properly.
# From //tools/gyp/pylib/gyp/msvs_emulation.py:23.
if arg == '' or ' ' in arg or '"' in arg: if arg == '' or ' ' in arg or '"' in arg:
quote_re = re.compile(r'(\\*)"') quote_re = re.compile(r'(\\*)"')
arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg)) arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg))

280
tools_webrtc/mb/mb_unittest.py
View File

@ -69,8 +69,6 @@ class FakeMBW(mb.MetaBuildWrapper):
self.files[path] = contents self.files[path] = contents
def Call(self, cmd, env=None, buffer_output=True): def Call(self, cmd, env=None, buffer_output=True):
if env:
self.cross_compile = env.get('GYP_CROSSCOMPILE')
self.calls.append(cmd) self.calls.append(cmd)
if self.cmds: if self.cmds:
return self.cmds.pop(0) return self.cmds.pop(0)
@ -116,49 +114,34 @@ TEST_CONFIG = """\
'masters': { 'masters': {
'chromium': {}, 'chromium': {},
'fake_master': { 'fake_master': {
'fake_builder': 'gyp_rel_bot', 'fake_builder': 'rel_bot',
'fake_gn_builder': 'gn_rel_bot', 'fake_debug_builder': 'debug_goma',
'fake_gyp_crosscompile_builder': 'gyp_crosscompile', 'fake_args_bot': '//build/args/bots/fake_master/fake_args_bot.gn',
'fake_gn_debug_builder': 'gn_debug_goma', 'fake_memcheck_bot': 'memcheck_bot',
'fake_gyp_builder': 'gyp_debug', 'fake_multi_phase': { 'phase_1': 'phase_1', 'phase_2': 'phase_2'},
'fake_gn_args_bot': '//build/args/bots/fake_master/fake_gn_args_bot.gn', 'fake_android_bot': 'android_bot',
'fake_memcheck_bot': 'gn_memcheck_bot',
'fake_multi_phase': { 'phase_1': 'gn_phase_1', 'phase_2': 'gn_phase_2'},
'fake_android_bot': 'gn_android_bot',
}, },
}, },
'configs': { 'configs': {
'gyp_rel_bot': ['gyp', 'rel', 'goma'], 'rel_bot': ['rel', 'goma', 'fake_feature1'],
'gn_debug_goma': ['gn', 'debug', 'goma'], 'debug_goma': ['debug', 'goma'],
'gyp_debug': ['gyp', 'debug', 'fake_feature1'], 'phase_1': ['phase_1'],
'gn_rel_bot': ['gn', 'rel', 'goma'], 'phase_2': ['phase_2'],
'gyp_crosscompile': ['gyp', 'crosscompile'], 'memcheck_bot': ['memcheck'],
'gn_phase_1': ['gn', 'phase_1'], 'android_bot': ['android'],
'gn_phase_2': ['gn', 'phase_2'],
'gn_memcheck_bot': ['gn', 'memcheck'],
'gn_android_bot': ['gn', 'android'],
}, },
'mixins': { 'mixins': {
'crosscompile': {
'gyp_crosscompile': True,
},
'fake_feature1': { 'fake_feature1': {
'gn_args': 'enable_doom_melon=true', 'gn_args': 'enable_doom_melon=true',
'gyp_defines': 'doom_melon=1',
}, },
'gyp': {'type': 'gyp'},
'gn': {'type': 'gn'},
'goma': { 'goma': {
'gn_args': 'use_goma=true', 'gn_args': 'use_goma=true',
'gyp_defines': 'goma=1',
}, },
'phase_1': { 'phase_1': {
'gn_args': 'phase=1', 'gn_args': 'phase=1',
'gyp_args': 'phase=1',
}, },
'phase_2': { 'phase_2': {
'gn_args': 'phase=2', 'gn_args': 'phase=2',
'gyp_args': 'phase=2',
}, },
'rel': { 'rel': {
'gn_args': 'is_debug=false', 'gn_args': 'is_debug=false',
@ -176,29 +159,6 @@ TEST_CONFIG = """\
} }
""" """
GYP_HACKS_CONFIG = """\
{
'masters': {
'chromium': {},
'fake_master': {
'fake_builder': 'fake_config',
},
},
'configs': {
'fake_config': ['fake_mixin'],
},
'mixins': {
'fake_mixin': {
'type': 'gyp',
'gn_args': '',
'gyp_defines':
('foo=bar llvm_force_head_revision=1 '
'gyp_link_concurrency=1 baz=1'),
},
},
}
"""
class UnitTest(unittest.TestCase): class UnitTest(unittest.TestCase):
def fake_mbw(self, files=None, win32=False): def fake_mbw(self, files=None, win32=False):
@ -214,7 +174,7 @@ class UnitTest(unittest.TestCase):
}, },
}''') }''')
mbw.files.setdefault( mbw.files.setdefault(
mbw.ToAbsPath('//build/args/bots/fake_master/fake_gn_args_bot.gn'), mbw.ToAbsPath('//build/args/bots/fake_master/fake_args_bot.gn'),
'is_debug = false\n') 'is_debug = false\n')
if files: if files:
for path, contents in files.items(): for path, contents in files.items():
@ -234,38 +194,7 @@ class UnitTest(unittest.TestCase):
self.assertEqual(mbw.err, err) self.assertEqual(mbw.err, err)
return mbw return mbw
def test_clobber(self): def test_analyze(self):
files = {
'/fake_src/out/Debug': None,
'/fake_src/out/Debug/mb_type': None,
}
mbw = self.fake_mbw(files)
# The first time we run this, the build dir doesn't exist, so no clobber.
self.check(['gen', '-c', 'gn_debug_goma', '//out/Debug'], mbw=mbw, ret=0)
self.assertEqual(mbw.rmdirs, [])
self.assertEqual(mbw.files['/fake_src/out/Debug/mb_type'], 'gn')
# The second time we run this, the build dir exists and matches, so no
# clobber.
self.check(['gen', '-c', 'gn_debug_goma', '//out/Debug'], mbw=mbw, ret=0)
self.assertEqual(mbw.rmdirs, [])
self.assertEqual(mbw.files['/fake_src/out/Debug/mb_type'], 'gn')
# Now we switch build types; this should result in a clobber.
self.check(['gen', '-c', 'gyp_debug', '//out/Debug'], mbw=mbw, ret=0)
self.assertEqual(mbw.rmdirs, ['/fake_src/out/Debug'])
self.assertEqual(mbw.files['/fake_src/out/Debug/mb_type'], 'gyp')
# Now we delete mb_type; this checks the case where the build dir
# exists but wasn't populated by mb; this should also result in a clobber.
del mbw.files['/fake_src/out/Debug/mb_type']
self.check(['gen', '-c', 'gyp_debug', '//out/Debug'], mbw=mbw, ret=0)
self.assertEqual(mbw.rmdirs,
['/fake_src/out/Debug', '/fake_src/out/Debug'])
self.assertEqual(mbw.files['/fake_src/out/Debug/mb_type'], 'gyp')
def test_gn_analyze(self):
files = {'/tmp/in.json': '''{\ files = {'/tmp/in.json': '''{\
"files": ["foo/foo_unittest.cc"], "files": ["foo/foo_unittest.cc"],
"test_targets": ["foo_unittests"], "test_targets": ["foo_unittests"],
@ -280,7 +209,7 @@ class UnitTest(unittest.TestCase):
mbw = self.fake_mbw(files) mbw = self.fake_mbw(files)
mbw.Call = lambda cmd, env=None, buffer_output=True: (0, '', '') mbw.Call = lambda cmd, env=None, buffer_output=True: (0, '', '')
self.check(['analyze', '-c', 'gn_debug_goma', '//out/Default', self.check(['analyze', '-c', 'debug_goma', '//out/Default',
'/tmp/in.json', '/tmp/out.json'], mbw=mbw, ret=0) '/tmp/in.json', '/tmp/out.json'], mbw=mbw, ret=0)
out = json.loads(mbw.files['/tmp/out.json']) out = json.loads(mbw.files['/tmp/out.json'])
self.assertEqual(out, { self.assertEqual(out, {
@ -289,9 +218,9 @@ class UnitTest(unittest.TestCase):
'test_targets': ['foo_unittests'] 'test_targets': ['foo_unittests']
}) })
def test_gn_gen(self): def test_gen(self):
mbw = self.fake_mbw() mbw = self.fake_mbw()
self.check(['gen', '-c', 'gn_debug_goma', '//out/Default', '-g', '/goma'], self.check(['gen', '-c', 'debug_goma', '//out/Default', '-g', '/goma'],
mbw=mbw, ret=0) mbw=mbw, ret=0)
self.assertMultiLineEqual(mbw.files['/fake_src/out/Default/args.gn'], self.assertMultiLineEqual(mbw.files['/fake_src/out/Default/args.gn'],
('goma_dir = "/goma"\n' ('goma_dir = "/goma"\n'
@ -304,7 +233,7 @@ class UnitTest(unittest.TestCase):
mbw.out) mbw.out)
mbw = self.fake_mbw(win32=True) mbw = self.fake_mbw(win32=True)
self.check(['gen', '-c', 'gn_debug_goma', '-g', 'c:\\goma', '//out/Debug'], self.check(['gen', '-c', 'debug_goma', '-g', 'c:\\goma', '//out/Debug'],
mbw=mbw, ret=0) mbw=mbw, ret=0)
self.assertMultiLineEqual(mbw.files['c:\\fake_src\\out\\Debug\\args.gn'], self.assertMultiLineEqual(mbw.files['c:\\fake_src\\out\\Debug\\args.gn'],
('goma_dir = "c:\\\\goma"\n' ('goma_dir = "c:\\\\goma"\n'
@ -314,45 +243,44 @@ class UnitTest(unittest.TestCase):
'--check\n', mbw.out) '--check\n', mbw.out)
mbw = self.fake_mbw() mbw = self.fake_mbw()
self.check(['gen', '-m', 'fake_master', '-b', 'fake_gn_args_bot', self.check(['gen', '-m', 'fake_master', '-b', 'fake_args_bot',
'//out/Debug'], '//out/Debug'],
mbw=mbw, ret=0) mbw=mbw, ret=0)
self.assertEqual( self.assertEqual(
mbw.files['/fake_src/out/Debug/args.gn'], mbw.files['/fake_src/out/Debug/args.gn'],
'import("//build/args/bots/fake_master/fake_gn_args_bot.gn")\n') 'import("//build/args/bots/fake_master/fake_args_bot.gn")\n')
def test_gn_gen_fails(self): def test_gen_fails(self):
mbw = self.fake_mbw() mbw = self.fake_mbw()
mbw.Call = lambda cmd, env=None, buffer_output=True: (1, '', '') mbw.Call = lambda cmd, env=None, buffer_output=True: (1, '', '')
self.check(['gen', '-c', 'gn_debug_goma', '//out/Default'], mbw=mbw, ret=1) self.check(['gen', '-c', 'debug_goma', '//out/Default'], mbw=mbw, ret=1)
def test_gn_gen_swarming(self): def test_gen_swarming(self):
files = { files = {
'/tmp/swarming_targets': 'cc_perftests\n', '/tmp/swarming_targets': 'base_unittests\n',
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
"{'cc_perftests': {" "{'base_unittests': {"
" 'label': '//cc:cc_perftests'," " 'label': '//base:base_unittests',"
" 'type': 'console_test_launcher'," " 'type': 'raw',"
" 'args': [],"
"}}\n" "}}\n"
), ),
'c:\\fake_src\out\Default\cc_perftests.exe.runtime_deps': ( '/fake_src/out/Default/base_unittests.runtime_deps': (
"cc_perftests\n" "base_unittests\n"
), ),
} }
mbw = self.fake_mbw(files=files, win32=True) mbw = self.fake_mbw(files)
self.check(['gen', self.check(['gen',
'-c', 'gn_debug_goma', '-c', 'debug_goma',
'--swarming-targets-file', '/tmp/swarming_targets', '--swarming-targets-file', '/tmp/swarming_targets',
'--isolate-map-file',
'/fake_src/testing/buildbot/gn_isolate_map.pyl',
'//out/Default'], mbw=mbw, ret=0) '//out/Default'], mbw=mbw, ret=0)
self.assertIn('c:\\fake_src\\out\\Default\\cc_perftests.isolate', self.assertIn('/fake_src/out/Default/base_unittests.isolate',
mbw.files) mbw.files)
self.assertIn('c:\\fake_src\\out\\Default\\cc_perftests.isolated.gen.json', self.assertIn('/fake_src/out/Default/base_unittests.isolated.gen.json',
mbw.files) mbw.files)
def test_gn_gen_swarming_android(self): def test_gen_swarming_android(self):
test_files = { test_files = {
'/tmp/swarming_targets': 'base_unittests\n', '/tmp/swarming_targets': 'base_unittests\n',
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
@ -365,7 +293,7 @@ class UnitTest(unittest.TestCase):
"base_unittests\n" "base_unittests\n"
), ),
} }
mbw = self.check(['gen', '-c', 'gn_android_bot', '//out/Default', mbw = self.check(['gen', '-c', 'android_bot', '//out/Default',
'--swarming-targets-file', '/tmp/swarming_targets', '--swarming-targets-file', '/tmp/swarming_targets',
'--isolate-map-file', '--isolate-map-file',
'/fake_src/testing/buildbot/gn_isolate_map.pyl'], '/fake_src/testing/buildbot/gn_isolate_map.pyl'],
@ -376,7 +304,8 @@ class UnitTest(unittest.TestCase):
files = isolate_file_contents['variables']['files'] files = isolate_file_contents['variables']['files']
command = isolate_file_contents['variables']['command'] command = isolate_file_contents['variables']['command']
self.assertEqual(files, ['../../.vpython', 'base_unittests']) self.assertEqual(files, ['../../.vpython', '../../testing/test_env.py',
'base_unittests'])
self.assertEqual(command, [ self.assertEqual(command, [
'../../build/android/test_wrapper/logdog_wrapper.py', '../../build/android/test_wrapper/logdog_wrapper.py',
'--target', 'base_unittests', '--target', 'base_unittests',
@ -385,7 +314,7 @@ class UnitTest(unittest.TestCase):
'--store-tombstones', '--store-tombstones',
]) ])
def test_gn_gen_swarming_android_junit_test(self): def test_gen_swarming_android_junit_test(self):
test_files = { test_files = {
'/tmp/swarming_targets': 'base_unittests\n', '/tmp/swarming_targets': 'base_unittests\n',
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
@ -398,7 +327,7 @@ class UnitTest(unittest.TestCase):
"base_unittests\n" "base_unittests\n"
), ),
} }
mbw = self.check(['gen', '-c', 'gn_android_bot', '//out/Default', mbw = self.check(['gen', '-c', 'android_bot', '//out/Default',
'--swarming-targets-file', '/tmp/swarming_targets', '--swarming-targets-file', '/tmp/swarming_targets',
'--isolate-map-file', '--isolate-map-file',
'/fake_src/testing/buildbot/gn_isolate_map.pyl'], '/fake_src/testing/buildbot/gn_isolate_map.pyl'],
@ -409,7 +338,8 @@ class UnitTest(unittest.TestCase):
files = isolate_file_contents['variables']['files'] files = isolate_file_contents['variables']['files']
command = isolate_file_contents['variables']['command'] command = isolate_file_contents['variables']['command']
self.assertEqual(files, ['../../.vpython', 'base_unittests']) self.assertEqual(files, ['../../.vpython', '../../testing/test_env.py',
'base_unittests'])
self.assertEqual(command, [ self.assertEqual(command, [
'../../build/android/test_wrapper/logdog_wrapper.py', '../../build/android/test_wrapper/logdog_wrapper.py',
'--target', 'base_unittests', '--target', 'base_unittests',
@ -418,7 +348,7 @@ class UnitTest(unittest.TestCase):
'--store-tombstones', '--store-tombstones',
]) ])
def test_gn_gen_timeout(self): def test_gen_timeout(self):
test_files = { test_files = {
'/tmp/swarming_targets': 'base_unittests\n', '/tmp/swarming_targets': 'base_unittests\n',
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
@ -432,7 +362,7 @@ class UnitTest(unittest.TestCase):
"base_unittests\n" "base_unittests\n"
), ),
} }
mbw = self.check(['gen', '-c', 'gn_debug_goma', '//out/Default', mbw = self.check(['gen', '-c', 'debug_goma', '//out/Default',
'--swarming-targets-file', '/tmp/swarming_targets', '--swarming-targets-file', '/tmp/swarming_targets',
'--isolate-map-file', '--isolate-map-file',
'/fake_src/testing/buildbot/gn_isolate_map.pyl'], '/fake_src/testing/buildbot/gn_isolate_map.pyl'],
@ -466,7 +396,7 @@ class UnitTest(unittest.TestCase):
'--tsan=0', '--tsan=0',
]) ])
def test_gn_gen_script(self): def test_gen_script(self):
test_files = { test_files = {
'/tmp/swarming_targets': 'base_unittests_script\n', '/tmp/swarming_targets': 'base_unittests_script\n',
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
@ -481,7 +411,7 @@ class UnitTest(unittest.TestCase):
"base_unittests_script.py\n" "base_unittests_script.py\n"
), ),
} }
mbw = self.check(['gen', '-c', 'gn_debug_goma', '//out/Default', mbw = self.check(['gen', '-c', 'debug_goma', '//out/Default',
'--swarming-targets-file', '/tmp/swarming_targets', '--swarming-targets-file', '/tmp/swarming_targets',
'--isolate-map-file', '--isolate-map-file',
'/fake_src/testing/buildbot/gn_isolate_map.pyl'], '/fake_src/testing/buildbot/gn_isolate_map.pyl'],
@ -494,15 +424,14 @@ class UnitTest(unittest.TestCase):
command = isolate_file_contents['variables']['command'] command = isolate_file_contents['variables']['command']
self.assertEqual(files, [ self.assertEqual(files, [
'../../.vpython', '../../.vpython', '../../testing/test_env.py',
'base_unittests', 'base_unittests', 'base_unittests_script.py',
'base_unittests_script.py',
]) ])
self.assertEqual(command, [ self.assertEqual(command, [
'../../base/base_unittests_script.py', '../../base/base_unittests_script.py',
]) ])
def test_gn_gen_raw(self): def test_gen_raw(self):
test_files = { test_files = {
'/tmp/swarming_targets': 'base_unittests\n', '/tmp/swarming_targets': 'base_unittests\n',
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
@ -515,7 +444,7 @@ class UnitTest(unittest.TestCase):
"base_unittests\n" "base_unittests\n"
), ),
} }
mbw = self.check(['gen', '-c', 'gn_debug_goma', '//out/Default', mbw = self.check(['gen', '-c', 'debug_goma', '//out/Default',
'--swarming-targets-file', '/tmp/swarming_targets', '--swarming-targets-file', '/tmp/swarming_targets',
'--isolate-map-file', '--isolate-map-file',
'/fake_src/testing/buildbot/gn_isolate_map.pyl'], '/fake_src/testing/buildbot/gn_isolate_map.pyl'],
@ -539,7 +468,7 @@ class UnitTest(unittest.TestCase):
'--tsan=0', '--tsan=0',
]) ])
def test_gn_gen_non_parallel_console_test_launcher(self): def test_gen_non_parallel_console_test_launcher(self):
test_files = { test_files = {
'/tmp/swarming_targets': 'base_unittests\n', '/tmp/swarming_targets': 'base_unittests\n',
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
@ -552,7 +481,7 @@ class UnitTest(unittest.TestCase):
"base_unittests\n" "base_unittests\n"
), ),
} }
mbw = self.check(['gen', '-c', 'gn_debug_goma', '//out/Default', mbw = self.check(['gen', '-c', 'debug_goma', '//out/Default',
'--swarming-targets-file', '/tmp/swarming_targets', '--swarming-targets-file', '/tmp/swarming_targets',
'--isolate-map-file', '--isolate-map-file',
'/fake_src/testing/buildbot/gn_isolate_map.pyl'], '/fake_src/testing/buildbot/gn_isolate_map.pyl'],
@ -586,7 +515,7 @@ class UnitTest(unittest.TestCase):
'--tsan=0', '--tsan=0',
]) ])
def test_gn_isolate_windowed_test_launcher_linux(self): def test_isolate_windowed_test_launcher_linux(self):
test_files = { test_files = {
'/tmp/swarming_targets': 'base_unittests\n', '/tmp/swarming_targets': 'base_unittests\n',
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
@ -600,7 +529,7 @@ class UnitTest(unittest.TestCase):
"some_resource_file\n" "some_resource_file\n"
), ),
} }
mbw = self.check(['gen', '-c', 'gn_debug_goma', '//out/Default', mbw = self.check(['gen', '-c', 'debug_goma', '//out/Default',
'--swarming-targets-file', '/tmp/swarming_targets', '--swarming-targets-file', '/tmp/swarming_targets',
'--isolate-map-file', '--isolate-map-file',
'/fake_src/testing/buildbot/gn_isolate_map.pyl'], '/fake_src/testing/buildbot/gn_isolate_map.pyl'],
@ -635,7 +564,7 @@ class UnitTest(unittest.TestCase):
'--tsan=0', '--tsan=0',
]) ])
def test_gn_gen_windowed_test_launcher_win(self): def test_gen_windowed_test_launcher_win(self):
files = { files = {
'/tmp/swarming_targets': 'unittests\n', '/tmp/swarming_targets': 'unittests\n',
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
@ -651,7 +580,7 @@ class UnitTest(unittest.TestCase):
} }
mbw = self.fake_mbw(files=files, win32=True) mbw = self.fake_mbw(files=files, win32=True)
self.check(['gen', self.check(['gen',
'-c', 'gn_debug_goma', '-c', 'debug_goma',
'--swarming-targets-file', '/tmp/swarming_targets', '--swarming-targets-file', '/tmp/swarming_targets',
'--isolate-map-file', '--isolate-map-file',
'/fake_src/testing/buildbot/gn_isolate_map.pyl', '/fake_src/testing/buildbot/gn_isolate_map.pyl',
@ -685,7 +614,7 @@ class UnitTest(unittest.TestCase):
'--tsan=0', '--tsan=0',
]) ])
def test_gn_gen_console_test_launcher(self): def test_gen_console_test_launcher(self):
test_files = { test_files = {
'/tmp/swarming_targets': 'base_unittests\n', '/tmp/swarming_targets': 'base_unittests\n',
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
@ -698,7 +627,7 @@ class UnitTest(unittest.TestCase):
"base_unittests\n" "base_unittests\n"
), ),
} }
mbw = self.check(['gen', '-c', 'gn_debug_goma', '//out/Default', mbw = self.check(['gen', '-c', 'debug_goma', '//out/Default',
'--swarming-targets-file', '/tmp/swarming_targets', '--swarming-targets-file', '/tmp/swarming_targets',
'--isolate-map-file', '--isolate-map-file',
'/fake_src/testing/buildbot/gn_isolate_map.pyl'], '/fake_src/testing/buildbot/gn_isolate_map.pyl'],
@ -731,7 +660,7 @@ class UnitTest(unittest.TestCase):
'--tsan=0', '--tsan=0',
]) ])
def test_gn_isolate_console_test_launcher_memcheck(self): def test_isolate_console_test_launcher_memcheck(self):
test_files = { test_files = {
'/tmp/swarming_targets': 'base_unittests\n', '/tmp/swarming_targets': 'base_unittests\n',
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
@ -746,7 +675,7 @@ class UnitTest(unittest.TestCase):
"../../tools_webrtc/valgrind/webrtc_tests.sh\n" "../../tools_webrtc/valgrind/webrtc_tests.sh\n"
), ),
} }
mbw = self.check(['gen', '-c', 'gn_memcheck_bot', '//out/Release', mbw = self.check(['gen', '-c', 'memcheck_bot', '//out/Release',
'--swarming-targets-file', '/tmp/swarming_targets', '--swarming-targets-file', '/tmp/swarming_targets',
'--isolate-map-file', '--isolate-map-file',
'/fake_src/testing/buildbot/gn_isolate_map.pyl'], '/fake_src/testing/buildbot/gn_isolate_map.pyl'],
@ -782,7 +711,7 @@ class UnitTest(unittest.TestCase):
'--tsan=0', '--tsan=0',
]) ])
def test_gn_isolate(self): def test_isolate(self):
files = { files = {
'/fake_src/out/Default/toolchain.ninja': "", '/fake_src/out/Default/toolchain.ninja': "",
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
@ -795,7 +724,7 @@ class UnitTest(unittest.TestCase):
"base_unittests\n" "base_unittests\n"
), ),
} }
self.check(['isolate', '-c', 'gn_debug_goma', '//out/Default', self.check(['isolate', '-c', 'debug_goma', '//out/Default',
'base_unittests'], files=files, ret=0) 'base_unittests'], files=files, ret=0)
# test running isolate on an existing build_dir # test running isolate on an existing build_dir
@ -806,7 +735,7 @@ class UnitTest(unittest.TestCase):
self.check(['isolate', '//out/Default', 'base_unittests'], self.check(['isolate', '//out/Default', 'base_unittests'],
files=files, ret=0) files=files, ret=0)
def test_gn_run(self): def test_run(self):
files = { files = {
'/fake_src/testing/buildbot/gn_isolate_map.pyl': ( '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
"{'base_unittests': {" "{'base_unittests': {"
@ -818,55 +747,50 @@ class UnitTest(unittest.TestCase):
"base_unittests\n" "base_unittests\n"
), ),
} }
self.check(['run', '-c', 'gn_debug_goma', '//out/Default', self.check(['run', '-c', 'debug_goma', '//out/Default',
'base_unittests'], files=files, ret=0) 'base_unittests'], files=files, ret=0)
def test_gn_lookup(self): def test_run_swarmed(self):
self.check(['lookup', '-c', 'gn_debug_goma'], ret=0) files = {
'/fake_src/testing/buildbot/gn_isolate_map.pyl': (
"{'base_unittests': {"
" 'label': '//base:base_unittests',"
" 'type': 'raw',"
" 'args': [],"
"}}\n"
),
'/fake_src/out/Default/base_unittests.runtime_deps': (
"base_unittests\n"
),
}
def test_gn_lookup_goma_dir_expansion(self): def run_stub(cmd, **_kwargs):
self.check(['lookup', '-c', 'gn_rel_bot', '-g', '/foo'], ret=0, if 'isolate.py' in cmd[1]:
return 0, 'fake_hash base_unittests', ''
else:
return 0, '', ''
mbw = self.fake_mbw(files=files)
mbw.Run = run_stub
self.check(['run', '-s', '-c', 'debug_goma', '//out/Default',
'base_unittests'], mbw=mbw, ret=0)
self.check(['run', '-s', '-c', 'debug_goma', '-d', 'os', 'Win7',
'//out/Default', 'base_unittests'], mbw=mbw, ret=0)
def test_lookup(self):
self.check(['lookup', '-c', 'debug_goma'], ret=0)
def test_lookup_goma_dir_expansion(self):
self.check(['lookup', '-c', 'rel_bot', '-g', '/foo'], ret=0,
out=('\n' out=('\n'
'Writing """\\\n' 'Writing """\\\n'
'enable_doom_melon = true\n'
'goma_dir = "/foo"\n' 'goma_dir = "/foo"\n'
'is_debug = false\n' 'is_debug = false\n'
'use_goma = true\n' 'use_goma = true\n'
'""" to _path_/args.gn.\n\n' '""" to _path_/args.gn.\n\n'
'/fake_src/buildtools/linux64/gn gen _path_\n')) '/fake_src/buildtools/linux64/gn gen _path_\n'))
def test_gyp_analyze(self):
mbw = self.check(['analyze', '-c', 'gyp_rel_bot', '//out/Release',
'/tmp/in.json', '/tmp/out.json'], ret=0)
self.assertIn('analyzer', mbw.calls[0])
def test_gyp_crosscompile(self):
mbw = self.fake_mbw()
self.check(['gen', '-c', 'gyp_crosscompile', '//out/Release'],
mbw=mbw, ret=0)
self.assertTrue(mbw.cross_compile)
def test_gyp_gen(self):
self.check(['gen', '-c', 'gyp_rel_bot', '-g', '/goma', '//out/Release'],
ret=0,
out=("GYP_DEFINES='goma=1 gomadir=/goma'\n"
"python build/gyp_chromium -G output_dir=out\n"))
mbw = self.fake_mbw(win32=True)
self.check(['gen', '-c', 'gyp_rel_bot', '-g', 'c:\\goma', '//out/Release'],
mbw=mbw, ret=0,
out=("set GYP_DEFINES=goma=1 gomadir='c:\\goma'\n"
"python build\\gyp_chromium -G output_dir=out\n"))
def test_gyp_gen_fails(self):
mbw = self.fake_mbw()
mbw.Call = lambda cmd, env=None, buffer_output=True: (1, '', '')
self.check(['gen', '-c', 'gyp_rel_bot', '//out/Release'], mbw=mbw, ret=1)
def test_gyp_lookup_goma_dir_expansion(self):
self.check(['lookup', '-c', 'gyp_rel_bot', '-g', '/foo'], ret=0,
out=("GYP_DEFINES='goma=1 gomadir=/foo'\n"
"python build/gyp_chromium -G output_dir=_path_\n"))
def test_help(self): def test_help(self):
orig_stdout = sys.stdout orig_stdout = sys.stdout
try: try:
@ -884,7 +808,7 @@ class UnitTest(unittest.TestCase):
self.assertIn('Must specify a build --phase', mbw.out) self.assertIn('Must specify a build --phase', mbw.out)
# Check that passing a --phase to a single-phase builder fails. # Check that passing a --phase to a single-phase builder fails.
mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_gn_builder', mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_builder',
'--phase', 'phase_1'], ret=1) '--phase', 'phase_1'], ret=1)
self.assertIn('Must not specify a build --phase', mbw.out) self.assertIn('Must not specify a build --phase', mbw.out)
@ -906,16 +830,6 @@ class UnitTest(unittest.TestCase):
mbw = self.fake_mbw() mbw = self.fake_mbw()
self.check(['validate'], mbw=mbw, ret=0) self.check(['validate'], mbw=mbw, ret=0)
def test_gyp_env_hacks(self):
mbw = self.fake_mbw()
mbw.files[mbw.default_config] = GYP_HACKS_CONFIG
self.check(['lookup', '-c', 'fake_config'], mbw=mbw,
ret=0,
out=("GYP_DEFINES='foo=bar baz=1'\n"
"GYP_LINK_CONCURRENCY=1\n"
"LLVM_FORCE_HEAD_REVISION=1\n"
"python build/gyp_chromium -G output_dir=_path_\n"))
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()