4e869e9f34
NOTRY=True BUG=webrtc:5949 Review-Url: https://codereview.webrtc.org/2320873003 Cr-Commit-Position: refs/heads/master@{#14123}
469 lines
17 KiB
Python
Executable File
469 lines
17 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
|
|
#
|
|
# Use of this source code is governed by a BSD-style license
|
|
# that can be found in the LICENSE file in the root of the source
|
|
# tree. An additional intellectual property rights grant can be found
|
|
# in the file PATENTS. All contributing project authors may
|
|
# be found in the AUTHORS file in the root of the source tree.
|
|
|
|
"""Given the output of -t commands from a ninja build for a gyp and GN generated
|
|
build, report on differences between the command lines.
|
|
|
|
|
|
When invoked from the command line, this script assumes that the GN and GYP
|
|
targets have been generated in the specified folders. It is meant to be used as
|
|
follows:
|
|
$ python tools/gyp_flag_compare.py gyp_dir gn_dir target
|
|
|
|
When the GN and GYP target names differ, it should be called invoked as follows:
|
|
$ python tools/gyp_flag_compare.py gyp_dir gn_dir gyp_target gn_target
|
|
|
|
When all targets want to be compared, it should be called without a target name,
|
|
i.e.:
|
|
$ python tools/gyp_flag_compare.py gyp_dir gn_dir
|
|
|
|
This script can also be used interactively. Then ConfigureBuild can optionally
|
|
be used to generate ninja files with GYP and GN.
|
|
Here's an example setup. Note that the current working directory must be the
|
|
project root:
|
|
$ PYTHONPATH=tools python
|
|
>>> import sys
|
|
>>> import pprint
|
|
>>> sys.displayhook = pprint.pprint
|
|
>>> import gyp_flag_compare as fc
|
|
>>> fc.ConfigureBuild(['gyp_define=1', 'define=2'], ['gn_arg=1', 'arg=2'])
|
|
>>> modules_unittests = fc.Comparison('modules_unittests')
|
|
|
|
The above starts interactive Python, sets up the output to be pretty-printed
|
|
(useful for making lists, dicts, and sets readable), configures the build with
|
|
GN arguments and GYP defines, and then generates a comparison for that build
|
|
configuration for the "modules_unittests" target.
|
|
|
|
After that, the |modules_unittests| object can be used to investigate
|
|
differences in the build.
|
|
|
|
To configure an official build, use this configuration. Disabling NaCl produces
|
|
a more meaningful comparison, as certain files need to get compiled twice
|
|
for the IRT build, which uses different flags:
|
|
>>> fc.ConfigureBuild(
|
|
['disable_nacl=1', 'buildtype=Official', 'branding=Chrome'],
|
|
['enable_nacl=false', 'is_official_build=true',
|
|
'is_chrome_branded=true'])
|
|
"""
|
|
|
|
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
|
|
# Must be in src/.
|
|
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
os.chdir(BASE_DIR)
|
|
|
|
_DEFAULT_GN_DIR = 'out/gn'
|
|
_DEFAULT_GYP_DIR = 'out/Release'
|
|
|
|
|
|
def FilterChromium(filename):
|
|
"""Replaces 'chromium/src/' by '' in the filename."""
|
|
return filename.replace('chromium/src/', '')
|
|
|
|
|
|
def ConfigureBuild(gyp_args=None, gn_args=None, gn_dir=_DEFAULT_GN_DIR):
|
|
"""Generates gn and gyp targets with the given arguments."""
|
|
gyp_args = gyp_args or []
|
|
gn_args = gn_args or []
|
|
|
|
print >> sys.stderr, 'Regenerating GN in %s...' % gn_dir
|
|
# Currently only Release, non-component.
|
|
Run('gn gen %s --args="is_debug=false is_component_build=false %s"' % \
|
|
(gn_dir, ' '.join(gn_args)))
|
|
|
|
os.environ.pop('GYP_DEFINES', None)
|
|
# Remove environment variables required by gn but conflicting with GYP.
|
|
# Relevant if Windows toolchain isn't provided by depot_tools.
|
|
os.environ.pop('GYP_MSVS_OVERRIDE_PATH', None)
|
|
os.environ.pop('WINDOWSSDKDIR', None)
|
|
|
|
gyp_defines = ''
|
|
if len(gyp_args) > 0:
|
|
gyp_defines = '-D' + ' -D'.join(gyp_args)
|
|
|
|
print >> sys.stderr, 'Regenerating GYP in %s...' % _DEFAULT_GYP_DIR
|
|
Run('python webrtc/build/gyp_webrtc.py -Gconfig=Release %s' % gyp_defines)
|
|
|
|
|
|
def Counts(dict_of_list):
|
|
"""Given a dictionary whose value are lists, returns a dictionary whose values
|
|
are the length of the list. This can be used to summarize a dictionary.
|
|
"""
|
|
return {k: len(v) for k, v in dict_of_list.iteritems()}
|
|
|
|
|
|
def CountsByDirname(dict_of_list):
|
|
"""Given a list of files, returns a dict of dirname to counts in that dir."""
|
|
r = {}
|
|
for path in dict_of_list:
|
|
dirname = os.path.dirname(path)
|
|
r.setdefault(dirname, 0)
|
|
r[dirname] += 1
|
|
return r
|
|
|
|
|
|
class Comparison(object):
|
|
"""A comparison of the currently-configured build for a target."""
|
|
|
|
def __init__(self, gyp_target="", gn_target=None, gyp_dir=_DEFAULT_GYP_DIR,
|
|
gn_dir=_DEFAULT_GN_DIR):
|
|
"""Creates a comparison of a GN and GYP target. If the target names differ
|
|
between the two build systems, then two names may be passed.
|
|
"""
|
|
if gn_target is None:
|
|
gn_target = gyp_target
|
|
|
|
self._gyp_target = gyp_target
|
|
self._gn_target = gn_target
|
|
|
|
self._gyp_dir = gyp_dir
|
|
self._gn_dir = gn_dir
|
|
|
|
self._skipped = []
|
|
|
|
self._total_diffs = 0
|
|
|
|
self._missing_gyp_flags = {}
|
|
self._missing_gn_flags = {}
|
|
|
|
self._missing_gyp_files = {}
|
|
self._missing_gn_files = {}
|
|
|
|
self._CompareFiles()
|
|
|
|
@property
|
|
def gyp_files(self):
|
|
"""Returns the set of files that are in the GYP target."""
|
|
return set(self._gyp_flags.keys())
|
|
|
|
@property
|
|
def gn_files(self):
|
|
"""Returns the set of files that are in the GN target."""
|
|
return set(self._gn_flags.keys())
|
|
|
|
@property
|
|
def skipped(self):
|
|
"""Returns the list of compiler commands that were not processed during the
|
|
comparison.
|
|
"""
|
|
return self._skipped
|
|
|
|
@property
|
|
def total_differences(self):
|
|
"""Returns the total number of differences detected."""
|
|
return self._total_diffs
|
|
|
|
@property
|
|
def missing_in_gyp(self):
|
|
"""Differences that are only in GYP build but not in GN, indexed by the
|
|
difference."""
|
|
return self._missing_gyp_flags
|
|
|
|
@property
|
|
def missing_in_gn(self):
|
|
"""Differences that are only in the GN build but not in GYP, indexed by
|
|
the difference."""
|
|
return self._missing_gn_flags
|
|
|
|
@property
|
|
def missing_in_gyp_by_file(self):
|
|
"""Differences that are only in the GYP build but not in GN, indexed by
|
|
file.
|
|
"""
|
|
return self._missing_gyp_files
|
|
|
|
@property
|
|
def missing_in_gn_by_file(self):
|
|
"""Differences that are only in the GYP build but not in GN, indexed by
|
|
file.
|
|
"""
|
|
return self._missing_gn_files
|
|
|
|
def _CompareFiles(self):
|
|
"""Performs the actual target comparison."""
|
|
if sys.platform == 'win32':
|
|
# On Windows flags are stored in .rsp files which are created by building.
|
|
print >> sys.stderr, 'Building in %s...' % self._gn_dir
|
|
Run('ninja -C %s -d keeprsp %s' % (self._gn_dir, self._gn_target))
|
|
print >> sys.stderr, 'Building in %s...' % self._gyp_dir
|
|
Run('ninja -C %s -d keeprsp %s' % (self._gyp_dir, self._gn_target))
|
|
|
|
gn = Run('ninja -C %s -t commands %s' % (self._gn_dir, self._gn_target))
|
|
gyp = Run('ninja -C %s -t commands %s' % (self._gyp_dir, self._gyp_target))
|
|
|
|
self._gn_flags = self._GetFlags(gn.splitlines(),
|
|
os.path.join(os.getcwd(), self._gn_dir))
|
|
self._gyp_flags = self._GetFlags(gyp.splitlines(),
|
|
os.path.join(os.getcwd(), self._gyp_dir))
|
|
|
|
self._gn_flags = dict((FilterChromium(filename), value)
|
|
for filename, value in self._gn_flags.iteritems())
|
|
self._gyp_flags = dict((FilterChromium(filename), value)
|
|
for filename, value in self._gyp_flags.iteritems())
|
|
|
|
all_files = sorted(self.gn_files & self.gyp_files)
|
|
for filename in all_files:
|
|
gyp_flags = self._gyp_flags[filename]
|
|
gn_flags = self._gn_flags[filename]
|
|
self._CompareLists(filename, gyp_flags, gn_flags, 'dash_f')
|
|
self._CompareLists(filename, gyp_flags, gn_flags, 'defines',
|
|
# These defines are not used by WebRTC
|
|
dont_care_gyp=[
|
|
'-DENABLE_WEBVR',
|
|
'-DUSE_EXTERNAL_POPUP_MENU',
|
|
'-DUSE_LIBJPEG_TURBO=1',
|
|
'-DUSE_MINIKIN_HYPHENATION=1',
|
|
'-DV8_USE_EXTERNAL_STARTUP_DATA',
|
|
'-DCR_CLANG_REVISION=280106-1',
|
|
'-DUSE_LIBPCI=1'
|
|
],
|
|
dont_care_gn=[
|
|
'-DUSE_EXTERNAL_POPUP_MENU=1'
|
|
])
|
|
self._CompareLists(filename, gyp_flags, gn_flags, 'include_dirs')
|
|
self._CompareLists(filename, gyp_flags, gn_flags, 'warnings',
|
|
# More conservative warnings in GN we consider to be OK.
|
|
dont_care_gyp=[
|
|
'/wd4091', # 'keyword' : ignored on left of 'type' when no variable
|
|
# is declared.
|
|
'/wd4456', # Declaration hides previous local declaration.
|
|
'/wd4457', # Declaration hides function parameter.
|
|
'/wd4458', # Declaration hides class member.
|
|
'/wd4459', # Declaration hides global declaration.
|
|
'/wd4702', # Unreachable code.
|
|
'/wd4800', # Forcing value to bool 'true' or 'false'.
|
|
'/wd4838', # Conversion from 'type' to 'type' requires a narrowing
|
|
# conversion.
|
|
] if sys.platform == 'win32' else None,
|
|
dont_care_gn=[
|
|
'-Wendif-labels',
|
|
'-Wextra',
|
|
'-Wsign-compare',
|
|
] if not sys.platform == 'win32' else None)
|
|
self._CompareLists(filename, gyp_flags, gn_flags, 'other',
|
|
dont_care_gyp=['-g'], dont_care_gn=['-g2'])
|
|
|
|
def _CompareLists(self, filename, gyp, gn, name,
|
|
dont_care_gyp=None, dont_care_gn=None):
|
|
"""Return a report of any differences between gyp and gn lists, ignoring
|
|
anything in |dont_care_{gyp|gn}| respectively."""
|
|
if gyp[name] == gn[name]:
|
|
return
|
|
if not dont_care_gyp:
|
|
dont_care_gyp = []
|
|
if not dont_care_gn:
|
|
dont_care_gn = []
|
|
gyp_set = set(gyp[name])
|
|
gn_set = set(gn[name])
|
|
missing_in_gyp = gyp_set - gn_set
|
|
missing_in_gn = gn_set - gyp_set
|
|
missing_in_gyp -= set(dont_care_gyp)
|
|
missing_in_gn -= set(dont_care_gn)
|
|
|
|
for m in missing_in_gyp:
|
|
self._missing_gyp_flags.setdefault(name, {}) \
|
|
.setdefault(m, []).append(filename)
|
|
self._total_diffs += 1
|
|
self._missing_gyp_files.setdefault(filename, {}) \
|
|
.setdefault(name, set()).update(missing_in_gyp)
|
|
|
|
for m in missing_in_gn:
|
|
self._missing_gn_flags.setdefault(name, {}) \
|
|
.setdefault(m, []).append(filename)
|
|
self._total_diffs += 1
|
|
self._missing_gn_files.setdefault(filename, {}) \
|
|
.setdefault(name, set()).update(missing_in_gn)
|
|
|
|
def _GetFlags(self, lines, build_dir):
|
|
"""Turn a list of command lines into a semi-structured dict."""
|
|
is_win = sys.platform == 'win32'
|
|
flags_by_output = {}
|
|
for line in lines:
|
|
line = FilterChromium(line)
|
|
line = line.replace(os.getcwd(), '../../')
|
|
line = line.replace('//', '/')
|
|
command_line = shlex.split(line.strip(), posix=not is_win)[1:]
|
|
|
|
output_name = _FindAndRemoveArgWithValue(command_line, '-o')
|
|
dep_name = _FindAndRemoveArgWithValue(command_line, '-MF')
|
|
|
|
command_line = _MergeSpacedArgs(command_line, '-Xclang')
|
|
|
|
cc_file = [x for x in command_line if x.endswith('.cc') or
|
|
x.endswith('.c') or
|
|
x.endswith('.cpp') or
|
|
x.endswith('.mm') or
|
|
x.endswith('.m')]
|
|
if len(cc_file) != 1:
|
|
self._skipped.append(command_line)
|
|
continue
|
|
assert len(cc_file) == 1
|
|
|
|
if is_win:
|
|
rsp_file = [x for x in command_line if x.endswith('.rsp')]
|
|
assert len(rsp_file) <= 1
|
|
if rsp_file:
|
|
rsp_file = os.path.join(build_dir, rsp_file[0][1:])
|
|
with open(rsp_file, "r") as open_rsp_file:
|
|
command_line = shlex.split(open_rsp_file, posix=False)
|
|
|
|
defines = [x for x in command_line if x.startswith('-D')]
|
|
include_dirs = [x for x in command_line if x.startswith('-I')]
|
|
dash_f = [x for x in command_line if x.startswith('-f')]
|
|
warnings = \
|
|
[x for x in command_line if x.startswith('/wd' if is_win else '-W')]
|
|
others = [x for x in command_line if x not in defines and \
|
|
x not in include_dirs and \
|
|
x not in dash_f and \
|
|
x not in warnings and \
|
|
x not in cc_file]
|
|
|
|
for index, value in enumerate(include_dirs):
|
|
if value == '-Igen':
|
|
continue
|
|
path = value[2:]
|
|
if not os.path.isabs(path):
|
|
path = os.path.join(build_dir, path)
|
|
include_dirs[index] = '-I' + os.path.normpath(path)
|
|
|
|
# GYP supports paths above the source root like <(DEPTH)/../foo while such
|
|
# paths are unsupported by gn. But gn allows to use system-absolute paths
|
|
# instead (paths that start with single '/'). Normalize all paths.
|
|
cc_file = [os.path.normpath(os.path.join(build_dir, cc_file[0]))]
|
|
|
|
# Filter for libFindBadConstructs.so having a relative path in one and
|
|
# absolute path in the other.
|
|
others_filtered = []
|
|
for x in others:
|
|
if x.startswith('-Xclang ') and \
|
|
(x.endswith('libFindBadConstructs.so') or \
|
|
x.endswith('libFindBadConstructs.dylib')):
|
|
others_filtered.append(
|
|
'-Xclang ' +
|
|
os.path.join(os.getcwd(), os.path.normpath(
|
|
os.path.join('out/gn_flags', x.split(' ', 1)[1]))))
|
|
elif x.startswith('-B'):
|
|
others_filtered.append(
|
|
'-B' +
|
|
os.path.join(os.getcwd(), os.path.normpath(
|
|
os.path.join('out/gn_flags', x[2:]))))
|
|
else:
|
|
others_filtered.append(x)
|
|
others = others_filtered
|
|
|
|
flags_by_output[cc_file[0]] = {
|
|
'output': output_name,
|
|
'depname': dep_name,
|
|
'defines': sorted(defines),
|
|
'include_dirs': sorted(include_dirs), # TODO(scottmg): This is wrong.
|
|
'dash_f': sorted(dash_f),
|
|
'warnings': sorted(warnings),
|
|
'other': sorted(others),
|
|
}
|
|
return flags_by_output
|
|
|
|
|
|
def _FindAndRemoveArgWithValue(command_line, argname):
|
|
"""Given a command line as a list, remove and return the value of an option
|
|
that takes a value as a separate entry.
|
|
|
|
Modifies |command_line| in place.
|
|
"""
|
|
if argname not in command_line:
|
|
return ''
|
|
location = command_line.index(argname)
|
|
value = command_line[location + 1]
|
|
command_line[location:location + 2] = []
|
|
return value
|
|
|
|
|
|
def _MergeSpacedArgs(command_line, argname):
|
|
"""Combine all arguments |argname| with their values, separated by a space."""
|
|
i = 0
|
|
result = []
|
|
while i < len(command_line):
|
|
arg = command_line[i]
|
|
if arg == argname:
|
|
result.append(arg + ' ' + command_line[i + 1])
|
|
i += 1
|
|
else:
|
|
result.append(arg)
|
|
i += 1
|
|
return result
|
|
|
|
|
|
def Run(command_line):
|
|
"""Run |command_line| as a subprocess and return stdout. Raises on error."""
|
|
print >> sys.stderr, command_line
|
|
return subprocess.check_output(command_line, shell=True)
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 3:
|
|
print 'usage: %s gyp_dir gn_dir' % __file__
|
|
print ' or: %s gyp_dir gn_dir target' % __file__
|
|
print ' or: %s gyp_dir gn_dir gyp_target gn_target' % __file__
|
|
return 1
|
|
|
|
gyp_dir = sys.argv[1]
|
|
gn_dir = sys.argv[2]
|
|
|
|
gyp_target = gn_target = ""
|
|
|
|
if len(sys.argv) > 3:
|
|
gyp_target = sys.argv[3]
|
|
if len(sys.argv) > 4:
|
|
gn_target = sys.argv[4]
|
|
|
|
print 'GYP output directory is %s' % gyp_dir
|
|
print 'GN output directory is %s' % gn_dir
|
|
|
|
comparison = Comparison(gyp_target, gn_target, gyp_dir, gn_dir)
|
|
|
|
differing_files = set(comparison.missing_in_gn_by_file.keys()) & \
|
|
set(comparison.missing_in_gyp_by_file.keys())
|
|
files_with_given_differences = {}
|
|
for filename in differing_files:
|
|
output = ''
|
|
missing_in_gyp = comparison.missing_in_gyp_by_file.get(filename, {})
|
|
missing_in_gn = comparison.missing_in_gn_by_file.get(filename, {})
|
|
difference_types = sorted(set(missing_in_gyp.keys() + missing_in_gn.keys()))
|
|
for difference_type in difference_types:
|
|
if (len(missing_in_gyp[difference_type]) == 0 and
|
|
len(missing_in_gn[difference_type]) == 0):
|
|
continue
|
|
output += ' %s differ:\n' % difference_type
|
|
if (difference_type in missing_in_gyp and
|
|
len(missing_in_gyp[difference_type])):
|
|
output += ' In gyp, but not in GN:\n %s' % '\n '.join(
|
|
sorted(missing_in_gyp[difference_type])) + '\n'
|
|
if (difference_type in missing_in_gn and
|
|
len(missing_in_gn[difference_type])):
|
|
output += ' In GN, but not in gyp:\n %s' % '\n '.join(
|
|
sorted(missing_in_gn[difference_type])) + '\n'
|
|
if output:
|
|
files_with_given_differences.setdefault(output, []).append(filename)
|
|
|
|
for diff, files in files_with_given_differences.iteritems():
|
|
print '\n'.join(sorted(files))
|
|
print diff
|
|
|
|
print 'Total differences:', comparison.total_differences
|
|
# TODO(scottmg): Return failure on difference once we're closer to identical.
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|