tools_webrtc dir converted to py3 + top level PRESUBMIT script
Bug: webrtc:13607 Change-Id: Ib018e43ea977cc24dd71048e68e3343741f7f31b Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/249083 Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org> Reviewed-by: Harald Alvestrand <hta@webrtc.org> Reviewed-by: Jeremy Leconte <jleconte@google.com> Commit-Queue: Christoffer Jansson <jansson@google.com> Cr-Commit-Position: refs/heads/main@{#35953}
This commit is contained in:
Christoffer Jansson
committed by
WebRTC LUCI CQ
WebRTC LUCI CQ
parent
b5cba85c2f
commit
4e8a773b4b
@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env python3
|
||||
#!/usr/bin/env vpython3
|
||||
|
||||
# Copyright 2016 The WebRTC project authors. All Rights Reserved.
|
||||
#
|
||||
@ -13,7 +13,8 @@ Licenses are taken from dependent libraries which are determined by
|
||||
GN desc command `gn desc` on all targets specified via `--target` argument.
|
||||
|
||||
One can see all dependencies by invoking this command:
|
||||
$ gn.py desc --all --format=json <out_directory> <target> | python -m json.tool
|
||||
$ gn.py desc --all --format=json <out_directory> <target> | \
|
||||
vpython3 -m json.tool
|
||||
(see "deps" subarray)
|
||||
|
||||
Libraries are mapped to licenses via LIB_TO_LICENSES_DICT dictionary.
|
||||
@ -21,18 +22,13 @@ Libraries are mapped to licenses via LIB_TO_LICENSES_DICT dictionary.
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
try:
|
||||
# python 3.2+
|
||||
from html import escape
|
||||
except ImportError:
|
||||
from cgi import escape
|
||||
from html import escape
|
||||
|
||||
# Third_party library to licences mapping. Keys are names of the libraries
|
||||
# (right after the `third_party/` prefix)
|
||||
@ -107,11 +103,11 @@ LIB_REGEX_TO_LICENSES_DICT = {
|
||||
|
||||
|
||||
def FindSrcDirPath():
|
||||
"""Returns the abs path to the src/ dir of the project."""
|
||||
src_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
while os.path.basename(src_dir) != 'src':
|
||||
src_dir = os.path.normpath(os.path.join(src_dir, os.pardir))
|
||||
return src_dir
|
||||
"""Returns the abs path to the src/ dir of the project."""
|
||||
src_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
while os.path.basename(src_dir) != 'src':
|
||||
src_dir = os.path.normpath(os.path.join(src_dir, os.pardir))
|
||||
return src_dir
|
||||
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))
|
||||
@ -124,29 +120,29 @@ THIRD_PARTY_LIB_SIMPLE_NAME_REGEX = r'^.*/third_party/([\w\-+]+).*$'
|
||||
THIRD_PARTY_LIB_REGEX_TEMPLATE = r'^.*/third_party/%s$'
|
||||
|
||||
|
||||
class LicenseBuilder(object):
|
||||
def __init__(self,
|
||||
buildfile_dirs,
|
||||
targets,
|
||||
lib_to_licenses_dict=None,
|
||||
lib_regex_to_licenses_dict=None):
|
||||
if lib_to_licenses_dict is None:
|
||||
lib_to_licenses_dict = LIB_TO_LICENSES_DICT
|
||||
class LicenseBuilder:
|
||||
def __init__(self,
|
||||
buildfile_dirs,
|
||||
targets,
|
||||
lib_to_licenses_dict=None,
|
||||
lib_regex_to_licenses_dict=None):
|
||||
if lib_to_licenses_dict is None:
|
||||
lib_to_licenses_dict = LIB_TO_LICENSES_DICT
|
||||
|
||||
if lib_regex_to_licenses_dict is None:
|
||||
lib_regex_to_licenses_dict = LIB_REGEX_TO_LICENSES_DICT
|
||||
if lib_regex_to_licenses_dict is None:
|
||||
lib_regex_to_licenses_dict = LIB_REGEX_TO_LICENSES_DICT
|
||||
|
||||
self.buildfile_dirs = buildfile_dirs
|
||||
self.targets = targets
|
||||
self.lib_to_licenses_dict = lib_to_licenses_dict
|
||||
self.lib_regex_to_licenses_dict = lib_regex_to_licenses_dict
|
||||
self.buildfile_dirs = buildfile_dirs
|
||||
self.targets = targets
|
||||
self.lib_to_licenses_dict = lib_to_licenses_dict
|
||||
self.lib_regex_to_licenses_dict = lib_regex_to_licenses_dict
|
||||
|
||||
self.common_licenses_dict = self.lib_to_licenses_dict.copy()
|
||||
self.common_licenses_dict.update(self.lib_regex_to_licenses_dict)
|
||||
self.common_licenses_dict = self.lib_to_licenses_dict.copy()
|
||||
self.common_licenses_dict.update(self.lib_regex_to_licenses_dict)
|
||||
|
||||
@staticmethod
|
||||
def _ParseLibraryName(dep):
|
||||
"""Returns library name after third_party
|
||||
@staticmethod
|
||||
def _ParseLibraryName(dep):
|
||||
"""Returns library name after third_party
|
||||
|
||||
Input one of:
|
||||
//a/b/third_party/libname:c
|
||||
@ -155,11 +151,11 @@ class LicenseBuilder(object):
|
||||
|
||||
Outputs libname or None if this is not a third_party dependency.
|
||||
"""
|
||||
groups = re.match(THIRD_PARTY_LIB_SIMPLE_NAME_REGEX, dep)
|
||||
return groups.group(1) if groups else None
|
||||
groups = re.match(THIRD_PARTY_LIB_SIMPLE_NAME_REGEX, dep)
|
||||
return groups.group(1) if groups else None
|
||||
|
||||
def _ParseLibrary(self, dep):
|
||||
"""Returns library simple or regex name that matches `dep` after third_party
|
||||
def _ParseLibrary(self, dep):
|
||||
"""Returns library simple or regex name that matches `dep` after third_party
|
||||
|
||||
This method matches `dep` dependency against simple names in
|
||||
LIB_TO_LICENSES_DICT and regular expression names in
|
||||
@ -167,109 +163,104 @@ class LicenseBuilder(object):
|
||||
|
||||
Outputs matched dict key or None if this is not a third_party dependency.
|
||||
"""
|
||||
libname = LicenseBuilder._ParseLibraryName(dep)
|
||||
libname = LicenseBuilder._ParseLibraryName(dep)
|
||||
|
||||
for lib_regex in self.lib_regex_to_licenses_dict:
|
||||
if re.match(THIRD_PARTY_LIB_REGEX_TEMPLATE % lib_regex, dep):
|
||||
return lib_regex
|
||||
for lib_regex in self.lib_regex_to_licenses_dict:
|
||||
if re.match(THIRD_PARTY_LIB_REGEX_TEMPLATE % lib_regex, dep):
|
||||
return lib_regex
|
||||
|
||||
return libname
|
||||
return libname
|
||||
|
||||
@staticmethod
|
||||
def _RunGN(buildfile_dir, target):
|
||||
cmd = [
|
||||
sys.executable,
|
||||
os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py'),
|
||||
'desc',
|
||||
'--all',
|
||||
'--format=json',
|
||||
os.path.abspath(buildfile_dir),
|
||||
target,
|
||||
]
|
||||
logging.debug('Running: %r', cmd)
|
||||
output_json = subprocess.check_output(cmd, cwd=WEBRTC_ROOT).decode('UTF-8')
|
||||
logging.debug('Output: %s', output_json)
|
||||
return output_json
|
||||
@staticmethod
|
||||
def _RunGN(buildfile_dir, target):
|
||||
cmd = [
|
||||
sys.executable,
|
||||
os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py'),
|
||||
'desc',
|
||||
'--all',
|
||||
'--format=json',
|
||||
os.path.abspath(buildfile_dir),
|
||||
target,
|
||||
]
|
||||
logging.debug('Running: %r', cmd)
|
||||
output_json = subprocess.check_output(cmd, cwd=WEBRTC_ROOT).decode('UTF-8')
|
||||
logging.debug('Output: %s', output_json)
|
||||
return output_json
|
||||
|
||||
def _GetThirdPartyLibraries(self, buildfile_dir, target):
|
||||
output = json.loads(LicenseBuilder._RunGN(buildfile_dir, target))
|
||||
libraries = set()
|
||||
for described_target in output.values():
|
||||
third_party_libs = (self._ParseLibrary(dep)
|
||||
for dep in described_target['deps'])
|
||||
libraries |= set(lib for lib in third_party_libs if lib)
|
||||
return libraries
|
||||
def _GetThirdPartyLibraries(self, buildfile_dir, target):
|
||||
output = json.loads(LicenseBuilder._RunGN(buildfile_dir, target))
|
||||
libraries = set()
|
||||
for described_target in list(output.values()):
|
||||
third_party_libs = (self._ParseLibrary(dep)
|
||||
for dep in described_target['deps'])
|
||||
libraries |= set(lib for lib in third_party_libs if lib)
|
||||
return libraries
|
||||
|
||||
def GenerateLicenseText(self, output_dir):
|
||||
# Get a list of third_party libs from gn. For fat libraries we must consider
|
||||
# all architectures, hence the multiple buildfile directories.
|
||||
third_party_libs = set()
|
||||
for buildfile in self.buildfile_dirs:
|
||||
for target in self.targets:
|
||||
third_party_libs |= self._GetThirdPartyLibraries(
|
||||
buildfile, target)
|
||||
assert len(third_party_libs) > 0
|
||||
def GenerateLicenseText(self, output_dir):
|
||||
# Get a list of third_party libs from gn. For fat libraries we must consider
|
||||
# all architectures, hence the multiple buildfile directories.
|
||||
third_party_libs = set()
|
||||
for buildfile in self.buildfile_dirs:
|
||||
for target in self.targets:
|
||||
third_party_libs |= self._GetThirdPartyLibraries(buildfile, target)
|
||||
assert len(third_party_libs) > 0
|
||||
|
||||
missing_licenses = third_party_libs - set(
|
||||
self.common_licenses_dict.keys())
|
||||
if missing_licenses:
|
||||
error_msg = 'Missing licenses for following third_party targets: %s' % \
|
||||
', '.join(sorted(missing_licenses))
|
||||
logging.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
missing_licenses = third_party_libs - set(self.common_licenses_dict.keys())
|
||||
if missing_licenses:
|
||||
error_msg = 'Missing licenses for following third_party targets: %s' % \
|
||||
', '.join(sorted(missing_licenses))
|
||||
logging.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
# Put webrtc at the front of the list.
|
||||
license_libs = sorted(third_party_libs)
|
||||
license_libs.insert(0, 'webrtc')
|
||||
# Put webrtc at the front of the list.
|
||||
license_libs = sorted(third_party_libs)
|
||||
license_libs.insert(0, 'webrtc')
|
||||
|
||||
logging.info('List of licenses: %s', ', '.join(license_libs))
|
||||
logging.info('List of licenses: %s', ', '.join(license_libs))
|
||||
|
||||
# Generate markdown.
|
||||
output_license_file = open(os.path.join(output_dir, 'LICENSE.md'),
|
||||
'w+')
|
||||
for license_lib in license_libs:
|
||||
if len(self.common_licenses_dict[license_lib]) == 0:
|
||||
logging.info(
|
||||
'Skipping compile time or internal dependency: %s',
|
||||
license_lib)
|
||||
continue # Compile time dependency
|
||||
# Generate markdown.
|
||||
output_license_file = open(os.path.join(output_dir, 'LICENSE.md'), 'w+')
|
||||
for license_lib in license_libs:
|
||||
if len(self.common_licenses_dict[license_lib]) == 0:
|
||||
logging.info('Skipping compile time or internal dependency: %s',
|
||||
license_lib)
|
||||
continue # Compile time dependency
|
||||
|
||||
output_license_file.write('# %s\n' % license_lib)
|
||||
output_license_file.write('```\n')
|
||||
for path in self.common_licenses_dict[license_lib]:
|
||||
license_path = os.path.join(WEBRTC_ROOT, path)
|
||||
with open(license_path, 'r') as license_file:
|
||||
license_text = escape(license_file.read(), quote=True)
|
||||
output_license_file.write(license_text)
|
||||
output_license_file.write('\n')
|
||||
output_license_file.write('```\n\n')
|
||||
output_license_file.write('# %s\n' % license_lib)
|
||||
output_license_file.write('```\n')
|
||||
for path in self.common_licenses_dict[license_lib]:
|
||||
license_path = os.path.join(WEBRTC_ROOT, path)
|
||||
with open(license_path, 'r') as license_file:
|
||||
license_text = escape(license_file.read(), quote=True)
|
||||
output_license_file.write(license_text)
|
||||
output_license_file.write('\n')
|
||||
output_license_file.write('```\n\n')
|
||||
|
||||
output_license_file.close()
|
||||
output_license_file.close()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Generate WebRTC LICENSE.md')
|
||||
parser.add_argument('--verbose',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Debug logging.')
|
||||
parser.add_argument('--target',
|
||||
required=True,
|
||||
action='append',
|
||||
default=[],
|
||||
help='Name of the GN target to generate a license for')
|
||||
parser.add_argument('output_dir',
|
||||
help='Directory to output LICENSE.md to.')
|
||||
parser.add_argument('buildfile_dirs',
|
||||
nargs='+',
|
||||
help='Directories containing gn generated ninja files')
|
||||
args = parser.parse_args()
|
||||
parser = argparse.ArgumentParser(description='Generate WebRTC LICENSE.md')
|
||||
parser.add_argument('--verbose',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Debug logging.')
|
||||
parser.add_argument('--target',
|
||||
required=True,
|
||||
action='append',
|
||||
default=[],
|
||||
help='Name of the GN target to generate a license for')
|
||||
parser.add_argument('output_dir', help='Directory to output LICENSE.md to.')
|
||||
parser.add_argument('buildfile_dirs',
|
||||
nargs='+',
|
||||
help='Directories containing gn generated ninja files')
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
|
||||
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
|
||||
|
||||
builder = LicenseBuilder(args.buildfile_dirs, args.target)
|
||||
builder.GenerateLicenseText(args.output_dir)
|
||||
builder = LicenseBuilder(args.buildfile_dirs, args.target)
|
||||
builder.GenerateLicenseText(args.output_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
sys.exit(main())
|
||||
|
||||
Reference in New Issue
Block a user