Change structure of deps file and tool for adding chromium dep.

Change structure of chromium owned dependencies file to JSON to
simplify work with it in different tools. Also add tool to check in
new chromium owned dep with single command like this:
./tools_webrtc/autoroller/checkin_chromium_dep.py -d <dep name>

Introduce separate file with list of webrtc owned dependencies.



Bug: webrtc:8366
Change-Id: I30a828af34cd105ce7e6bc76d6b5889e6bf7574d
Reviewed-on: https://webrtc-review.googlesource.com/76840
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23270}
This commit is contained in:
Artem Titov
2018-05-17 10:00:20 +02:00
committed by Commit Bot
parent ff1de0af6b
commit c47c9c0234
17 changed files with 490 additions and 126 deletions

View File

@ -752,6 +752,7 @@ def CommonChecks(input_api, output_api):
r'^out.*[\\\/].*\.py$',
r'^testing[\\\/].*\.py$',
r'^third_party[\\\/].*\.py$',
r'^third_party_chromium[\\\/].*\.py$',
r'^tools[\\\/].*\.py$',
# TODO(phoglund): should arguably be checked.
r'^tools_webrtc[\\\/]mb[\\\/].*\.py$',
@ -770,7 +771,8 @@ def CommonChecks(input_api, output_api):
# Skip long-lines check for DEPS and GN files.
build_file_filter_list = (r'.+\.gn$', r'.+\.gni$', 'DEPS')
# Also we will skip most checks for third_party directory.
third_party_filter_list = (r'^third_party[\\\/].+',)
third_party_filter_list = (r'^third_party[\\\/].+',
r'^third_party_chromium[\\\/].+')
eighty_char_sources = lambda x: input_api.FilterSourceFile(x,
black_list=build_file_filter_list + objc_filter_list +
third_party_filter_list)

View File

@ -0,0 +1,79 @@
{
"comment": [
"This file determines which parts of Chromium's third_party/ we copy.",
"",
"If you want to add new chromium owned dependency, run",
"./tools_webrtc/autoroller/checkin_chromium_dep.py -d <dep name>",
"",
"It will add specified dependency to third_party directory and will add it to this list",
"properly."
],
"dependencies": [
"abseil-cpp",
"binutils",
"boringssl",
"ced",
"freetype",
"googletest",
"harfbuzz-ng",
"instrumented_libraries",
"jsoncpp",
"libFuzzer",
"libpng",
"libvpx",
"mockito",
"openh264",
"opus",
"protobuf",
"pymock",
"requests",
"rnnoise",
"usrsctp",
"yasm",
"zlib",
"colorama",
"accessibility_test_framework",
"android_platform",
"android_support_test_runner",
"apk-patch-size-estimator",
"ashmem",
"auto",
"bazel",
"bouncycastle",
"breakpad",
"byte_buddy",
"closure_compiler",
"errorprone",
"espresso",
"eu-strip",
"gson",
"guava",
"hamcrest",
"icu4j",
"ijar",
"intellij",
"javax_inject",
"jinja2",
"jsr-305",
"junit",
"libxml",
"markupsafe",
"modp_b64",
"objenesis",
"ow2_asm",
"robolectric",
"sqlite4java",
"tcmalloc",
"ub-uiautomator",
"xstream",
"proguard",
"android_system_sdk",
"ocmock",
"BUILD.gn",
"DEPS",
"libjpeg.gni",
"PRESUBMIT.py",
"README.chromium",
"README.chromium.template"
]
}

View File

@ -1,118 +0,0 @@
# This file determines which parts of Chromium's third_party/ we copy.
# If a dependency is missing, add <dep> to the DEPS list in order to start
# copying third_party/<dep> from Chromium's third_party.
#
# Test your change like this:
# 1. Edit this file
# 2. Add
# "custom_vars": {
# "roll_chromium_into_webrtc": True,
# },
# to the WebRTC solution in your gclient.
# 3. gclient sync --deps=all
# 4. python tools_webrtc/autoroller/roll_deps.py --dry-run --ignore-unclean-workdir
# 5. git checkout -- . # Throw away rolls of existing deps.
# 6. You should now get untracked files in third_party/<dep>.
# 7. git add third_party/<dep>
# 8. git commit -am "Add third_party/<dep> to WebRTC"
# 9. git cl upload
DEPS = [
# Common
'abseil-cpp',
'binutils',
'boringssl',
'ced',
'freetype',
'googletest',
'harfbuzz-ng',
'instrumented_libraries',
'jsoncpp',
'libFuzzer',
'libpng',
'libvpx',
'mockito',
'openh264',
'opus',
'protobuf',
'pymock',
'requests',
'rnnoise',
'usrsctp',
'yasm',
'zlib',
'colorama',
# These common deps will be synced by gclient:
#'depot_tools',
#'ffmpeg',
#'icu',
#'libjpeg_turbo',
#'libsrtp',
#'libyuv',
#'llvm-build',
#'lss',
#'openmax_dl',
#'catapult',
#'gtest-parallel',
# Windows specific deps will be synced by gclient:
#'syzygy',
#'winsdk_samples',
# Android specific deps
# compile time deps
'accessibility_test_framework',
'android_platform',
'android_support_test_runner',
'apk-patch-size-estimator',
'ashmem',
'auto',
'bazel',
'bouncycastle',
'breakpad',
'byte_buddy',
'closure_compiler',
'errorprone',
'espresso',
'eu-strip',
'gson',
'guava',
'hamcrest',
'icu4j',
'ijar',
'intellij',
'javax_inject',
'jinja2',
'jsr-305',
'junit',
'libxml',
'markupsafe',
'modp_b64',
'objenesis',
'ow2_asm',
'robolectric',
'sqlite4java',
'tcmalloc',
'ub-uiautomator',
'xstream',
# test time deps
'proguard',
'android_system_sdk',
# These Android specific deps will be synced by gclient:
#'android_ndk',
#'android_tools',
#'findbugs',
# Mac and iOS specific deps
'ocmock',
# List of files to sync
'BUILD.gn',
'DEPS',
'libjpeg.gni',
'PRESUBMIT.py',
'README.chromium',
'README.chromium.template',
]

View File

@ -0,0 +1,9 @@
{
"comment": [
"This file contains list of third party dependencies owned by WebRTC directly,",
"e.g. they're not from Chromium's third party (those are tracked in ",
"THIRD_PARTY_CHROMIUM_DEPS.json)."
],
"dependencies": [
]
}

View File

@ -0,0 +1,246 @@
#!/usr/bin/env python
# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""
Script to automatically add specified chromium dependency to WebRTC repo.
If you want to add new chromium owned dependency foo to the WebRTC repo
you have to run this tool like this:
./checkin_chromium_deps.py -d foo
It will check in dependency into third_party directory and will add it into
git index. Also it will update chromium dependencies list with new dependency
to insure that it will be correctly auto updated in future.
"""
import argparse
import errno
import json
import logging
import os.path
import shutil
import subprocess
import sys
import tempfile
REMOTE_URL = 'https://chromium.googlesource.com/chromium/src/third_party'
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
os.pardir))
class DependencyAlreadyCheckedIn(Exception):
pass
class DependencyNotFound(Exception):
pass
class Config(object):
def __init__(self, src_root, remote_url, temp_dir):
self.src_root = src_root
self.dependencies_file = os.path.join(self.src_root,
'THIRD_PARTY_CHROMIUM_DEPS.json')
self.deps_file = os.path.join(self.src_root, 'DEPS')
self.third_party_dir = os.path.join(self.src_root, 'third_party')
self.remote_url = remote_url
self.temp_dir = temp_dir
def VarLookup(local_scope):
return lambda var_name: local_scope['vars'][var_name]
def ParseDepsDict(deps_content):
local_scope = {}
global_scope = {
'Var': VarLookup(local_scope),
'deps_os': {},
}
exec (deps_content, global_scope, local_scope)
return local_scope
def ParseLocalDepsFile(filename):
with open(filename, 'rb') as f:
deps_content = f.read()
return ParseDepsDict(deps_content)
def RunCommand(command, working_dir=None, ignore_exit_code=False,
extra_env=None, input_data=None):
"""Runs a command and returns the output from that command.
If the command fails (exit code != 0), the function will exit the process.
Returns:
A tuple containing the stdout and stderr outputs as strings.
"""
working_dir = working_dir or CHECKOUT_SRC_DIR
logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
env = os.environ.copy()
if extra_env:
assert all(type(value) == str for value in extra_env.values())
logging.debug('extra env: %s', extra_env)
env.update(extra_env)
p = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env,
cwd=working_dir, universal_newlines=True)
std_output, err_output = p.communicate(input_data)
p.stdout.close()
p.stderr.close()
if not ignore_exit_code and p.returncode != 0:
logging.error('Command failed: %s\n'
'stdout:\n%s\n'
'stderr:\n%s\n', ' '.join(command), std_output, err_output)
sys.exit(p.returncode)
return std_output, err_output
def LoadThirdPartyRevision(deps_file):
logging.debug('Loading chromium third_party revision from %s', deps_file)
webrtc_deps = ParseLocalDepsFile(deps_file)
return webrtc_deps['vars']['chromium_third_party_revision']
def CheckoutRequiredDependency(dep_name, config):
third_party_revision = LoadThirdPartyRevision(config.deps_file)
logging.debug('Initializing git repo in %s...', config.temp_dir)
RunCommand(['git', 'init'], working_dir=config.temp_dir)
logging.debug('Adding remote to %s. It may take some time...',
config.remote_url)
RunCommand(['git', 'remote', 'add', '-f', 'origin', config.remote_url],
working_dir=config.temp_dir)
logging.debug('Configuring sparse checkout...')
RunCommand(['git', 'config', 'core.sparseCheckout', 'true'],
working_dir=config.temp_dir)
sparse_checkout_config_path = os.path.join(config.temp_dir, '.git', 'info',
'sparse-checkout')
with open(sparse_checkout_config_path, 'wb') as f:
f.write(dep_name)
logging.debug('Pulling changes...')
_, stderr = RunCommand(['git', 'pull', 'origin', 'master'],
working_dir=config.temp_dir,
ignore_exit_code=True)
if "Sparse checkout leaves no entry on working directory" in stderr:
# There are no such dependency in chromium third_party
raise DependencyNotFound(
"Dependency %s not found in chromium repo" % dep_name)
logging.debug('Switching to revision %s...', third_party_revision)
RunCommand(['git', 'checkout', third_party_revision],
working_dir=config.temp_dir)
return os.path.join(config.temp_dir, dep_name)
def CopyDependency(dep_name, source_path, third_party_dir):
dest_path = os.path.join(third_party_dir, dep_name)
logging.debug('Copying dependency from %s to %s...', source_path, dest_path)
shutil.copytree(source_path, dest_path)
def AppendToChromiumOwnedDependenciesList(dep_name, dep_file):
with open(dep_file, 'rb') as f:
data = json.load(f)
dep_list = data.get('dependencies', [])
dep_list.append(dep_name)
data['dependencies'] = dep_list
with open(dep_file, 'wb') as f:
json.dump(data, f, indent=2, sort_keys=True, separators=(',', ': '))
def AddToGitIndex(dep_name, config):
logging.debug('Adding required changes to git index and commit set...')
dest_path = os.path.join(config.third_party_dir, dep_name)
RunCommand(['git', 'add', dest_path], working_dir=config.src_root)
RunCommand(['git', 'add', config.dependencies_file],
working_dir=config.src_root)
def CheckinDependency(dep_name, config):
dep_path = CheckoutRequiredDependency(dep_name, config)
CopyDependency(dep_name, dep_path, config.third_party_dir)
AppendToChromiumOwnedDependenciesList(dep_name, config.dependencies_file)
AddToGitIndex(dep_name, config)
logging.info('Dependency checked into current working tree and added into\n'
'git index. You have to commit generated changes and\n'
'file the CL to finish adding the dependency')
def DefaultConfig(temp_dir):
return Config(CHECKOUT_SRC_DIR, REMOTE_URL, temp_dir)
def CheckinDependencyWithNewTempDir(dep_name):
temp_dir = tempfile.mkdtemp()
try:
logging.info('Using temp directory: %s', temp_dir)
config = DefaultConfig(temp_dir)
CheckinDependency(dep_name, config)
finally:
shutil.rmtree(temp_dir)
def CheckDependencyNotCheckedIn(dep_name):
config = Config(CHECKOUT_SRC_DIR, REMOTE_URL, '')
with open(config.dependencies_file, 'rb') as f:
data = json.load(f)
dep_list = data.get('dependencies', [])
if dep_name in dep_list:
raise DependencyAlreadyCheckedIn("Dependency %s has been already checked "
"into WebRTC repo" % dep_name)
if dep_name in os.listdir(config.third_party_dir):
raise DependencyAlreadyCheckedIn("Directory for dependency %s already "
"exists in third_party" % dep_name)
def main():
p = argparse.ArgumentParser()
p.add_argument('-d', '--dependency', required=True,
help='Name of chromium dependency to check in.')
p.add_argument('--temp-dir',
help='Temp working directory to use. By default the one '
'provided via tempfile will be used')
p.add_argument('-v', '--verbose', action='store_true', default=False,
help='Be extra verbose in printing of log messages.')
args = p.parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
CheckDependencyNotCheckedIn(args.dependency)
if args.temp_dir:
if not os.path.exists(args.temp_dir):
# Raise system error "No such file or directory"
raise OSError(
errno.ENOENT, os.strerror(errno.ENOENT), args.temp_dir)
config = DefaultConfig(args.temp_dir)
CheckinDependency(args.dependency, config)
else:
CheckinDependencyWithNewTempDir(args.dependency)
return 0
if __name__ == '__main__':
sys.exit(main())

View File

@ -12,6 +12,7 @@
import argparse
import base64
import collections
import json
import logging
import os
import re
@ -414,12 +415,9 @@ def UpdateDepsFile(deps_filename, rev_update, changed_deps):
def _LoadThirdPartyDepsAndFiles(filename):
third_party_deps = {}
with open(filename, 'rb') as f:
deps_content = f.read()
global_scope = {}
exec (deps_content, global_scope, third_party_deps)
return third_party_deps.get('DEPS', [])
data = json.load(f)
return data.get('dependencies', [])
def UpdateThirdPartyDeps(new_rev, dest_dir, source_dir,
@ -613,10 +611,12 @@ def main():
logging.debug('Commit message:\n%s', commit_msg)
_CreateRollBranch(opts.dry_run)
third_party_chromium_deps_list = os.path.join(
CHECKOUT_SRC_DIR, 'THIRD_PARTY_CHROMIUM_DEPS.json')
UpdateThirdPartyDeps(rev_update.new_third_party_rev,
os.path.join(CHECKOUT_SRC_DIR, 'third_party'),
cr_3p_repo,
os.path.join(CHECKOUT_SRC_DIR, 'THIRD_PARTY_DEPS'))
third_party_chromium_deps_list)
UpdateDepsFile(deps_filename, rev_update, changed_deps)
if _IsTreeClean():
logging.info("No DEPS changes detected, skipping CL creation.")

View File

@ -0,0 +1,116 @@
#!/usr/bin/env python
# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
import json
import os.path
import shutil
import sys
import tempfile
import unittest
import distutils.dir_util
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir)
sys.path.append(PARENT_DIR)
from checkin_chromium_dep import Config, CheckinDependency, RunCommand, \
DependencyNotFound
CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
os.pardir))
FAKE_REMOTE_TEMPLATE_ROOT = os.path.join(SCRIPT_DIR, 'testdata',
'checkin_chromium_dep', 'remote_root')
FAKE_SOURCE_TEMPLATE_ROOT = os.path.join(SCRIPT_DIR, 'testdata',
'checkin_chromium_dep', 'src_root')
class TestCheckInChromiumDep(unittest.TestCase):
def setUp(self):
self._temp_dir = tempfile.mkdtemp(prefix='webrtc_test_')
self._fake_chromium_repo = tempfile.mkdtemp(prefix='webrtc_test_')
self._fake_source_repo = tempfile.mkdtemp(prefix='webrtc_test_')
print("Temp dir: %s\n"
"Chromium third_party fake repo: %s\n"
"WebRTC source fake repo: %s" % (
self._temp_dir, self._fake_chromium_repo,
self._fake_source_repo))
self._fake_chromium_revision = TestCheckInChromiumDep._InitFakeChromiumRepo(
self._fake_chromium_repo)
TestCheckInChromiumDep._InitFakeSourceRepo(self._fake_source_repo,
self._fake_chromium_revision)
@staticmethod
def _InitFakeChromiumRepo(repo_dir):
RunCommand(['git', 'init'], working_dir=repo_dir)
distutils.dir_util.copy_tree(FAKE_REMOTE_TEMPLATE_ROOT, repo_dir)
RunCommand(['git', 'add', '-A', '.'], working_dir=repo_dir)
RunCommand(['git', 'commit', '-m', 'Init'],
working_dir=repo_dir)
stdout, _ = RunCommand(['git', 'rev-parse', 'HEAD'], working_dir=repo_dir)
return stdout.strip()
@staticmethod
def _InitFakeSourceRepo(repo_dir, chromium_third_party_revision):
RunCommand(['git', 'init'], working_dir=repo_dir)
# Copy repo template
distutils.dir_util.copy_tree(FAKE_SOURCE_TEMPLATE_ROOT, repo_dir)
# Set right chromium third_party revision in DEPS file
with open(os.path.join(repo_dir, 'DEPS'), 'rb') as f:
deps_content = f.read()
deps_content = deps_content % chromium_third_party_revision
with open(os.path.join(repo_dir, 'DEPS'), 'wb') as f:
f.write(deps_content)
# Commit all repo content
RunCommand(['git', 'add', '-A', '.'], working_dir=repo_dir)
RunCommand(['git', 'commit', '-m', 'Init'],
working_dir=repo_dir)
def tearDown(self):
shutil.rmtree(self._temp_dir)
shutil.rmtree(self._fake_chromium_repo)
shutil.rmtree(self._fake_source_repo)
def testCheckIn(self):
third_party_dir = os.path.join(self._fake_source_repo, 'third_party')
CheckinDependency('dep_bar',
Config(
self._fake_source_repo,
'file://%s' % self._fake_chromium_repo,
self._temp_dir))
third_party_deps_list_file = os.path.join(self._fake_source_repo,
'THIRD_PARTY_CHROMIUM_DEPS.json')
with open(third_party_deps_list_file, 'rb') as f:
deps_list = json.load(f).get('dependencies', [])
# New dependency appended to deps list file
self.assertIn('dep_foo', deps_list)
self.assertIn('dep_bar', deps_list)
# Only new dependency was appended
self.assertNotIn('dep_buzz', deps_list)
# New dependency was copied into source tree
self.assertIn('dep_bar', os.listdir(third_party_dir))
self.assertIn(
'source_file.js', os.listdir(os.path.join(third_party_dir, 'dep_bar')))
# Only new dependency was copied into source tree
self.assertNotIn('dep_buzz', os.listdir(third_party_dir))
def testCheckInNotExistingDep(self):
self.assertRaises(DependencyNotFound,
CheckinDependency,
'dep_missing',
Config(self._fake_source_repo,
'file://%s' % self._fake_chromium_repo,
self._temp_dir))
if __name__ == '__main__':
unittest.main()

View File

@ -68,7 +68,8 @@ class FakeCmd(object):
class TestRollChromiumRevision(unittest.TestCase):
def setUp(self):
self._output_dir = tempfile.mkdtemp()
for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')):
test_data_dir = os.path.join(SCRIPT_DIR, 'testdata', 'roll_deps')
for test_file in glob.glob(os.path.join(test_data_dir, '*')):
shutil.copy(test_file, self._output_dir)
self._webrtc_depsfile = os.path.join(self._output_dir, 'DEPS')
self._new_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.new')

View File

@ -0,0 +1,6 @@
# DEPS file for unit tests.
vars = {
'chromium_third_party_revision': '%s',
}

View File

@ -0,0 +1,14 @@
{
"comment": [
"This file determines which parts of Chromium's third_party/ we copy.",
"",
"If you want to add new chromium owned dependency, run",
"./tools_webrtc/autoroller/checkin_chromium_dep.py -d <dep name>",
"",
"It will add specified dependency to third_party directory and will add it to this list",
"properly."
],
"dependencies": [
"dep_foo"
]
}

View File

@ -0,0 +1,9 @@
{
"comment": [
"This file contains list of third party dependencies owned by WebRTC directly,",
"e.g. they're not from Chromium's third party (those are tracked in ",
"THIRD_PARTY_CHROMIUM_DEPS.json)."
],
"dependencies": [
]
}