Roll back checking in the third_party directory
This goes back to using a subtree mirror of Chromium's third_party directory (managed by gclient). The related scripts for syncing the files are also deleted. The plan is to solve the conflict by creating third_party directories in subdirectories of WebRTC rather than the repo root. Bug: webrtc:8366 Change-Id: I0b9f6a86c6d4075e2fa12c2db19aa54682ddb11f Reviewed-on: https://webrtc-review.googlesource.com/85300 Reviewed-by: Oleh Prypin <oprypin@webrtc.org> Reviewed-by: Karl Wiberg <kwiberg@webrtc.org> Cr-Commit-Position: refs/heads/master@{#23757}
This commit is contained in:
Artem Titov
@ -1,252 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
#
|
||||
# Use of this source code is governed by a BSD-style license
|
||||
# that can be found in the LICENSE file in the root of the source
|
||||
# tree. An additional intellectual property rights grant can be found
|
||||
# in the file PATENTS. All contributing project authors may
|
||||
# be found in the AUTHORS file in the root of the source tree.
|
||||
|
||||
|
||||
"""
|
||||
Script to automatically add specified chromium dependency to WebRTC repo.
|
||||
|
||||
If you want to add new chromium owned dependency foo to the WebRTC repo
|
||||
you have to run this tool like this:
|
||||
./checkin_chromium_deps.py -d foo
|
||||
|
||||
It will check in dependency into third_party directory and will add it into
|
||||
git index. Also it will update chromium dependencies list with new dependency
|
||||
to insure that it will be correctly auto updated in future.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import errno
|
||||
import json
|
||||
import logging
|
||||
import os.path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
REMOTE_URL = 'https://chromium.googlesource.com/chromium/src/third_party'
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
|
||||
os.pardir))
|
||||
|
||||
|
||||
class DependencyAlreadyCheckedIn(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DependencyNotFound(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Config(object):
|
||||
|
||||
def __init__(self, src_root, remote_url, temp_dir):
|
||||
self.src_root = src_root
|
||||
self.dependencies_file = os.path.join(self.src_root,
|
||||
'THIRD_PARTY_CHROMIUM_DEPS.json')
|
||||
self.deps_file = os.path.join(self.src_root, 'DEPS')
|
||||
self.third_party_dir = os.path.join(self.src_root, 'third_party')
|
||||
self.remote_url = remote_url
|
||||
self.temp_dir = temp_dir
|
||||
|
||||
|
||||
def VarLookup(local_scope):
|
||||
return lambda var_name: local_scope['vars'][var_name]
|
||||
|
||||
|
||||
def ParseDepsDict(deps_content):
|
||||
local_scope = {}
|
||||
global_scope = {
|
||||
'Var': VarLookup(local_scope),
|
||||
'deps_os': {},
|
||||
}
|
||||
exec (deps_content, global_scope, local_scope)
|
||||
return local_scope
|
||||
|
||||
|
||||
def ParseLocalDepsFile(filename):
|
||||
with open(filename, 'rb') as f:
|
||||
deps_content = f.read()
|
||||
return ParseDepsDict(deps_content)
|
||||
|
||||
|
||||
def RunCommand(command, working_dir=None, ignore_exit_code=False,
|
||||
extra_env=None, input_data=None):
|
||||
"""Runs a command and returns the output from that command.
|
||||
|
||||
If the command fails (exit code != 0), the function will exit the process.
|
||||
|
||||
Returns:
|
||||
A tuple containing the stdout and stderr outputs as strings.
|
||||
"""
|
||||
working_dir = working_dir or CHECKOUT_SRC_DIR
|
||||
logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
|
||||
env = os.environ.copy()
|
||||
if extra_env:
|
||||
assert all(isinstance(value, str) for value in extra_env.values())
|
||||
logging.debug('extra env: %s', extra_env)
|
||||
env.update(extra_env)
|
||||
p = subprocess.Popen(command,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, env=env,
|
||||
cwd=working_dir, universal_newlines=True)
|
||||
std_output, err_output = p.communicate(input_data)
|
||||
p.stdout.close()
|
||||
p.stderr.close()
|
||||
if not ignore_exit_code and p.returncode != 0:
|
||||
logging.error('Command failed: %s\n'
|
||||
'stdout:\n%s\n'
|
||||
'stderr:\n%s\n', ' '.join(command), std_output, err_output)
|
||||
sys.exit(p.returncode)
|
||||
return std_output, err_output
|
||||
|
||||
|
||||
def LoadThirdPartyRevision(deps_file):
|
||||
logging.debug('Loading chromium third_party revision from %s', deps_file)
|
||||
webrtc_deps = ParseLocalDepsFile(deps_file)
|
||||
return webrtc_deps['vars']['chromium_third_party_revision']
|
||||
|
||||
|
||||
def CheckoutRequiredDependency(dep_name, config):
|
||||
third_party_revision = LoadThirdPartyRevision(config.deps_file)
|
||||
|
||||
logging.debug('Initializing git repo in %s...', config.temp_dir)
|
||||
RunCommand(['git', 'init'], working_dir=config.temp_dir)
|
||||
|
||||
logging.debug('Adding remote to %s. It may take some time...',
|
||||
config.remote_url)
|
||||
RunCommand(['git', 'remote', 'add', '-f', 'origin', config.remote_url],
|
||||
working_dir=config.temp_dir)
|
||||
|
||||
logging.debug('Configuring sparse checkout...')
|
||||
RunCommand(['git', 'config', 'core.sparseCheckout', 'true'],
|
||||
working_dir=config.temp_dir)
|
||||
sparse_checkout_config_path = os.path.join(config.temp_dir, '.git', 'info',
|
||||
'sparse-checkout')
|
||||
with open(sparse_checkout_config_path, 'wb') as f:
|
||||
f.write(dep_name)
|
||||
|
||||
logging.debug('Pulling changes...')
|
||||
_, stderr = RunCommand(['git', 'pull', 'origin', 'master'],
|
||||
working_dir=config.temp_dir,
|
||||
ignore_exit_code=True)
|
||||
if "Sparse checkout leaves no entry on working directory" in stderr:
|
||||
# There are no such dependency in chromium third_party
|
||||
raise DependencyNotFound(
|
||||
"Dependency %s not found in chromium repo" % dep_name)
|
||||
|
||||
logging.debug('Switching to revision %s...', third_party_revision)
|
||||
RunCommand(['git', 'checkout', third_party_revision],
|
||||
working_dir=config.temp_dir)
|
||||
return os.path.join(config.temp_dir, dep_name)
|
||||
|
||||
|
||||
def CopyDependency(dep_name, source_path, third_party_dir):
|
||||
dest_path = os.path.join(third_party_dir, dep_name)
|
||||
logging.debug('Copying dependency from %s to %s...', source_path, dest_path)
|
||||
shutil.copytree(source_path, dest_path)
|
||||
|
||||
|
||||
def AppendToChromiumOwnedDependenciesList(dep_name, dep_file):
|
||||
with open(dep_file, 'rb') as f:
|
||||
data = json.load(f)
|
||||
dep_list = data.get('dependencies', [])
|
||||
dep_list.append(dep_name)
|
||||
data['dependencies'] = dep_list
|
||||
|
||||
with open(dep_file, 'wb') as f:
|
||||
json.dump(data, f, indent=2, sort_keys=True, separators=(',', ': '))
|
||||
|
||||
|
||||
def AddToGitIndex(dep_name, config):
|
||||
logging.debug('Adding required changes to git index and commit set...')
|
||||
dest_path = os.path.join(config.third_party_dir, dep_name)
|
||||
RunCommand(['git', 'add', dest_path], working_dir=config.src_root)
|
||||
RunCommand(['git', 'add', config.dependencies_file],
|
||||
working_dir=config.src_root)
|
||||
|
||||
|
||||
def CheckinDependency(dep_name, config):
|
||||
dep_path = CheckoutRequiredDependency(dep_name, config)
|
||||
CopyDependency(dep_name, dep_path, config.third_party_dir)
|
||||
AppendToChromiumOwnedDependenciesList(dep_name, config.dependencies_file)
|
||||
AddToGitIndex(dep_name, config)
|
||||
logging.info('Successfully added %s.', dep_path)
|
||||
logging.info('You now have to:')
|
||||
logging.info('1. Check if you need to add an entry to DEPS (check if ')
|
||||
logging.info(' Chromium has %s in its DEPS file and copy that)', dep_name)
|
||||
logging.info('2. Copy third_party/.gitignore for %s from Chromium to\n'
|
||||
' third_party/.gitignore', dep_name)
|
||||
logging.info('3. Commit locally and upload.')
|
||||
|
||||
|
||||
def DefaultConfig(temp_dir):
|
||||
return Config(CHECKOUT_SRC_DIR, REMOTE_URL, temp_dir)
|
||||
|
||||
|
||||
def CheckinDependencyWithNewTempDir(dep_name):
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
try:
|
||||
logging.info('Using temp directory: %s', temp_dir)
|
||||
config = DefaultConfig(temp_dir)
|
||||
CheckinDependency(dep_name, config)
|
||||
finally:
|
||||
shutil.rmtree(temp_dir)
|
||||
|
||||
|
||||
def CheckDependencyNotCheckedIn(dep_name):
|
||||
config = Config(CHECKOUT_SRC_DIR, REMOTE_URL, '')
|
||||
with open(config.dependencies_file, 'rb') as f:
|
||||
data = json.load(f)
|
||||
dep_list = data.get('dependencies', [])
|
||||
if dep_name in dep_list:
|
||||
raise DependencyAlreadyCheckedIn("Dependency %s has been already checked "
|
||||
"into WebRTC repo" % dep_name)
|
||||
if dep_name in os.listdir(config.third_party_dir):
|
||||
raise DependencyAlreadyCheckedIn("Directory for dependency %s already "
|
||||
"exists in third_party" % dep_name)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('-d', '--dependency', required=True,
|
||||
help=('Name of chromium dependency to check in. For instance, '
|
||||
'if you want to check in third_party/xyz, then pass '
|
||||
'-d xyz.'))
|
||||
p.add_argument('--temp-dir',
|
||||
help='Temp working directory to use. By default the one '
|
||||
'provided via tempfile will be used')
|
||||
p.add_argument('-v', '--verbose', action='store_true', default=False,
|
||||
help='Be extra verbose in printing of log messages.')
|
||||
args = p.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
else:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
CheckDependencyNotCheckedIn(args.dependency)
|
||||
|
||||
if args.temp_dir:
|
||||
if not os.path.exists(args.temp_dir):
|
||||
# Raise system error "No such file or directory"
|
||||
raise OSError(
|
||||
errno.ENOENT, os.strerror(errno.ENOENT), args.temp_dir)
|
||||
config = DefaultConfig(args.temp_dir)
|
||||
CheckinDependency(args.dependency, config)
|
||||
else:
|
||||
CheckinDependencyWithNewTempDir(args.dependency)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@ -12,7 +12,6 @@
|
||||
import argparse
|
||||
import base64
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
@ -23,7 +22,6 @@ import urllib2
|
||||
# Skip these dependencies (list without solution name prefix).
|
||||
DONT_AUTOROLL_THESE = [
|
||||
'src/examples/androidtests/third_party/gradle',
|
||||
'src/third_party_chromium',
|
||||
'src/third_party/ffmpeg',
|
||||
]
|
||||
|
||||
@ -34,11 +32,9 @@ EXTRA_TRYBOTS = (
|
||||
|
||||
WEBRTC_URL = 'https://webrtc.googlesource.com/src'
|
||||
CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src'
|
||||
CHROMIUM_THIRD_PARTY_URL = '%s/third_party' % CHROMIUM_SRC_URL
|
||||
CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s'
|
||||
CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s'
|
||||
CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s'
|
||||
CHROMIUM_3P_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/third_party/+log/%s'
|
||||
|
||||
COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$')
|
||||
CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$')
|
||||
@ -67,9 +63,7 @@ ChangedCipdPackage = collections.namedtuple(
|
||||
|
||||
ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate',
|
||||
('current_chromium_rev '
|
||||
'new_chromium_rev '
|
||||
'current_third_party_rev '
|
||||
'new_third_party_rev'))
|
||||
'new_chromium_rev '))
|
||||
|
||||
|
||||
class RollError(Exception):
|
||||
@ -331,18 +325,13 @@ def GenerateCommitMessage(rev_update, current_commit_pos,
|
||||
current_cr_rev = rev_update.current_chromium_rev[0:10]
|
||||
new_cr_rev = rev_update.new_chromium_rev[0:10]
|
||||
rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev)
|
||||
rev_3p_interval = '%s..%s' % (rev_update.current_third_party_rev[0:10],
|
||||
rev_update.new_third_party_rev[0:10])
|
||||
git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos)
|
||||
|
||||
commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval,
|
||||
git_number_interval),
|
||||
'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval),
|
||||
'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE %
|
||||
rev_interval),
|
||||
'Roll chromium third_party %s' % rev_3p_interval,
|
||||
'Change log: %s\n' % (
|
||||
CHROMIUM_3P_LOG_TEMPLATE % rev_3p_interval)]
|
||||
rev_interval)]
|
||||
tbr_authors = ''
|
||||
if changed_deps_list:
|
||||
commit_msg.append('Changed dependencies:')
|
||||
@ -380,7 +369,6 @@ def GenerateCommitMessage(rev_update, current_commit_pos,
|
||||
commit_msg.append('TBR=%s' % tbr_authors)
|
||||
commit_msg.append('BUG=None')
|
||||
commit_msg.append('CQ_INCLUDE_TRYBOTS=%s' % EXTRA_TRYBOTS)
|
||||
commit_msg.append('NO_AUTOIMPORT_DEPS_CHECK=true')
|
||||
return '\n'.join(commit_msg)
|
||||
|
||||
|
||||
@ -392,8 +380,6 @@ def UpdateDepsFile(deps_filename, rev_update, changed_deps):
|
||||
deps_content = deps_file.read()
|
||||
deps_content = deps_content.replace(rev_update.current_chromium_rev,
|
||||
rev_update.new_chromium_rev)
|
||||
deps_content = deps_content.replace(rev_update.current_third_party_rev,
|
||||
rev_update.new_third_party_rev)
|
||||
with open(deps_filename, 'wb') as deps_file:
|
||||
deps_file.write(deps_content)
|
||||
|
||||
@ -416,48 +402,6 @@ def UpdateDepsFile(deps_filename, rev_update, changed_deps):
|
||||
working_dir=CHECKOUT_SRC_DIR)
|
||||
|
||||
|
||||
def _LoadThirdPartyDepsAndFiles(filename):
|
||||
with open(filename, 'rb') as f:
|
||||
data = json.load(f)
|
||||
return data.get('dependencies', [])
|
||||
|
||||
|
||||
def UpdateThirdPartyDeps(new_rev, dest_dir, source_dir,
|
||||
third_party_deps_file):
|
||||
"""Syncing deps, specified in third_party_deps_file with repo in source_dir.
|
||||
|
||||
Will exit if sync failed for some reasons.
|
||||
Params:
|
||||
new_rev - revision of third_party to update to
|
||||
dest_dir - webrtc directory, that will be used as root for third_party deps
|
||||
source_dir - checked out chromium third_party repo
|
||||
third_party_deps_file - file with list of third_party deps to copy
|
||||
"""
|
||||
|
||||
deps_to_checkout = _LoadThirdPartyDepsAndFiles(third_party_deps_file)
|
||||
# Update existing chromium third_party checkout to new rev.
|
||||
_RunCommand(['git', 'fetch', 'origin'], working_dir=source_dir)
|
||||
_RunCommand(['git', 'checkout', new_rev], working_dir=source_dir)
|
||||
# Checkout chromium repo into dest dir basing on source checkout.
|
||||
_RunCommand(
|
||||
['git', '--git-dir', '%s/.git' % source_dir, 'checkout',
|
||||
new_rev] + deps_to_checkout, working_dir=dest_dir)
|
||||
# Ensure all chromium dependencies are presented
|
||||
deps_set = set(deps_to_checkout)
|
||||
stdout, _ = _RunCommand(['git', 'ls-tree', '--name-only', 'HEAD'],
|
||||
working_dir=source_dir)
|
||||
if not deps_set.issubset(set(stdout.split('\n'))):
|
||||
raise RollError('Missed required chromium dependencies in chromium '
|
||||
'third_party repo: %s' % json.dumps(
|
||||
list(deps_set.difference(set(stdout.split('\n'))))))
|
||||
stdout, _ = _RunCommand(['git', 'ls-tree', '--name-only', 'HEAD'],
|
||||
working_dir=dest_dir)
|
||||
if not deps_set.issubset(set(stdout.split('\n'))):
|
||||
raise RollError('Missed required chromium dependencies after checking in '
|
||||
'chromium third_party repo: %s' % json.dumps(
|
||||
list(deps_set.difference(set(stdout.split('\n'))))))
|
||||
|
||||
|
||||
def _IsTreeClean():
|
||||
stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
|
||||
if len(stdout) == 0:
|
||||
@ -500,7 +444,6 @@ def _LocalCommit(commit_msg, dry_run):
|
||||
logging.info('Committing changes locally.')
|
||||
if not dry_run:
|
||||
_RunCommand(['git', 'add', '--update', '.'])
|
||||
_RunCommand(['git', 'add', '-A', 'third_party'])
|
||||
_RunCommand(['git', 'commit', '-m', commit_msg])
|
||||
|
||||
|
||||
@ -532,7 +475,6 @@ def _UploadCL(commit_queue_mode):
|
||||
|
||||
def GetRollRevisionRanges(opts, webrtc_deps):
|
||||
current_cr_rev = webrtc_deps['vars']['chromium_revision']
|
||||
current_third_party_rev = webrtc_deps['vars']['chromium_third_party_revision']
|
||||
new_cr_rev = opts.revision
|
||||
if not new_cr_rev:
|
||||
stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
|
||||
@ -540,18 +482,7 @@ def GetRollRevisionRanges(opts, webrtc_deps):
|
||||
logging.info('No revision specified. Using HEAD: %s', head_rev)
|
||||
new_cr_rev = head_rev
|
||||
|
||||
new_third_party_rev = opts.third_party_revision
|
||||
if not new_third_party_rev:
|
||||
stdout, _ = _RunCommand(
|
||||
['git', 'ls-remote', CHROMIUM_THIRD_PARTY_URL, 'HEAD'])
|
||||
new_third_party_rev = stdout.strip().split('\t')[0]
|
||||
logging.info(
|
||||
'No third_party revision specified. Using HEAD: %s',
|
||||
new_third_party_rev)
|
||||
|
||||
return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev,
|
||||
current_third_party_rev,
|
||||
new_third_party_rev)
|
||||
return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev)
|
||||
|
||||
|
||||
def main():
|
||||
@ -561,9 +492,6 @@ def main():
|
||||
p.add_argument('-r', '--revision',
|
||||
help=('Chromium Git revision to roll to. Defaults to the '
|
||||
'Chromium HEAD revision if omitted.'))
|
||||
p.add_argument('--third-party-revision',
|
||||
help=('Chromium third_party Git revision to roll to. Default '
|
||||
'to the Chromium third_party HEAD revision if omitted.'))
|
||||
p.add_argument('-u', '--rietveld-email',
|
||||
help=('E-mail address to use for creating the CL at Rietveld'
|
||||
'If omitted a previously cached one will be used or an '
|
||||
@ -603,15 +531,6 @@ def main():
|
||||
|
||||
deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS')
|
||||
webrtc_deps = ParseLocalDepsFile(deps_filename)
|
||||
cr_3p_repo = os.path.join(CHECKOUT_SRC_DIR, 'third_party_chromium')
|
||||
if not os.path.exists(cr_3p_repo):
|
||||
raise RollError('missing third_party_chromium/. '
|
||||
'Please add this to your gclient: \n'
|
||||
'"custom_vars": {\n'
|
||||
' "roll_chromium_into_webrtc": True,\n'
|
||||
'},\n'
|
||||
'Then run "gclient sync" again.')
|
||||
|
||||
rev_update = GetRollRevisionRanges(opts, webrtc_deps)
|
||||
|
||||
current_commit_pos = ParseCommitPosition(
|
||||
@ -628,12 +547,6 @@ def main():
|
||||
logging.debug('Commit message:\n%s', commit_msg)
|
||||
|
||||
_CreateRollBranch(opts.dry_run)
|
||||
third_party_chromium_deps_list = os.path.join(
|
||||
CHECKOUT_SRC_DIR, 'THIRD_PARTY_CHROMIUM_DEPS.json')
|
||||
UpdateThirdPartyDeps(rev_update.new_third_party_rev,
|
||||
os.path.join(CHECKOUT_SRC_DIR, 'third_party'),
|
||||
cr_3p_repo,
|
||||
third_party_chromium_deps_list)
|
||||
UpdateDepsFile(deps_filename, rev_update, changed_deps)
|
||||
if _IsTreeClean():
|
||||
logging.info("No DEPS changes detected, skipping CL creation.")
|
||||
|
||||
@ -1,130 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
#
|
||||
# Use of this source code is governed by a BSD-style license
|
||||
# that can be found in the LICENSE file in the root of the source
|
||||
# tree. An additional intellectual property rights grant can be found
|
||||
# in the file PATENTS. All contributing project authors may
|
||||
# be found in the AUTHORS file in the root of the source tree.
|
||||
|
||||
|
||||
import errno
|
||||
import json
|
||||
import os.path
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import distutils.dir_util
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir)
|
||||
sys.path.append(PARENT_DIR)
|
||||
from checkin_chromium_dep import Config, CheckinDependency, RunCommand, \
|
||||
DependencyNotFound
|
||||
|
||||
CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
|
||||
os.pardir))
|
||||
FAKE_REMOTE_TEMPLATE_ROOT = os.path.join(SCRIPT_DIR, 'testdata',
|
||||
'checkin_chromium_dep', 'remote_root')
|
||||
FAKE_SOURCE_TEMPLATE_ROOT = os.path.join(SCRIPT_DIR, 'testdata',
|
||||
'checkin_chromium_dep', 'src_root')
|
||||
|
||||
|
||||
def _HandleRemoveReadonly(func, path, exc):
|
||||
excvalue = exc[1]
|
||||
if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
|
||||
os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777
|
||||
func(path)
|
||||
else:
|
||||
raise excvalue
|
||||
|
||||
|
||||
class TestCheckInChromiumDep(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._temp_dir = tempfile.mkdtemp(prefix='webrtc_test_')
|
||||
self._fake_chromium_repo = tempfile.mkdtemp(prefix='webrtc_test_')
|
||||
self._fake_source_repo = tempfile.mkdtemp(prefix='webrtc_test_')
|
||||
|
||||
print("Temp dir: %s\n"
|
||||
"Chromium third_party fake repo: %s\n"
|
||||
"WebRTC source fake repo: %s" % (
|
||||
self._temp_dir, self._fake_chromium_repo,
|
||||
self._fake_source_repo))
|
||||
|
||||
self._fake_chromium_revision = TestCheckInChromiumDep._InitFakeChromiumRepo(
|
||||
self._fake_chromium_repo)
|
||||
TestCheckInChromiumDep._InitFakeSourceRepo(self._fake_source_repo,
|
||||
self._fake_chromium_revision)
|
||||
|
||||
@staticmethod
|
||||
def _InitFakeChromiumRepo(repo_dir):
|
||||
RunCommand(['git', 'init'], working_dir=repo_dir)
|
||||
distutils.dir_util.copy_tree(FAKE_REMOTE_TEMPLATE_ROOT, repo_dir)
|
||||
RunCommand(['git', 'add', '-A', '.'], working_dir=repo_dir)
|
||||
RunCommand(['git', 'commit', '-m', 'Init'],
|
||||
working_dir=repo_dir)
|
||||
stdout, _ = RunCommand(['git', 'rev-parse', 'HEAD'], working_dir=repo_dir)
|
||||
return stdout.strip()
|
||||
|
||||
@staticmethod
|
||||
def _InitFakeSourceRepo(repo_dir, chromium_third_party_revision):
|
||||
RunCommand(['git', 'init'], working_dir=repo_dir)
|
||||
# Copy repo template
|
||||
distutils.dir_util.copy_tree(FAKE_SOURCE_TEMPLATE_ROOT, repo_dir)
|
||||
# Set right chromium third_party revision in DEPS file
|
||||
with open(os.path.join(repo_dir, 'DEPS'), 'rb') as f:
|
||||
deps_content = f.read()
|
||||
deps_content = deps_content % chromium_third_party_revision
|
||||
with open(os.path.join(repo_dir, 'DEPS'), 'wb') as f:
|
||||
f.write(deps_content)
|
||||
# Commit all repo content
|
||||
RunCommand(['git', 'add', '-A', '.'], working_dir=repo_dir)
|
||||
RunCommand(['git', 'commit', '-m', 'Init'],
|
||||
working_dir=repo_dir)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self._temp_dir, ignore_errors=False,
|
||||
onerror=_HandleRemoveReadonly)
|
||||
shutil.rmtree(self._fake_chromium_repo, ignore_errors=False,
|
||||
onerror=_HandleRemoveReadonly)
|
||||
shutil.rmtree(self._fake_source_repo, ignore_errors=False,
|
||||
onerror=_HandleRemoveReadonly)
|
||||
|
||||
def testCheckIn(self):
|
||||
third_party_dir = os.path.join(self._fake_source_repo, 'third_party')
|
||||
|
||||
CheckinDependency('dep_bar',
|
||||
Config(
|
||||
self._fake_source_repo,
|
||||
'file://%s' % self._fake_chromium_repo,
|
||||
self._temp_dir))
|
||||
third_party_deps_list_file = os.path.join(self._fake_source_repo,
|
||||
'THIRD_PARTY_CHROMIUM_DEPS.json')
|
||||
with open(third_party_deps_list_file, 'rb') as f:
|
||||
deps_list = json.load(f).get('dependencies', [])
|
||||
|
||||
# New dependency appended to deps list file
|
||||
self.assertIn('dep_foo', deps_list)
|
||||
self.assertIn('dep_bar', deps_list)
|
||||
# Only new dependency was appended
|
||||
self.assertNotIn('dep_buzz', deps_list)
|
||||
# New dependency was copied into source tree
|
||||
self.assertIn('dep_bar', os.listdir(third_party_dir))
|
||||
self.assertIn(
|
||||
'source_file.js', os.listdir(os.path.join(third_party_dir, 'dep_bar')))
|
||||
# Only new dependency was copied into source tree
|
||||
self.assertNotIn('dep_buzz', os.listdir(third_party_dir))
|
||||
|
||||
def testCheckInNotExistingDep(self):
|
||||
self.assertRaises(DependencyNotFound,
|
||||
CheckinDependency,
|
||||
'dep_missing',
|
||||
Config(self._fake_source_repo,
|
||||
'file://%s' % self._fake_chromium_repo,
|
||||
self._temp_dir))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -93,8 +93,7 @@ class TestRollChromiumRevision(unittest.TestCase):
|
||||
|
||||
current_rev = TEST_DATA_VARS['chromium_revision']
|
||||
UpdateDepsFile(self._webrtc_depsfile,
|
||||
ChromiumRevisionUpdate(
|
||||
current_rev, new_rev, current_rev, new_rev),
|
||||
ChromiumRevisionUpdate(current_rev, new_rev),
|
||||
[])
|
||||
with open(self._webrtc_depsfile) as deps_file:
|
||||
deps_contents = deps_file.read()
|
||||
|
||||
@ -1,6 +0,0 @@
|
||||
# DEPS file for unit tests.
|
||||
|
||||
vars = {
|
||||
'chromium_third_party_revision': '%s',
|
||||
}
|
||||
|
||||
@ -1,14 +0,0 @@
|
||||
{
|
||||
"comment": [
|
||||
"This file determines which parts of Chromium's third_party/ we copy.",
|
||||
"",
|
||||
"If you want to add new chromium owned dependency, run",
|
||||
"./tools_webrtc/autoroller/checkin_chromium_dep.py -d <dep name>",
|
||||
"",
|
||||
"It will add specified dependency to third_party directory and will add it to this list",
|
||||
"properly."
|
||||
],
|
||||
"dependencies": [
|
||||
"dep_foo"
|
||||
]
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
{
|
||||
"comment": [
|
||||
"This file contains list of third party dependencies owned by WebRTC directly,",
|
||||
"e.g. they're not from Chromium's third party (those are tracked in ",
|
||||
"THIRD_PARTY_CHROMIUM_DEPS.json)."
|
||||
],
|
||||
"dependencies": [
|
||||
]
|
||||
}
|
||||
@ -1,175 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
#
|
||||
# Use of this source code is governed by a BSD-style license
|
||||
# that can be found in the LICENSE file in the root of the source
|
||||
# tree. An additional intellectual property rights grant can be found
|
||||
# in the file PATENTS. All contributing project authors may
|
||||
# be found in the AUTHORS file in the root of the source tree.
|
||||
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os.path
|
||||
import subprocess
|
||||
import sys
|
||||
import re
|
||||
|
||||
|
||||
def CheckThirdPartyDirectory(input_api, output_api):
|
||||
# We have to put something in black_list here that won't blacklist
|
||||
# third_party/* because otherwise default black list will be used. Default
|
||||
# list contains third_party, so source set will become empty.
|
||||
third_party_sources = lambda x: (
|
||||
input_api.FilterSourceFile(x, white_list=(r'^third_party[\\\/].+',),
|
||||
black_list=(r'^_',)))
|
||||
|
||||
webrtc_owned_deps_list_path = input_api.os_path.join(
|
||||
input_api.PresubmitLocalPath(),
|
||||
'THIRD_PARTY_WEBRTC_DEPS.json')
|
||||
chromium_owned_deps_list_path = input_api.os_path.join(
|
||||
input_api.PresubmitLocalPath(),
|
||||
'THIRD_PARTY_CHROMIUM_DEPS.json')
|
||||
webrtc_owned_deps = _LoadDepsList(webrtc_owned_deps_list_path)
|
||||
chromium_owned_deps = _LoadDepsList(chromium_owned_deps_list_path)
|
||||
chromium_added_deps = GetChromiumOwnedAddedDeps(input_api)
|
||||
|
||||
results = []
|
||||
results.extend(CheckNoNotOwned3ppDeps(input_api, output_api,
|
||||
webrtc_owned_deps, chromium_owned_deps))
|
||||
results.extend(CheckNoBothOwned3ppDeps(output_api, webrtc_owned_deps,
|
||||
chromium_owned_deps))
|
||||
results.extend(CheckNoChangesInAutoImportedDeps(input_api, output_api,
|
||||
webrtc_owned_deps,
|
||||
chromium_owned_deps,
|
||||
chromium_added_deps,
|
||||
third_party_sources))
|
||||
return results
|
||||
|
||||
|
||||
def GetChromiumOwnedAddedDeps(input_api):
|
||||
"""Return list of deps that were added into chromium owned deps list."""
|
||||
|
||||
chromium_owned_deps_list_source = lambda x: (
|
||||
input_api.FilterSourceFile(x,
|
||||
white_list=('THIRD_PARTY_CHROMIUM_DEPS.json',),
|
||||
black_list=(r'^_',)))
|
||||
|
||||
chromium_owned_deps_list = input_api.AffectedFiles(
|
||||
file_filter=chromium_owned_deps_list_source)
|
||||
modified_deps_file = next(iter(chromium_owned_deps_list), None)
|
||||
if not modified_deps_file:
|
||||
return []
|
||||
if modified_deps_file.Action() != 'M':
|
||||
return []
|
||||
prev_json = json.loads('\n'.join(modified_deps_file.OldContents()))
|
||||
new_json = json.loads('\n'.join(modified_deps_file.NewContents()))
|
||||
prev_deps_set = set(prev_json.get('dependencies', []))
|
||||
new_deps_set = set(new_json.get('dependencies', []))
|
||||
return list(new_deps_set.difference(prev_deps_set))
|
||||
|
||||
|
||||
def CheckNoNotOwned3ppDeps(input_api, output_api,
|
||||
webrtc_owned_deps, chromium_owned_deps):
|
||||
"""Checks that there are no any not owned third_party deps."""
|
||||
error_msg = ('Third party dependency [{}] have to be specified either in '
|
||||
'THIRD_PARTY_WEBRTC_DEPS.json or in '
|
||||
'THIRD_PARTY_CHROMIUM_DEPS.json.\n'
|
||||
'If you want to add chromium-specific'
|
||||
'dependency you can run this command (better in separate CL): \n'
|
||||
'./tools_webrtc/autoroller/checkin_chromium_dep.py -d {}\n'
|
||||
'If you want to add WebRTC-specific dependency just add it into '
|
||||
'THIRD_PARTY_WEBRTC_DEPS.json manually')
|
||||
|
||||
third_party_dir = os.path.join(input_api.PresubmitLocalPath(), 'third_party')
|
||||
os.listdir(third_party_dir)
|
||||
stdout, _ = _RunCommand(['git', 'ls-tree', '--name-only', 'HEAD'],
|
||||
working_dir=third_party_dir)
|
||||
not_owned_deps = set()
|
||||
results = []
|
||||
for dep_name in stdout.split('\n'):
|
||||
dep_name = dep_name.strip()
|
||||
if len(dep_name) == 0:
|
||||
continue
|
||||
if dep_name == '.gitignore':
|
||||
continue
|
||||
if (dep_name not in webrtc_owned_deps
|
||||
and dep_name not in chromium_owned_deps):
|
||||
results.append(
|
||||
output_api.PresubmitError(error_msg.format(dep_name, dep_name)))
|
||||
not_owned_deps.add(dep_name)
|
||||
return results
|
||||
|
||||
|
||||
def CheckNoBothOwned3ppDeps(output_api, webrtc_owned_deps, chromium_owned_deps):
|
||||
"""Checks that there are no any not owned third_party deps."""
|
||||
error_msg = ('Third party dependencies {} can\'t be a WebRTC- and '
|
||||
'Chromium-specific dependency at the same time. '
|
||||
'Remove them from one of these files: '
|
||||
'THIRD_PARTY_WEBRTC_DEPS.json or THIRD_PARTY_CHROMIUM_DEPS.json')
|
||||
|
||||
both_owned_deps = set(chromium_owned_deps).intersection(
|
||||
set(webrtc_owned_deps))
|
||||
results = []
|
||||
if both_owned_deps:
|
||||
results.append(output_api.PresubmitError(error_msg.format(
|
||||
json.dumps(list(both_owned_deps)))))
|
||||
return results
|
||||
|
||||
|
||||
def CheckNoChangesInAutoImportedDeps(input_api, output_api,
|
||||
webrtc_owned_deps, chromium_owned_deps, chromium_added_deps,
|
||||
third_party_sources):
|
||||
"""Checks that there are no changes in deps imported by autoroller."""
|
||||
|
||||
tag = input_api.change.NO_AUTOIMPORT_DEPS_CHECK
|
||||
if tag is not None and tag.lower() == 'true':
|
||||
# If there is a tag NO_AUTOIMPORT_DEPS_CHECK in the commit message, then
|
||||
# permit any changes in chromium's specific deps.
|
||||
return []
|
||||
|
||||
error_msg = ('Changes in [{}] will be overridden during chromium third_party '
|
||||
'autoroll. If you really want to change this code you have to '
|
||||
'do it upstream in Chromium\'s third_party.')
|
||||
results = []
|
||||
for f in input_api.AffectedFiles(file_filter=third_party_sources):
|
||||
file_path = f.LocalPath()
|
||||
split = re.split(r'[\\\/]', file_path)
|
||||
dep_name = split[1]
|
||||
if (dep_name not in webrtc_owned_deps
|
||||
and dep_name in chromium_owned_deps
|
||||
and dep_name not in chromium_added_deps):
|
||||
results.append(output_api.PresubmitError(error_msg.format(file_path)))
|
||||
return results
|
||||
|
||||
|
||||
def _LoadDepsList(file_name):
|
||||
with open(file_name, 'rb') as f:
|
||||
content = json.load(f)
|
||||
return content.get('dependencies', [])
|
||||
|
||||
|
||||
def _RunCommand(command, working_dir):
|
||||
"""Runs a command and returns the output from that command.
|
||||
|
||||
If the command fails (exit code != 0), the function will exit the process.
|
||||
|
||||
Returns:
|
||||
A tuple containing the stdout and stderr outputs as strings.
|
||||
"""
|
||||
env = os.environ.copy()
|
||||
p = subprocess.Popen(command,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, env=env,
|
||||
cwd=working_dir, universal_newlines=True)
|
||||
std_output, err_output = p.communicate()
|
||||
p.stdout.close()
|
||||
p.stderr.close()
|
||||
if p.returncode != 0:
|
||||
logging.error('Command failed: %s\n'
|
||||
'stdout:\n%s\n'
|
||||
'stderr:\n%s\n', ' '.join(command), std_output, err_output)
|
||||
sys.exit(p.returncode)
|
||||
return std_output, err_output
|
||||
@ -1,138 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||||
#
|
||||
# Use of this source code is governed by a BSD-style license
|
||||
# that can be found in the LICENSE file in the root of the source
|
||||
# tree. An additional intellectual property rights grant can be found
|
||||
# in the file PATENTS. All contributing project authors may
|
||||
# be found in the AUTHORS file in the root of the source tree.
|
||||
|
||||
import os.path
|
||||
import sys
|
||||
import unittest
|
||||
import check_3pp
|
||||
|
||||
SCRIPT_DIR = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
|
||||
CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
|
||||
os.pardir))
|
||||
TEST_DATA_DIR = os.path.join(SCRIPT_DIR, 'testdata',
|
||||
'check_third_party_changes')
|
||||
sys.path.append(CHECKOUT_SRC_DIR)
|
||||
from presubmit_test_mocks import MockInputApi, MockOutputApi, MockFile
|
||||
|
||||
|
||||
class CheckThirdPartyChangesTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self._input_api = MockInputApi()
|
||||
self._output_api = MockOutputApi()
|
||||
|
||||
def testGetChromiumOwnedAddedDeps(self):
|
||||
self._input_api.files = [
|
||||
MockFile('THIRD_PARTY_CHROMIUM_DEPS.json',
|
||||
new_contents=[
|
||||
"""
|
||||
{
|
||||
"dependencies": [
|
||||
"foo",
|
||||
"bar",
|
||||
"buzz",
|
||||
"xyz"
|
||||
]
|
||||
}
|
||||
"""
|
||||
],
|
||||
old_contents=[
|
||||
"""
|
||||
{
|
||||
"dependencies": [
|
||||
"foo",
|
||||
"buzz"
|
||||
]
|
||||
}
|
||||
"""
|
||||
],
|
||||
action='M')
|
||||
]
|
||||
added_deps = check_3pp.GetChromiumOwnedAddedDeps(self._input_api)
|
||||
self.assertEqual(len(added_deps), 2, added_deps)
|
||||
self.assertIn('bar', added_deps)
|
||||
self.assertIn('xyz', added_deps)
|
||||
|
||||
def testCheckNoNotOwned3ppDepsFire(self):
|
||||
self._input_api.presubmit_local_path = os.path.join(TEST_DATA_DIR,
|
||||
'not_owned_dep')
|
||||
errors = check_3pp.CheckNoNotOwned3ppDeps(self._input_api, self._output_api,
|
||||
['webrtc'], ['chromium'])
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn('not_owned', errors[0].message)
|
||||
|
||||
def testCheckNoNotOwned3ppDepsNotFire(self):
|
||||
self._input_api.presubmit_local_path = os.path.join(TEST_DATA_DIR,
|
||||
'not_owned_dep')
|
||||
errors = check_3pp.CheckNoNotOwned3ppDeps(self._input_api, self._output_api,
|
||||
['webrtc', 'not_owned'],
|
||||
['chromium'])
|
||||
self.assertEqual(len(errors), 0, errors)
|
||||
|
||||
def testCheckNoBothOwned3ppDepsFire(self):
|
||||
errors = check_3pp.CheckNoBothOwned3ppDeps(self._output_api, ['foo', 'bar'],
|
||||
['buzz', 'bar'])
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn('bar', errors[0].message)
|
||||
|
||||
def testCheckNoBothOwned3ppDepsNotFire(self):
|
||||
errors = check_3pp.CheckNoBothOwned3ppDeps(self._output_api, ['foo', 'bar'],
|
||||
['buzz'])
|
||||
self.assertEqual(len(errors), 0, errors)
|
||||
|
||||
def testCheckNoChangesInAutoImportedDepsFire(self):
|
||||
self._input_api.files = [
|
||||
MockFile('third_party/chromium/source.js')
|
||||
]
|
||||
errors = check_3pp.CheckNoChangesInAutoImportedDeps(self._input_api,
|
||||
self._output_api,
|
||||
['webrtc'],
|
||||
['chromium'], [],
|
||||
None)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn('chromium', errors[0].message)
|
||||
|
||||
def testCheckNoChangesInAutoImportedDepsNotFire(self):
|
||||
self._input_api.files = [
|
||||
MockFile('third_party/webrtc/source.js')
|
||||
]
|
||||
errors = check_3pp.CheckNoChangesInAutoImportedDeps(self._input_api,
|
||||
self._output_api,
|
||||
['webrtc'],
|
||||
['chromium'], [],
|
||||
None)
|
||||
self.assertEqual(len(errors), 0, errors)
|
||||
|
||||
def testCheckNoChangesInAutoImportedDepsNotFireOnNewlyAdded(self):
|
||||
self._input_api.files = [
|
||||
MockFile('third_party/chromium/source.js')
|
||||
]
|
||||
errors = check_3pp.CheckNoChangesInAutoImportedDeps(self._input_api,
|
||||
self._output_api,
|
||||
['webrtc'],
|
||||
['chromium'],
|
||||
['chromium'], None)
|
||||
self.assertEqual(len(errors), 0, errors)
|
||||
|
||||
def testCheckNoChangesInAutoImportedDepsNotFireOnSpecialTag(self):
|
||||
self._input_api.files = [
|
||||
MockFile('third_party/chromium/source.js')
|
||||
]
|
||||
self._input_api.change.tags['NO_AUTOIMPORT_DEPS_CHECK'] = 'True'
|
||||
errors = check_3pp.CheckNoChangesInAutoImportedDeps(self._input_api,
|
||||
self._output_api,
|
||||
['webrtc'],
|
||||
['chromium'],
|
||||
[], None)
|
||||
self.assertEqual(len(errors), 0, errors)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user