diff --git a/tools/continuous_build/build_internal/masters/master.webrtc/master.cfg b/tools/continuous_build/build_internal/masters/master.webrtc/master.cfg index 1f2c6e580d..0675a7c941 100755 --- a/tools/continuous_build/build_internal/masters/master.webrtc/master.cfg +++ b/tools/continuous_build/build_internal/masters/master.webrtc/master.cfg @@ -147,6 +147,20 @@ windows_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS, linux_chrome_webrtc_tests = utils.GetEnabledTests(CHROME_WEBRTC_TESTS, 'Linux') ####### FACTORIES +CHROME_SVN_URL = 'http://src.chromium.org/svn/trunk/src' +CHROME_LKGR_URL = 'http://chromium-status.appspot.com/lkgr' +CHROME_GCLIENT_SOLUTION_NAME='src' +CHROME_CUSTOM_DEPS_LIST = [ + ('src/third_party/webrtc', 'http://webrtc.googlecode.com/svn/stable/src'), + ('src/third_party/WebKit/LayoutTests', None), + ('src/webkit/data/layout_tests/LayoutTests', None), +] + +VALGRIND_DEPS_LIST = [ + ('trunk/third_party/valgrind', + 'http://src.chromium.org/svn/trunk/deps/third_party/valgrind/binaries'), +] + # Linux linux_factory_64_dbg = utils.WebRTCLinuxFactory( utils.BuildStatusOracle('linux_factory_64_dbg')) @@ -170,11 +184,14 @@ linux_factory_32_dbg.EnableCoverage( linux_factory_32_dbg.EnableBuild(build32=True) linux_factory_32_dbg.EnableTests(linux_normal_tests) +# The video factory needs to prepare for valgrind since some tests will run +# under tooling. Not all tests will though since that would be too slow. linux_factory_video = utils.WebRTCLinuxFactory( - utils.BuildStatusOracle('linux_factory_video')) + utils.BuildStatusOracle('linux_factory_video'), + custom_deps_list=VALGRIND_DEPS_LIST) linux_factory_video.EnableCoverage( coverage_url='http://webrtc-build-bot-se.lul/coverage/') -linux_factory_video.EnableBuild() +linux_factory_video.EnableBuild(compile_for_memory_tooling=True) linux_factory_video.EnableTests(linux_physical_machine_tests) chromeos_factory = utils.WebRTCLinuxFactory( @@ -182,15 +199,6 @@ chromeos_factory = utils.WebRTCLinuxFactory( chromeos_factory.EnableBuild(chrome_os=True) chromeos_factory.EnableTests(linux_normal_tests) -CHROME_SVN_URL = 'http://src.chromium.org/svn/trunk/src' -CHROME_LKGR_URL = 'http://chromium-status.appspot.com/lkgr' -CHROME_GCLIENT_SOLUTION_NAME='src' -CHROME_CUSTOM_DEPS_LIST = [ - ('src/third_party/webrtc', 'http://webrtc.googlecode.com/svn/stable/src'), - ('src/third_party/WebKit/LayoutTests', None), - ('src/webkit/data/layout_tests/LayoutTests', None), -] - linux_chrome_factory = utils.WebRTCChromeFactory( utils.BuildStatusOracle('linux_chrome'), gclient_solution_name=CHROME_GCLIENT_SOLUTION_NAME, @@ -216,12 +224,9 @@ linux_clang.EnableBuild(clang=True) linux_clang.EnableTests(linux_normal_tests) linux_valgrind = utils.WebRTCLinuxFactory( - utils.BuildStatusOracle('linux_valgrind'), valgrind_enabled=True, - custom_deps_list=[ - ('trunk/third_party/valgrind', - 'http://src.chromium.org/svn/trunk/deps/third_party/valgrind/binaries'), - ]) -linux_valgrind.EnableBuild(release=True) + utils.BuildStatusOracle('linux_valgrind'), run_with_memcheck=True, + custom_deps_list=VALGRIND_DEPS_LIST) +linux_valgrind.EnableBuild(release=True, compile_for_memory_tooling=True) # Filter out disabled Valgrind tests: valgrind_tests = filter(lambda test: test not in VALGRIND_DISABLED_TESTS, linux_normal_tests) diff --git a/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py b/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py index 500ab5c43a..30e95d8728 100755 --- a/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py +++ b/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py @@ -32,7 +32,7 @@ WEBRTC_SVN_LOCATION = 'http://webrtc.googlecode.com/svn/trunk' WEBRTC_TRUNK_DIR = 'build/trunk' WEBRTC_BUILD_DIR = 'build' -VALGRIND_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '-t', 'cmdline'] +MEMCHECK_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '-t'] DEFAULT_COVERAGE_DIR = '/var/www/coverage' DEFAULT_BLOAT_DIR = '/var/www/bloat' @@ -615,13 +615,14 @@ class WebRTCLinuxFactory(WebRTCFactory): """ def __init__(self, build_status_oracle, is_try_slave=False, - valgrind_enabled=False, custom_deps_list=None): + run_with_memcheck=False, custom_deps_list=None): WebRTCFactory.__init__(self, build_status_oracle=build_status_oracle, is_try_slave=is_try_slave, custom_deps_list=custom_deps_list) self.build_enabled = False self.coverage_enabled = False - self.valgrind_enabled = valgrind_enabled + self.run_with_memcheck = run_with_memcheck + self.compile_for_memory_tooling = False def EnableCoverage(self, coverage_url, coverage_dir=DEFAULT_COVERAGE_DIR): """Enables coverage measurements using LCOV/GCOV. @@ -641,11 +642,12 @@ class WebRTCLinuxFactory(WebRTCFactory): self.coverage_dir = coverage_dir def EnableBuild(self, release=False, build32=False, chrome_os=False, - clang=False): + clang=False, compile_for_memory_tooling=False): if build32: self.gyp_params.append('-Dtarget_arch=ia32') self.build_enabled = True + self.compile_for_memory_tooling = compile_for_memory_tooling self.release = release self.AddSmartCleanStep() @@ -653,7 +655,7 @@ class WebRTCLinuxFactory(WebRTCFactory): # Valgrind bots need special GYP defines to enable memory profiling # friendly compilation. They already has a custom .gclient configuration # file created so they don't need one being generated like the other bots. - if self.valgrind_enabled: + if self.compile_for_memory_tooling: for gyp_define in MEMORY_TOOLS_GYP_DEFINES: self.gyp_params.append('-D' + gyp_define) self.AddGclientSyncStep() @@ -683,8 +685,8 @@ class WebRTCLinuxFactory(WebRTCFactory): if cmd is None: test_folder = 'Release' if self.release else 'Debug' cmd = ['out/%s/%s' % (test_folder, test)] - if self.valgrind_enabled: - cmd = VALGRIND_CMD + cmd + if self.run_with_memcheck: + cmd = MEMCHECK_CMD + cmd self.AddCommonStep(cmd, descriptor=descriptor, halt_build_on_failure=False) def AddXvfbTestRunStep(self, test_name, test_binary, test_arguments=''): @@ -791,12 +793,25 @@ class WebRTCLinuxFactory(WebRTCFactory): # TODO(phoglund): Enable the full stack test once it is completed and # nonflaky. binary = 'out/Debug/vie_auto_test' + filter = '-ViEVideoVerificationTest.RunsFullStack*:ViERtpFuzzTest*' args = ( - '--automated --gtest_filter="' - '-ViEVideoVerificationTest.RunsFullStackWithoutErrors" ' - '--capture_test_ensure_resolution_alignment_in_capture_device=false') + '--automated --gtest_filter="%s" ' + '--capture_test_ensure_resolution_alignment_in_capture_device=false') + args = args % filter self.AddXvfbTestRunStep(test_name=test, test_binary=binary, test_arguments=args) + + # Set up the fuzz tests as a separate step under memcheck. + # If this test is run we require that we have compiled for memory tools. + # We need to use some special black magic here: -- is replaced with ++ + # when calling the webrtc_tests.sh script since we want those parameters + # to not be caught by webrtc_tests.sh's options parser, but be passed on + # to vie_auto_test. This is a part of webrtc_tests.sh's contract. + assert self.compile_for_memory_tooling + fuzz_binary = (' '.join(MEMCHECK_CMD) + ' ' + binary + + ' ++automated ++gtest_filter=ViERtpFuzzTest*') + self.AddXvfbTestRunStep(test_name=test + ' (fuzz tests)', + test_binary=fuzz_binary) elif test == 'video_render_module_test': self.AddXvfbTestRunStep(test_name=test, test_binary='out/Debug/video_render_module_test') diff --git a/tools/valgrind-webrtc/webrtc_tests.py b/tools/valgrind-webrtc/webrtc_tests.py index b532a1f130..421dc2c59b 100755 --- a/tools/valgrind-webrtc/webrtc_tests.py +++ b/tools/valgrind-webrtc/webrtc_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. # # Use of this source code is governed by a BSD-style license # that can be found in the LICENSE file in the root of the source @@ -7,10 +7,14 @@ # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. -''' Runs various WebRTC tests through valgrind_test.py. +'''Runs various WebRTC tests through valgrind_test.py. -This script inherits the chrome_tests.py in Chrome, replacing its tests with -our own in WebRTC instead. +This script inherits the chrome_tests.py in Chrome, replacing its tests. We do +this by taking chrome's faux cmdline test and making that the standard, so that +we effectively can pass in any binary we feel like. It's also possible to pass +arguments to the test, provided that the arguments do not contain dashes (these +can be "escaped" by passing + instead, so -a becomes +a, and --my-option becomes +++my_option). ''' import optparse @@ -20,70 +24,11 @@ import logging_utils import chrome_tests -class WebRTCTests(chrome_tests.ChromeTests): - # WebRTC tests, similar functions for each tests as the Chrome tests in the - # parent class. - def TestSignalProcessing(self): - return self.SimpleTest("signal_processing", "signal_processing_unittests") - - def TestResampler(self): - return self.SimpleTest("resampler", "resampler_unittests") - - def TestVAD(self): - return self.SimpleTest("vad", "vad_unittests") - - def TestCNG(self): - return self.SimpleTest("cng", "cng_unittests") - - def TestG711(self): - return self.SimpleTest("g711", "g711_unittests") - - def TestG722(self): - return self.SimpleTest("g722", "g722_unittests") - - def TestPCM16B(self): - return self.SimpleTest("pcm16b", "pcm16b_unittests") - - def TestNetEQ(self): - return self.SimpleTest("neteq", "neteq_unittests") - - def TestAudioConferenceMixer(self): - return self.SimpleTest("audio_conference_mixer", "audio_conference_mixer_unittests") - - def TestMediaFile(self): - return self.SimpleTest("media_file", "media_file_unittests") - - def TestRTPRTCP(self): - return self.SimpleTest("rtp_rtcp", "rtp_rtcp_unittests") - - def TestBWE(self): - return self.SimpleTest("test_bwe", "test_bwe") - - def TestUDPTransport(self): - return self.SimpleTest("udp_transport", "udp_transport_unittests") - - def TestWebRTCUtility(self): - return self.SimpleTest("webrtc_utility", "webrtc_utility_unittests") - - def TestVP8(self): - return self.SimpleTest("vp8", "vp8_unittests") - - def TestVideoCoding(self): - return self.SimpleTest("video_coding", "video_coding_unittests") - - def TestVideoProcessing(self): - return self.SimpleTest("video_processing", "video_processing_unittests") - - def TestSystemWrappers(self): - return self.SimpleTest("system_wrappers", "system_wrappers_unittests") - - def TestTestSupport(self): - return self.SimpleTest("test_support", "test_support_unittests") - def _main(_): parser = optparse.OptionParser("usage: %prog -b -t " - "[-t ...]") - parser.disable_interspersed_args() + "[-t ...] " + "NOTE: when passing arguments to all tests, " + " replace any - with +.") parser.add_option("-b", "--build_dir", help="the location of the compiler output") parser.add_option("-t", "--test", action="append", default=[], @@ -119,38 +64,25 @@ def _main(_): if len(options.test) != 1 and options.gtest_filter: parser.error("--gtest_filter and multiple tests don't make sense together") + # Performs the deferred-argument black magic described in the usage. + translated_args = map(lambda arg: arg.replace('+', '-'), args) + for t in options.test: - tests = WebRTCTests(options, args, t) + tests = chrome_tests.ChromeTests(options, translated_args, t) ret = tests.Run() if ret: return ret return 0 if __name__ == "__main__": - # Overwrite the ChromeTests tests dictionary with our WebRTC tests. - # The cmdline option allows the user to pass any executable as parameter to - # the test script, which is useful when developing new tests that are not yet - # present in this script. + # Overwrite the ChromeTests tests dictionary. The cmdline option allows the + # user to pass any executable as parameter to the test script, so we'll use + # that to get our binaries in (hackish but convenient). chrome_tests.ChromeTests._test_list = { "cmdline": chrome_tests.ChromeTests.RunCmdLine, - "signal_processing": WebRTCTests.TestSignalProcessing, - "resampler": WebRTCTests.TestResampler, - "vad": WebRTCTests.TestVAD, - "cng": WebRTCTests.TestCNG, - "g711": WebRTCTests.TestG711, - "g722": WebRTCTests.TestG722, - "pcm16b": WebRTCTests.TestPCM16B, - "neteq": WebRTCTests.TestNetEQ, - "audio_conference_mixer": WebRTCTests.TestAudioConferenceMixer, - "media_file": WebRTCTests.TestMediaFile, - "rtp_rtcp": WebRTCTests.TestRTPRTCP, - "test_bwe": WebRTCTests.TestBWE, - "udp_transport": WebRTCTests.TestUDPTransport, - "webrtc_utility": WebRTCTests.TestWebRTCUtility, - "vp8": WebRTCTests.TestVP8, - "video_coding": WebRTCTests.TestVideoCoding, - "video_processing": WebRTCTests.TestVideoProcessing, - "system_wrappers": WebRTCTests.TestSystemWrappers, - "test_support": WebRTCTests.TestTestSupport, } + + # We do this so the user can write -t instead of -t cmdline . + sys.argv.insert(sys.argv.index('-t') + 1, 'cmdline') + print sys.argv ret = _main(sys.argv) - sys.exit(ret) \ No newline at end of file + sys.exit(ret) \ No newline at end of file