Revert 2224 - Revert 2213 - Moved the fuzz test under valgrind and added valgrind support to the video bot.

CL 2213 did not need to be reverted. 

BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/579008

TBR=phoglund@webrtc.org
Review URL: https://webrtc-codereview.appspot.com/569013

TBR=turaj@webrtc.org
Review URL: https://webrtc-codereview.appspot.com/576010

git-svn-id: http://webrtc.googlecode.com/svn/trunk@2227 4adac7df-926f-26a2-2b94-8c16560cd09d

tools/continuous_build/build_internal/masters/master.webrtc/master.cfg

@@ -147,6 +147,20 @@ windows_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS,
 linux_chrome_webrtc_tests = utils.GetEnabledTests(CHROME_WEBRTC_TESTS, 'Linux')
 
 ####### FACTORIES
+CHROME_SVN_URL = 'http://src.chromium.org/svn/trunk/src'
+CHROME_LKGR_URL = 'http://chromium-status.appspot.com/lkgr'
+CHROME_GCLIENT_SOLUTION_NAME='src'
+CHROME_CUSTOM_DEPS_LIST = [
+  ('src/third_party/webrtc', 'http://webrtc.googlecode.com/svn/stable/src'),
+  ('src/third_party/WebKit/LayoutTests', None),
+  ('src/webkit/data/layout_tests/LayoutTests', None),
+]
+
+VALGRIND_DEPS_LIST = [
+  ('trunk/third_party/valgrind',
+    'http://src.chromium.org/svn/trunk/deps/third_party/valgrind/binaries'),
+]
+
 # Linux
 linux_factory_64_dbg = utils.WebRTCLinuxFactory(
     utils.BuildStatusOracle('linux_factory_64_dbg'))
@@ -170,11 +184,14 @@ linux_factory_32_dbg.EnableCoverage(
 linux_factory_32_dbg.EnableBuild(build32=True)
 linux_factory_32_dbg.EnableTests(linux_normal_tests)
 
+# The video factory needs to prepare for valgrind since some tests will run
+# under tooling. Not all tests will though since that would be too slow.
 linux_factory_video = utils.WebRTCLinuxFactory(
-    utils.BuildStatusOracle('linux_factory_video'))
+    utils.BuildStatusOracle('linux_factory_video'),
+    custom_deps_list=VALGRIND_DEPS_LIST)
 linux_factory_video.EnableCoverage(
     coverage_url='http://webrtc-build-bot-se.lul/coverage/')
-linux_factory_video.EnableBuild()
+linux_factory_video.EnableBuild(compile_for_memory_tooling=True)
 linux_factory_video.EnableTests(linux_physical_machine_tests)
 
 chromeos_factory = utils.WebRTCLinuxFactory(
@@ -182,15 +199,6 @@ chromeos_factory = utils.WebRTCLinuxFactory(
 chromeos_factory.EnableBuild(chrome_os=True)
 chromeos_factory.EnableTests(linux_normal_tests)
 
-CHROME_SVN_URL = 'http://src.chromium.org/svn/trunk/src'
-CHROME_LKGR_URL = 'http://chromium-status.appspot.com/lkgr'
-CHROME_GCLIENT_SOLUTION_NAME='src'
-CHROME_CUSTOM_DEPS_LIST = [
-  ('src/third_party/webrtc', 'http://webrtc.googlecode.com/svn/stable/src'),
-  ('src/third_party/WebKit/LayoutTests', None),
-  ('src/webkit/data/layout_tests/LayoutTests', None),
-]
-
 linux_chrome_factory = utils.WebRTCChromeFactory(
     utils.BuildStatusOracle('linux_chrome'),
     gclient_solution_name=CHROME_GCLIENT_SOLUTION_NAME,
@@ -216,12 +224,9 @@ linux_clang.EnableBuild(clang=True)
 linux_clang.EnableTests(linux_normal_tests)
 
 linux_valgrind = utils.WebRTCLinuxFactory(
-    utils.BuildStatusOracle('linux_valgrind'), valgrind_enabled=True,
+    utils.BuildStatusOracle('linux_valgrind'), run_with_memcheck=True,
-    custom_deps_list=[
+    custom_deps_list=VALGRIND_DEPS_LIST)
-        ('trunk/third_party/valgrind',
+linux_valgrind.EnableBuild(release=True, compile_for_memory_tooling=True)
-         'http://src.chromium.org/svn/trunk/deps/third_party/valgrind/binaries'),
-    ])
-linux_valgrind.EnableBuild(release=True)
 # Filter out disabled Valgrind tests:
 valgrind_tests = filter(lambda test: test not in VALGRIND_DISABLED_TESTS,
                         linux_normal_tests)

tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py

@@ -32,7 +32,7 @@ WEBRTC_SVN_LOCATION = 'http://webrtc.googlecode.com/svn/trunk'
 WEBRTC_TRUNK_DIR = 'build/trunk'
 WEBRTC_BUILD_DIR = 'build'
 
-VALGRIND_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '-t', 'cmdline']
+MEMCHECK_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '-t']
 
 DEFAULT_COVERAGE_DIR = '/var/www/coverage'
 DEFAULT_BLOAT_DIR = '/var/www/bloat'
@@ -615,13 +615,14 @@ class WebRTCLinuxFactory(WebRTCFactory):
   """
 
   def __init__(self, build_status_oracle, is_try_slave=False,
-               valgrind_enabled=False, custom_deps_list=None):
+               run_with_memcheck=False, custom_deps_list=None):
     WebRTCFactory.__init__(self, build_status_oracle=build_status_oracle,
                            is_try_slave=is_try_slave,
                            custom_deps_list=custom_deps_list)
     self.build_enabled = False
     self.coverage_enabled = False
-    self.valgrind_enabled = valgrind_enabled
+    self.run_with_memcheck = run_with_memcheck
+    self.compile_for_memory_tooling = False
 
   def EnableCoverage(self, coverage_url, coverage_dir=DEFAULT_COVERAGE_DIR):
     """Enables coverage measurements using LCOV/GCOV.
@@ -641,11 +642,12 @@ class WebRTCLinuxFactory(WebRTCFactory):
     self.coverage_dir = coverage_dir
 
   def EnableBuild(self, release=False, build32=False, chrome_os=False,
-                  clang=False):
+                  clang=False, compile_for_memory_tooling=False):
     if build32:
       self.gyp_params.append('-Dtarget_arch=ia32')
 
     self.build_enabled = True
+    self.compile_for_memory_tooling = compile_for_memory_tooling
     self.release = release
 
     self.AddSmartCleanStep()
@@ -653,7 +655,7 @@ class WebRTCLinuxFactory(WebRTCFactory):
     # Valgrind bots need special GYP defines to enable memory profiling
     # friendly compilation. They already has a custom .gclient configuration
     # file created so they don't need one being generated like the other bots.
-    if self.valgrind_enabled:
+    if self.compile_for_memory_tooling:
       for gyp_define in MEMORY_TOOLS_GYP_DEFINES:
         self.gyp_params.append('-D' + gyp_define)
     self.AddGclientSyncStep()
@@ -683,8 +685,8 @@ class WebRTCLinuxFactory(WebRTCFactory):
     if cmd is None:
       test_folder = 'Release' if self.release else 'Debug'
       cmd = ['out/%s/%s' % (test_folder, test)]
-    if self.valgrind_enabled:
+    if self.run_with_memcheck:
-      cmd = VALGRIND_CMD + cmd
+      cmd = MEMCHECK_CMD + cmd
     self.AddCommonStep(cmd, descriptor=descriptor, halt_build_on_failure=False)
 
   def AddXvfbTestRunStep(self, test_name, test_binary, test_arguments=''):
@@ -791,12 +793,25 @@ class WebRTCLinuxFactory(WebRTCFactory):
       # TODO(phoglund): Enable the full stack test once it is completed and
       # nonflaky.
       binary = 'out/Debug/vie_auto_test'
+      filter = '-ViEVideoVerificationTest.RunsFullStack*:ViERtpFuzzTest*'
       args = (
-          '--automated --gtest_filter="'
+        '--automated --gtest_filter="%s" '
-          '-ViEVideoVerificationTest.RunsFullStackWithoutErrors" '
+        '--capture_test_ensure_resolution_alignment_in_capture_device=false')
-          '--capture_test_ensure_resolution_alignment_in_capture_device=false')
+      args = args % filter
       self.AddXvfbTestRunStep(test_name=test, test_binary=binary,
                               test_arguments=args)
+
+      # Set up the fuzz tests as a separate step under memcheck.
+      # If this test is run we require that we have compiled for memory tools.
+      # We need to use some special black magic here: -- is replaced with ++
+      # when calling the webrtc_tests.sh script since we want those parameters
+      # to not be caught by webrtc_tests.sh's options parser, but be passed on
+      # to vie_auto_test. This is a part of webrtc_tests.sh's contract.
+      assert self.compile_for_memory_tooling
+      fuzz_binary = (' '.join(MEMCHECK_CMD) + ' ' + binary +
+                     ' ++automated ++gtest_filter=ViERtpFuzzTest*')
+      self.AddXvfbTestRunStep(test_name=test + ' (fuzz tests)',
+                              test_binary=fuzz_binary)
     elif test == 'video_render_module_test':
       self.AddXvfbTestRunStep(test_name=test,
                               test_binary='out/Debug/video_render_module_test')

tools/valgrind-webrtc/webrtc_tests.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 #
 # Use of this source code is governed by a BSD-style license
 # that can be found in the LICENSE file in the root of the source
@@ -7,10 +7,14 @@
 # in the file PATENTS.  All contributing project authors may
 # be found in the AUTHORS file in the root of the source tree.
 
-''' Runs various WebRTC tests through valgrind_test.py.
+'''Runs various WebRTC tests through valgrind_test.py.
 
-This script inherits the chrome_tests.py in Chrome, replacing its tests with
+This script inherits the chrome_tests.py in Chrome, replacing its tests. We do
-our own in WebRTC instead.
+this by taking chrome's faux cmdline test and making that the standard, so that
+we effectively can pass in any binary we feel like. It's also possible to pass
+arguments to the test, provided that the arguments do not contain dashes (these
+can be "escaped" by passing + instead, so -a becomes +a, and --my-option becomes
+++my_option).
 '''
 
 import optparse
@@ -20,70 +24,11 @@ import logging_utils
 
 import chrome_tests
 
-class WebRTCTests(chrome_tests.ChromeTests):
-  # WebRTC tests, similar functions for each tests as the Chrome tests in the
-  # parent class.
-  def TestSignalProcessing(self):
-    return self.SimpleTest("signal_processing", "signal_processing_unittests")
-
-  def TestResampler(self):
-    return self.SimpleTest("resampler", "resampler_unittests")
-
-  def TestVAD(self):
-    return self.SimpleTest("vad", "vad_unittests")
-
-  def TestCNG(self):
-    return self.SimpleTest("cng", "cng_unittests")
-
-  def TestG711(self):
-    return self.SimpleTest("g711", "g711_unittests")
-
-  def TestG722(self):
-    return self.SimpleTest("g722", "g722_unittests")
-
-  def TestPCM16B(self):
-    return self.SimpleTest("pcm16b", "pcm16b_unittests")
-
-  def TestNetEQ(self):
-    return self.SimpleTest("neteq", "neteq_unittests")
-
-  def TestAudioConferenceMixer(self):
-    return self.SimpleTest("audio_conference_mixer", "audio_conference_mixer_unittests")
-
-  def TestMediaFile(self):
-    return self.SimpleTest("media_file", "media_file_unittests")
-
-  def TestRTPRTCP(self):
-    return self.SimpleTest("rtp_rtcp", "rtp_rtcp_unittests")
-
-  def TestBWE(self):
-    return self.SimpleTest("test_bwe", "test_bwe")
-
-  def TestUDPTransport(self):
-    return self.SimpleTest("udp_transport", "udp_transport_unittests")
-
-  def TestWebRTCUtility(self):
-    return self.SimpleTest("webrtc_utility", "webrtc_utility_unittests")
-
-  def TestVP8(self):
-    return self.SimpleTest("vp8", "vp8_unittests")
-
-  def TestVideoCoding(self):
-    return self.SimpleTest("video_coding", "video_coding_unittests")
-
-  def TestVideoProcessing(self):
-    return self.SimpleTest("video_processing", "video_processing_unittests")
-
-  def TestSystemWrappers(self):
-    return self.SimpleTest("system_wrappers", "system_wrappers_unittests")
-
-  def TestTestSupport(self):
-    return self.SimpleTest("test_support", "test_support_unittests")
-
 def _main(_):
   parser = optparse.OptionParser("usage: %prog -b <dir> -t <test> "
-                                 "[-t <test> ...]")
+                                 "[-t <test> ...] <arguments to all tests>"
-  parser.disable_interspersed_args()
+                                 "NOTE: when passing arguments to all tests, "
+                                 "      replace any - with +.")
   parser.add_option("-b", "--build_dir",
                     help="the location of the compiler output")
   parser.add_option("-t", "--test", action="append", default=[],
@@ -119,38 +64,25 @@ def _main(_):
   if len(options.test) != 1 and options.gtest_filter:
     parser.error("--gtest_filter and multiple tests don't make sense together")
 
+  # Performs the deferred-argument black magic described in the usage.
+  translated_args = map(lambda arg: arg.replace('+', '-'), args)
+
   for t in options.test:
-    tests = WebRTCTests(options, args, t)
+    tests = chrome_tests.ChromeTests(options, translated_args, t)
     ret = tests.Run()
     if ret: return ret
   return 0
 
 if __name__ == "__main__":
-  # Overwrite the ChromeTests tests dictionary with our WebRTC tests. 
+  # Overwrite the ChromeTests tests dictionary. The cmdline option allows the
-  # The cmdline option allows the user to pass any executable as parameter to
+  # user to pass any executable as parameter to the test script, so we'll use
-  # the test script, which is useful when developing new tests that are not yet
+  # that to get our binaries in (hackish but convenient).
-  # present in this script.
   chrome_tests.ChromeTests._test_list = {
     "cmdline": chrome_tests.ChromeTests.RunCmdLine,
-    "signal_processing": WebRTCTests.TestSignalProcessing,
-    "resampler": WebRTCTests.TestResampler,
-    "vad": WebRTCTests.TestVAD,
-    "cng": WebRTCTests.TestCNG,
-    "g711": WebRTCTests.TestG711,
-    "g722": WebRTCTests.TestG722,
-    "pcm16b": WebRTCTests.TestPCM16B,
-    "neteq": WebRTCTests.TestNetEQ,
-    "audio_conference_mixer": WebRTCTests.TestAudioConferenceMixer,
-    "media_file": WebRTCTests.TestMediaFile,
-    "rtp_rtcp": WebRTCTests.TestRTPRTCP,
-    "test_bwe": WebRTCTests.TestBWE,
-    "udp_transport": WebRTCTests.TestUDPTransport,
-    "webrtc_utility": WebRTCTests.TestWebRTCUtility,
-    "vp8": WebRTCTests.TestVP8,
-    "video_coding": WebRTCTests.TestVideoCoding,
-    "video_processing": WebRTCTests.TestVideoProcessing,
-    "system_wrappers": WebRTCTests.TestSystemWrappers,
-    "test_support": WebRTCTests.TestTestSupport,
   }
+
+  # We do this so the user can write -t <binary> instead of -t cmdline <binary>.
+  sys.argv.insert(sys.argv.index('-t') + 1, 'cmdline')
+  print sys.argv
   ret = _main(sys.argv)
   sys.exit(ret)