Implemented quick builds (e.g. only do full clean if the previous build failed).

BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/400009

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1752 4adac7df-926f-26a2-2b94-8c16560cd09d

tools/continuous_build/master.cfg

@@ -173,70 +173,79 @@ windows_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS,
                                                        'Windows')
 
 ############# Linux Builders #######################################
-linux_factory_64_dbg = utils.WebRTCLinuxFactory()
+linux_factory_64_dbg = utils.WebRTCLinuxFactory(
+    utils.BuildStatusOracle('linux_factory_64_dbg'))
 linux_factory_64_dbg.EnableBuild()
 linux_factory_64_dbg.EnableTests(linux_normal_tests)
 
-linux_factory_64_dbg_no_coverage = utils.WebRTCLinuxFactory()
+linux_factory_32_release = utils.WebRTCLinuxFactory(
-linux_factory_64_dbg_no_coverage.EnableBuild()
+    utils.BuildStatusOracle('linux_factory_32_release'))
-linux_factory_64_dbg_no_coverage.EnableTests(linux_normal_tests)
-
-linux_factory_32_release = utils.WebRTCLinuxFactory()
 linux_factory_32_release.EnableBuild(release=True, build32=True)
 linux_factory_32_release.EnableTests(linux_normal_tests)
 
-linux_factory_64_release = utils.WebRTCLinuxFactory()
+linux_factory_64_release = utils.WebRTCLinuxFactory(
+    utils.BuildStatusOracle('linux_factory_64_release'))
 linux_factory_64_release.EnableBuild(release=True)
 linux_factory_64_release.EnableTests(linux_normal_tests)
 
-linux_factory_32_dbg = utils.WebRTCLinuxFactory()
+linux_factory_32_dbg = utils.WebRTCLinuxFactory(
+    utils.BuildStatusOracle('linux_factory_32_dbg'))
 linux_factory_32_dbg.EnableCoverage(
     coverage_url='http://webrtc-cb-linux-slave-4.cbf.corp.google.com/coverage/')
 linux_factory_32_dbg.EnableBuild(build32=True)
 linux_factory_32_dbg.EnableTests(linux_normal_tests)
 
-linux_factory_video = utils.WebRTCLinuxFactory()
+linux_factory_video = utils.WebRTCLinuxFactory(
+    utils.BuildStatusOracle('linux_factory_video'))
 linux_factory_video.EnableCoverage(
     coverage_url='http://webrtc-build-bot-se.lul/coverage/')
 linux_factory_video.EnableBuild()
 linux_factory_video.EnableTests(linux_physical_machine_tests)
 
-chromeos_factory = utils.WebRTCLinuxFactory()
+chromeos_factory = utils.WebRTCLinuxFactory(
+    utils.BuildStatusOracle('chromeos_factory'))
 chromeos_factory.EnableBuild(chrome_os=True)
 chromeos_factory.EnableTests(linux_normal_tests)
 
-linux_clang = utils.WebRTCLinuxFactory()
+linux_clang = utils.WebRTCLinuxFactory(
+    utils.BuildStatusOracle('linux_clang'))
 linux_clang.EnableBuild(clang=True)
 linux_clang.EnableTests(linux_normal_tests)
 
-linux_valgrind = utils.WebRTCLinuxFactory(valgrind_enabled=True)
+linux_valgrind = utils.WebRTCLinuxFactory(
+    utils.BuildStatusOracle('linux_valgrind'), valgrind_enabled=True)
 linux_valgrind.EnableBuild(release=True)
 # Filter out disabled Valgrind tests:
 valgrind_tests = filter(lambda test: test not in VALGRIND_DISABLED_TESTS,
                         linux_normal_tests)
 linux_valgrind.EnableTests(valgrind_tests)
 
+android_factory = utils.WebRTCAndroidFactory(
+    utils.BuildStatusOracle('android_factory'))
+android_factory.EnableBuild(product='toro')
+
 ############## Mac Builders #######################################
-mac_factory_32_dbg = utils.WebRTCMacFactory()
+mac_factory_32_dbg = utils.WebRTCMacFactory(
+    utils.BuildStatusOracle('mac_factory_32_dbg'))
 mac_factory_32_dbg.EnableBuild(build_type='both')
 mac_factory_32_dbg.EnableTests(mac_normal_tests)
 
-mac_factory_32_release = utils.WebRTCMacFactory()
+mac_factory_32_release = utils.WebRTCMacFactory(
+    utils.BuildStatusOracle('mac_factory_32_release'))
 mac_factory_32_release.EnableBuild(build_type='both', release=True)
 mac_factory_32_release.EnableTests(mac_normal_tests)
 
 ############# Windows Builders #######################################
-win_factory_32_Debug = utils.WebRTCWinFactory()
+win_factory_32_Debug = utils.WebRTCWinFactory(
+    utils.BuildStatusOracle('win_factory_32_debug'))
 win_factory_32_Debug.EnableBuild(configuration='Debug')
 win_factory_32_Debug.EnableTests(windows_normal_tests)
 
-win_factory_32_Release = utils.WebRTCWinFactory()
+win_factory_32_Release = utils.WebRTCWinFactory(
+    utils.BuildStatusOracle('mac_factory_32_release'))
 win_factory_32_Release.EnableBuild(configuration='Release')
 win_factory_32_Release.EnableTests(windows_normal_tests)
 
-android_factory = utils.WebRTCAndroidFactory()
-android_factory.EnableBuild(product='toro')
-
 linux_builder_64_debug = {
       'name': 'Linux64DBG',
       'slavename': 'linux-slave-1',
@@ -319,7 +328,7 @@ linux_builder_gcc_4_6 = {
       'name': 'Linux64DBG-GCC4.6',
       'slavename': 'linux-slave-gcc-4.6',
       'builddir': 'linux-slave-gcc-4.6',
-      'factory': linux_factory_64_dbg_no_coverage,
+      'factory': linux_factory_64_dbg,
       }
 c['builders'] = [
       win_builder_32_debug,

tools/continuous_build/webrtc_buildbot/utils.py

@@ -9,6 +9,7 @@
 
 __author__ = 'ivinnichenko@webrtc.org (Illya Vinnichenko)'
 
+import buildbot
 import os
 import sys
 import urlparse
@@ -26,6 +27,7 @@ SVN_LOCATION = 'http://webrtc.googlecode.com/svn/trunk'
 VALGRIND_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '-t', 'cmdline']
 
 DEFAULT_COVERAGE_DIR = '/var/www/coverage/'
+DEFAULT_MASTER_WORK_DIR = '.'
 
 # Copied from trunk/tools/build/scripts/master/factory/chromium_factory.py
 # but converted to a list since we set defines instead of using an environment
@@ -66,9 +68,16 @@ class WebRTCFactory(factory.BuildFactory):
      can be overridden to create customized build sequences.
   """
 
-  def __init__(self):
+  def __init__(self, build_status_oracle):
+    """Creates the abstract factory.
+
+       Args:
+         build_status_oracle: An instance of BuildStatusOracle which is used to
+             keep track of our build state.
+    """
     factory.BuildFactory.__init__(self)
 
+    self.build_status_oracle = build_status_oracle
     self.properties = properties.Properties()
     self.build_enabled = False
     self.force_sync = False
@@ -101,9 +110,13 @@ class WebRTCFactory(factory.BuildFactory):
 
   def AddCommonStep(self, cmd, descriptor='', workdir='build',
                     halt_build_on_failure=True, warn_on_failure=False):
-    """Adds a common step which will run as a shell command on the slave.
+    """Adds a step which will run as a shell command on the slave.
 
-       A common step can be anything except a test execution step.
+       NOTE: you are recommended to use this method to add new shell commands
+       instead of the base-class addStep method, since steps added here will
+       work with the smart-clean system (e.g. only do a full rebuild if the
+       previous build failed). Steps handled outside this method will not lead
+       to a full rebuild on the next build if they fail.
 
        Args:
          cmd: The command to run. This command follows the contract for
@@ -124,14 +137,21 @@ class WebRTCFactory(factory.BuildFactory):
     # Add spaces to wrap long test names to make waterfall output more compact.
     wrapped_text = self._WrapLongLines(descriptor)
 
-    self.addStep(shell.ShellCommand(command=cmd, workdir=workdir,
+    self.addStep(MonitoredShellCommand(
-                                    description=wrapped_text + ['running...'],
+        build_status_oracle=self.build_status_oracle, command=cmd,
-                                    descriptionDone=wrapped_text,
+        workdir=workdir, description=wrapped_text + ['running...'],
-                                    warnOnFailure=warn_on_failure,
+        descriptionDone=wrapped_text, warnOnFailure=warn_on_failure,
-                                    flunkOnFailure=flunk_on_failure,
+        flunkOnFailure=flunk_on_failure, haltOnFailure=halt_build_on_failure,
-                                    haltOnFailure=halt_build_on_failure,
         name='_'.join(descriptor)))
 
+  def AddSmartCleanStep(self):
+    """Adds a smart clean step.
+
+       Smart clean only cleans the whole repository if the build status oracle
+       thinks the last build failed. Otherwise it cleans just the build output.
+    """
+    self.addStep(SmartClean(self.build_status_oracle))
+
   def AddCommonTestRunStep(self, test, descriptor='', cmd=None,
                            workdir='build/trunk'):
     """Adds a step for running a single test [must be overridden].
@@ -139,6 +159,9 @@ class WebRTCFactory(factory.BuildFactory):
        In general, failing tests should not halt the build and allow other tests
        to execute. A failing test should fail, or 'flunk', the build though.
 
+       Implementations of this method must add new steps through AddCommonStep
+       and not by calling addStep.
+
        Args:
          test: The test binary name. The step will attempt to execute this
              binary in the binary output folder, except if the cmd argument is
@@ -174,6 +197,9 @@ class WebRTCFactory(factory.BuildFactory):
        GYP will generate makefiles or its equivalent in a platform-specific
        manner. A failed GYP step will halt the build.
 
+       Implementations of this method must add new steps through AddCommonStep
+       and not by calling addStep.
+
        Args:
          gyp_file: The root GYP file to use.
          gyp_params: Custom GYP parameters (same semantics as the GYP_PARAMS
@@ -215,6 +241,80 @@ class WebRTCFactory(factory.BuildFactory):
       result.append(line)
     return result
 
+
+class BuildStatusOracle:
+  """Keeps track of a particular build's state.
+
+     The oracle uses files in the default master work directory to keep track
+     of whether a build has failed. It only keeps track of the most recent build
+     until told to forget it.
+  """
+
+  def __init__(self, builder_name):
+    """Creates the oracle.
+
+       Args:
+         builder_name: The name of the associated builder. This name is used
+             in the filename on disk. This name should be unique.
+    """
+    self.builder_name = builder_name
+    self.master_work_dir = DEFAULT_MASTER_WORK_DIR
+
+  def LastBuildFailed(self):
+    failure_file_path = self._GetFailureBuildPath()
+    return os.path.exists(failure_file_path)
+
+  def ForgetLastBuild(self):
+    if self.LastBuildFailed():
+      os.remove(self._GetFailureBuildPath())
+
+  def SetLastBuildAsFailed(self):
+    open(self._GetFailureBuildPath(), 'w').close()
+
+  def _GetFailureBuildPath(self):
+    return os.path.join(self.master_work_dir, self.builder_name + ".failed")
+
+
+class MonitoredShellCommand(ShellCommand):
+  """Wraps a shell command and notifies the oracle if the command fails."""
+
+  def __init__(self, build_status_oracle, **kwargs):
+    ShellCommand.__init__(self, **kwargs)
+
+    self.addFactoryArguments(build_status_oracle=build_status_oracle)
+    self.build_status_oracle = build_status_oracle
+
+  def finished(self, results):
+    if (results == buildbot.status.builder.FAILURE or
+        results == buildbot.status.builder.EXCEPTION):
+      self.build_status_oracle.SetLastBuildAsFailed()
+
+    ShellCommand.finished(self, results)
+
+
+class SmartClean(ShellCommand):
+  """Cleans the repository fully or partially depending on the build state."""
+  def __init__(self, build_status_oracle, **kwargs):
+    ShellCommand.__init__(self, **kwargs)
+
+    self.addFactoryArguments(build_status_oracle=build_status_oracle)
+    self.haltOnFailure = True
+    self.build_status_oracle = build_status_oracle
+
+  def start(self):
+    if self.build_status_oracle.LastBuildFailed():
+      self.build_status_oracle.ForgetLastBuild()
+      self.description = ['Nuke Repository', '(Previous Failed)']
+      self.setCommand(['rm', '-rf', 'trunk'])
+    else:
+      self.description = ['Clean']
+      self.setCommand('rm -rf trunk/out && '
+                      'rm -rf trunk/xcodebuild &&'
+                      'rm -rf trunk/build/Debug &&'
+                      'rm -rf trunk/build/Release')
+    ShellCommand.start(self)
+
+
 class GenerateCodeCoverage(ShellCommand):
   """This custom shell command generates coverage HTML using genhtml.
 
@@ -265,8 +365,8 @@ class GenerateCodeCoverage(ShellCommand):
 class WebRTCAndroidFactory(WebRTCFactory):
   """Sets up the Android build."""
 
-  def __init__(self):
+  def __init__(self, build_status_oracle):
-    WebRTCFactory.__init__(self)
+    WebRTCFactory.__init__(self, build_status_oracle)
 
   def EnableBuild(self, product='toro'):
     prefix = 'rm -rf out/target/product/%s/obj/' % product
@@ -277,25 +377,21 @@ class WebRTCAndroidFactory(WebRTCFactory):
         prefix + 'EXECUTABLES/webrtc_*'
         ]
     cmd = ' ; '.join(cleanup_list)
-    self.addStep(shell.Compile(command=(cmd), workdir='build/trunk',
+    self.AddCommonStep(cmd, descriptor='cleanup', workdir='build/trunk')
-                 description=['cleanup', 'running...'],
+
-                 descriptionDone=['cleanup'], name='cleanup'))
     cmd = 'svn checkout %s external/webrtc' % SVN_LOCATION
-    self.addStep(shell.Compile(command=(cmd),
+    self.AddCommonStep(cmd, descriptor='svn (checkout)', workdir='build/trunk')
-        workdir='build/trunk', description=['svn checkout', 'running...'],
+
-        descriptionDone=['svn checkout'], name='svn (checkout)'))
     cmd = ('source build/envsetup.sh && lunch full_%s-eng '
            '&& mmm external/webrtc showcommands' % product)
-    self.addStep(shell.Compile(command=(cmd),
+    self.AddCommonStep(cmd, descriptor='build', workdir='build/trunk')
-        workdir='build/trunk', description=['build', 'running...'],
-        descriptionDone=['build'], name='build'))
 
 
 class WebRTCChromeFactory(WebRTCFactory):
   """Sets up the Chrome OS build."""
 
-  def __init__(self):
+  def __init__(self, build_status_oracle):
-    WebRTCFactory.__init__(self)
+    WebRTCFactory.__init__(self, build_status_oracle)
 
   def EnableBuild(self):
     self.AddCommonStep(['rm', '-rf', 'src'], descriptor='Cleanup')
@@ -317,8 +413,8 @@ class WebRTCLinuxFactory(WebRTCFactory):
      This factory is quite configurable and can run a variety of builds.
   """
 
-  def __init__(self, valgrind_enabled=False):
+  def __init__(self, build_status_oracle, valgrind_enabled=False):
-    WebRTCFactory.__init__(self)
+    WebRTCFactory.__init__(self, build_status_oracle)
 
     self.coverage_enabled = False
     self.valgrind_enabled = valgrind_enabled
@@ -349,8 +445,7 @@ class WebRTCLinuxFactory(WebRTCFactory):
     self.force_sync = force_sync
     self.release = release
 
-    self.AddCommonStep(['rm', '-rf', 'trunk'], descriptor='Cleanup',
+    self.AddSmartCleanStep()
-                       warn_on_failure=True, halt_build_on_failure=False)
 
     # Valgrind bots need special GYP defines to enable memory profiling
     # friendly compilation. They already has a custom .gclient configuration
@@ -525,8 +620,8 @@ class WebRTCLinuxFactory(WebRTCFactory):
 class WebRTCMacFactory(WebRTCFactory):
   """Sets up the Mac build, both for make and xcode."""
 
-  def __init__(self):
+  def __init__(self, build_status_oracle):
-    WebRTCFactory.__init__(self)
+    WebRTCFactory.__init__(self, build_status_oracle)
     self.build_type = 'both'
     self.allowed_build_types = ['both', 'xcode', 'make']
 
@@ -540,7 +635,7 @@ class WebRTCMacFactory(WebRTCFactory):
       sys.exit(0)
     else:
       self.build_type = build_type
-    self.AddCommonStep(['rm', '-rf', 'trunk'], descriptor='Cleanup')
+    self.AddSmartCleanStep()
     self.AddCommonStep(['gclient', 'config', SVN_LOCATION],
                        descriptor='gclient_config')
     cmd = ['gclient', 'sync']
@@ -591,8 +686,8 @@ class WebRTCWinFactory(WebRTCFactory):
      Allows building with Debug, Release or both in sequence.
   """
 
-  def __init__(self):
+  def __init__(self, build_status_oracle):
-    WebRTCFactory.__init__(self)
+    WebRTCFactory.__init__(self, build_status_oracle)
     self.configuration = 'Debug'
     self.platform = 'x64'
     self.allowed_platforms = ['x64', 'Win32']
@@ -613,7 +708,7 @@ class WebRTCWinFactory(WebRTCFactory):
     else:
       self.configuration = configuration
     if not build_only:
-      self.AddCommonStep(['rm', '-rf', 'trunk'], descriptor='Cleanup')
+      self.AddSmartCleanStep()
       self.AddCommonStep(['gclient', 'config', SVN_LOCATION],
                          descriptor='gclient_config')
       cmd = ['gclient', 'sync']