Prepared tests for running on build bot.

The changes will mostly clarify why PSNR and SSIM thresholds are chosen. I think those explanations will be good since these tests may well be a bit flaky.

BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/343017

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1496 4adac7df-926f-26a2-2b94-8c16560cd09d

src/video_engine/test/auto_test/automated/vie_video_verification_test.cc

@@ -26,13 +26,6 @@ namespace {
 const int kInputWidth = 176;
 const int kInputHeight = 144;
 
-const double kMinPSNR420DefaultBitRateQCIF = 28;
-const double kMinSSIM420DefaultBitRateQCIF = 0.95;
-const double kMinPSNRCodecTestsQCIF = 20;
-const double kMinSSIMCodecTestsQCIF = 0.7;
-const double kMinPSNR50kbpsQCIF = 25;
-const double kMinSSIM50kbpsQCIF = 0.8;
-
 class ViEVideoVerificationTest : public testing::Test {
  protected:
   void SetUp() {
@@ -79,19 +72,25 @@ class ViEVideoVerificationTest : public testing::Test {
   void CompareFiles(const std::string& reference_file,
                     const std::string& test_file,
                     double minimum_psnr, double minimum_ssim) {
+    static const char* kPsnrSsimExplanation =
+        "Don't worry too much about this error if it only happens once. "
+        "It may be because mundane things like unfortunate OS scheduling. "
+        "If it keeps happening over and over though it's a cause of concern.";
+
     webrtc::test::QualityMetricsResult psnr;
     int error = I420PSNRFromFiles(reference_file.c_str(), test_file.c_str(),
                                   kInputWidth, kInputHeight, &psnr);
+
     EXPECT_EQ(0, error) << "PSNR routine failed - output files missing?";
-    EXPECT_GT(psnr.average, minimum_psnr);
+    EXPECT_GT(psnr.average, minimum_psnr) << kPsnrSsimExplanation;
 
     webrtc::test::QualityMetricsResult ssim;
     error = I420SSIMFromFiles(reference_file.c_str(), test_file.c_str(),
                               kInputWidth, kInputHeight, &ssim);
     EXPECT_EQ(0, error) << "SSIM routine failed - output files missing?";
-    EXPECT_GT(ssim.average, minimum_ssim);  // 1 = perfect, -1 = terrible
+    EXPECT_GT(ssim.average, minimum_ssim) << kPsnrSsimExplanation;
 
-    ViETest::Log("Results: PSNR: %f (db)   SSIM: %f",
+    ViETest::Log("Results: PSNR is %f (dB), SSIM is %f (1 is perfect)",
                  psnr.average, ssim.average);
   }
 
@@ -116,14 +115,20 @@ class ViEVideoVerificationTest : public testing::Test {
   }
 };
 
-TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors)  {
+TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
   ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight,
                                    local_file_renderer_,
                                    remote_file_renderer_));
   std::string output_file = remote_file_renderer_->GetFullOutputPath();
   StopRenderers();
-  CompareFiles(input_file_, output_file, kMinPSNR420DefaultBitRateQCIF,
+
-               kMinSSIM420DefaultBitRateQCIF);
+  // The I420 test should give pretty good values since it's a lossless codec
+  // running on the default bitrate. It should average about 30 dB but there
+  // may be cases where it dips as low as 26 under adverse conditions.
+  const double kExpectedMinimumPSNR = 28;
+  const double kExpectedMinimumSSIM = 0.95;
+  CompareFiles(input_file_, output_file, kExpectedMinimumPSNR,
+               kExpectedMinimumSSIM);
 }
 
 TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors)  {
@@ -138,18 +143,21 @@ TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors)  {
   // The reason is that it is hard to say when the three consecutive tests
   // switch over into each other, at which point we would have to restart the
   // original to get a fair comparison.
-  CompareFiles(reference_file, output_file, kMinPSNRCodecTestsQCIF,
+  //
-               kMinSSIMCodecTestsQCIF);
+  // The PSNR and SSIM values are quite low here, and they have to be since
-
+  // the codec switches will lead to lag in the output. This is considered
-  // TODO(phoglund): The values should be higher. Investigate why the remote
+  // acceptable, but it probably shouldn't get worse than this.
-  // file turns out 6 seconds shorter than the local file (frame dropping?).
+  const double kExpectedMinimumPSNR = 20;
+  const double kExpectedMinimumSSIM = 0.7;
+  CompareFiles(reference_file, output_file, kExpectedMinimumPSNR,
+               kExpectedMinimumSSIM);
 }
 
 // Runs a whole stack processing with tracking of which frames are dropped
 // in the encoder. The local and remote file will not be of equal size because
 // of unknown reasons. Tests show that they start at the same frame, which is
 // the important thing when doing frame-to-frame comparison with PSNR/SSIM.
-TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors)  {
+TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) {
   // Use our own FrameDropMonitoringRemoteFileRenderer instead of the
   // ViEToFileRenderer from the test fixture:
   // TODO(kjellander): Find a better way to reuse this code without duplication.
@@ -196,8 +204,13 @@ TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors)  {
       << "The output file size is incorrect. It should be equal to the number"
       "of frames multiplied by the frame size. This will likely affect "
       "PSNR/SSIM calculations in a bad way.";
-  CompareFiles(reference_file, output_file, kMinPSNR50kbpsQCIF,
+
-               kMinSSIM50kbpsQCIF);
+  // We are running on a lower bitrate here so we need to settle for somewhat
+  // lower PSNR and SSIM values.
+  const double kExpectedMinimumPSNR = 25;
+  const double kExpectedMinimumSSIM = 0.8;
+  CompareFiles(reference_file, output_file, kExpectedMinimumPSNR,
+               kExpectedMinimumSSIM);
 }
 
 }  // namespace