From d056abd62f50c9cd8c996fe0298c0f0b40b78317 Mon Sep 17 00:00:00 2001 From: "phoglund@webrtc.org" Date: Fri, 20 Jan 2012 12:48:13 +0000 Subject: [PATCH] Prepared tests for running on build bot. The changes will mostly clarify why PSNR and SSIM thresholds are chosen. I think those explanations will be good since these tests may well be a bit flaky. BUG= TEST= Review URL: https://webrtc-codereview.appspot.com/343017 git-svn-id: http://webrtc.googlecode.com/svn/trunk@1496 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../automated/vie_video_verification_test.cc | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc b/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc index 885c9640f1..69e9d3aa7e 100644 --- a/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc +++ b/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc @@ -26,13 +26,6 @@ namespace { const int kInputWidth = 176; const int kInputHeight = 144; -const double kMinPSNR420DefaultBitRateQCIF = 28; -const double kMinSSIM420DefaultBitRateQCIF = 0.95; -const double kMinPSNRCodecTestsQCIF = 20; -const double kMinSSIMCodecTestsQCIF = 0.7; -const double kMinPSNR50kbpsQCIF = 25; -const double kMinSSIM50kbpsQCIF = 0.8; - class ViEVideoVerificationTest : public testing::Test { protected: void SetUp() { @@ -79,19 +72,25 @@ class ViEVideoVerificationTest : public testing::Test { void CompareFiles(const std::string& reference_file, const std::string& test_file, double minimum_psnr, double minimum_ssim) { + static const char* kPsnrSsimExplanation = + "Don't worry too much about this error if it only happens once. " + "It may be because mundane things like unfortunate OS scheduling. " + "If it keeps happening over and over though it's a cause of concern."; + webrtc::test::QualityMetricsResult psnr; int error = I420PSNRFromFiles(reference_file.c_str(), test_file.c_str(), kInputWidth, kInputHeight, &psnr); + EXPECT_EQ(0, error) << "PSNR routine failed - output files missing?"; - EXPECT_GT(psnr.average, minimum_psnr); + EXPECT_GT(psnr.average, minimum_psnr) << kPsnrSsimExplanation; webrtc::test::QualityMetricsResult ssim; error = I420SSIMFromFiles(reference_file.c_str(), test_file.c_str(), kInputWidth, kInputHeight, &ssim); EXPECT_EQ(0, error) << "SSIM routine failed - output files missing?"; - EXPECT_GT(ssim.average, minimum_ssim); // 1 = perfect, -1 = terrible + EXPECT_GT(ssim.average, minimum_ssim) << kPsnrSsimExplanation; - ViETest::Log("Results: PSNR: %f (db) SSIM: %f", + ViETest::Log("Results: PSNR is %f (dB), SSIM is %f (1 is perfect)", psnr.average, ssim.average); } @@ -116,14 +115,20 @@ class ViEVideoVerificationTest : public testing::Test { } }; -TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) { +TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) { ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight, local_file_renderer_, remote_file_renderer_)); std::string output_file = remote_file_renderer_->GetFullOutputPath(); StopRenderers(); - CompareFiles(input_file_, output_file, kMinPSNR420DefaultBitRateQCIF, - kMinSSIM420DefaultBitRateQCIF); + + // The I420 test should give pretty good values since it's a lossless codec + // running on the default bitrate. It should average about 30 dB but there + // may be cases where it dips as low as 26 under adverse conditions. + const double kExpectedMinimumPSNR = 28; + const double kExpectedMinimumSSIM = 0.95; + CompareFiles(input_file_, output_file, kExpectedMinimumPSNR, + kExpectedMinimumSSIM); } TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) { @@ -138,18 +143,21 @@ TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) { // The reason is that it is hard to say when the three consecutive tests // switch over into each other, at which point we would have to restart the // original to get a fair comparison. - CompareFiles(reference_file, output_file, kMinPSNRCodecTestsQCIF, - kMinSSIMCodecTestsQCIF); - - // TODO(phoglund): The values should be higher. Investigate why the remote - // file turns out 6 seconds shorter than the local file (frame dropping?). + // + // The PSNR and SSIM values are quite low here, and they have to be since + // the codec switches will lead to lag in the output. This is considered + // acceptable, but it probably shouldn't get worse than this. + const double kExpectedMinimumPSNR = 20; + const double kExpectedMinimumSSIM = 0.7; + CompareFiles(reference_file, output_file, kExpectedMinimumPSNR, + kExpectedMinimumSSIM); } // Runs a whole stack processing with tracking of which frames are dropped // in the encoder. The local and remote file will not be of equal size because // of unknown reasons. Tests show that they start at the same frame, which is // the important thing when doing frame-to-frame comparison with PSNR/SSIM. -TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) { +TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) { // Use our own FrameDropMonitoringRemoteFileRenderer instead of the // ViEToFileRenderer from the test fixture: // TODO(kjellander): Find a better way to reuse this code without duplication. @@ -196,8 +204,13 @@ TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) { << "The output file size is incorrect. It should be equal to the number" "of frames multiplied by the frame size. This will likely affect " "PSNR/SSIM calculations in a bad way."; - CompareFiles(reference_file, output_file, kMinPSNR50kbpsQCIF, - kMinSSIM50kbpsQCIF); + + // We are running on a lower bitrate here so we need to settle for somewhat + // lower PSNR and SSIM values. + const double kExpectedMinimumPSNR = 25; + const double kExpectedMinimumSSIM = 0.8; + CompareFiles(reference_file, output_file, kExpectedMinimumPSNR, + kExpectedMinimumSSIM); } } // namespace