diff --git a/test/test.gyp b/test/test.gyp index d5354f3c9d..c995bb6cdc 100644 --- a/test/test.gyp +++ b/test/test.gyp @@ -34,6 +34,7 @@ 'test_suite.h', 'testsupport/fileutils.h', 'testsupport/fileutils.cc', + 'testsupport/metrics/video_metrics.h', 'testsupport/metrics/video_metrics.cc', ], }, @@ -58,6 +59,7 @@ ], 'sources': [ 'testsupport/fileutils_unittest.cc', + 'testsupport/metrics/video_metrics_unittest.cc', ], }, ], diff --git a/test/testsupport/metrics/video_metrics.cc b/test/testsupport/metrics/video_metrics.cc index 568c43ae3a..2aa72d4077 100644 --- a/test/testsupport/metrics/video_metrics.cc +++ b/test/testsupport/metrics/video_metrics.cc @@ -9,10 +9,10 @@ */ #include "video_metrics.h" + #include // min_element, max_element #include #include -#include "system_wrappers/interface/cpu_features_wrapper.h" // Calculates PSNR from MSE static inline double CalcPsnr(double mse) { @@ -21,11 +21,12 @@ static inline double CalcPsnr(double mse) { } // Used for calculating min and max values -static bool lessForFrameResultValue (const FrameResult& s1, const FrameResult& s2) { +static bool LessForFrameResultValue (const FrameResult& s1, + const FrameResult& s2) { return s1.value < s2.value; } -WebRtc_Word32 +int PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result) { @@ -86,7 +87,7 @@ PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp); } - if (mse == 0) + if (mseSum == 0) { // The PSNR value is undefined in this case. // This value effectively means that the files are equal. @@ -97,17 +98,25 @@ PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, result->average = CalcPsnr(mseSum / frames); } - // Calculate min/max statistics - std::vector::iterator element; - element = min_element(result->frames.begin(), - result->frames.end(), lessForFrameResultValue); - result->min = element->value; - result->min_frame_number = element->frame_number; - element = max_element(result->frames.begin(), - result->frames.end(), lessForFrameResultValue); - result->max = element->value; - result->max_frame_number = element->frame_number; - + if (result->frames.size() == 0) + { + fprintf(stderr, "Tried to measure SSIM from empty files (reference " + "file: %s test file: %s\n", refFileName, testFileName); + return -3; + } + else + { + // Calculate min/max statistics + std::vector::iterator element; + element = min_element(result->frames.begin(), + result->frames.end(), LessForFrameResultValue); + result->min = element->value; + result->min_frame_number = element->frame_number; + element = max_element(result->frames.begin(), + result->frames.end(), LessForFrameResultValue); + result->max = element->value; + result->max_frame_number = element->frame_number; + } delete [] ref; delete [] test; @@ -140,6 +149,7 @@ Similarity(WebRtc_UWord64 sum_s, WebRtc_UWord64 sum_r, WebRtc_UWord64 sum_sq_s, return ssim_n * 1.0 / ssim_d; } +#if !defined(WEBRTC_USE_SSE2) static double Ssim8x8C(WebRtc_UWord8 *s, WebRtc_Word32 sp, WebRtc_UWord8 *r, WebRtc_Word32 rp) @@ -164,6 +174,7 @@ Ssim8x8C(WebRtc_UWord8 *s, WebRtc_Word32 sp, } return Similarity(sum_s, sum_r, sum_sq_s, sum_sq_r, sum_sxr, 64); } +#endif #if defined(WEBRTC_USE_SSE2) #include @@ -256,13 +267,11 @@ SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2, WebRtc_Word32 stride_img1, double (*ssim_8x8)(WebRtc_UWord8*, WebRtc_Word32, WebRtc_UWord8*, WebRtc_Word32 rp); - ssim_8x8 = Ssim8x8C; - if (WebRtc_GetCPUInfo(kSSE2)) - { #if defined(WEBRTC_USE_SSE2) - ssim_8x8 = Ssim8x8Sse2; + ssim_8x8 = Ssim8x8Sse2; +#else + ssim_8x8 = Ssim8x8C; #endif - } // Sample point start with each 4x4 location for (i = 0; i < height - 8; i += 4, img1 += stride_img1 * 4, @@ -279,7 +288,7 @@ SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2, WebRtc_Word32 stride_img1, return ssim_total; } -WebRtc_Word32 +int SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result) { @@ -327,21 +336,29 @@ SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp); } - // SSIM: normalize/average for sequence - ssimScene = ssimScene / frames; - result->average = ssimScene; - - // Calculate min/max statistics - std::vector::iterator element; - element = min_element(result->frames.begin(), - result->frames.end(), lessForFrameResultValue); - result->min = element->value; - result->min_frame_number = element->frame_number; - element = max_element(result->frames.begin(), - result->frames.end(), lessForFrameResultValue); - result->max = element->value; - result->max_frame_number = element->frame_number; + if (result->frames.size() == 0) + { + fprintf(stderr, "Tried to measure SSIM from empty files (reference " + "file: %s test file: %s\n", refFileName, testFileName); + return -3; + } + else + { + // SSIM: normalize/average for sequence + ssimScene = ssimScene / frames; + result->average = ssimScene; + // Calculate min/max statistics + std::vector::iterator element; + element = min_element(result->frames.begin(), + result->frames.end(), LessForFrameResultValue); + result->min = element->value; + result->min_frame_number = element->frame_number; + element = max_element(result->frames.begin(), + result->frames.end(), LessForFrameResultValue); + result->max = element->value; + result->max_frame_number = element->frame_number; + } delete [] ref; delete [] test; diff --git a/test/testsupport/metrics/video_metrics.h b/test/testsupport/metrics/video_metrics.h index d25a1cfc57..43663d6e87 100644 --- a/test/testsupport/metrics/video_metrics.h +++ b/test/testsupport/metrics/video_metrics.h @@ -8,14 +8,13 @@ * be found in the AUTHORS file in the root of the source tree. */ - #ifndef WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_ #define WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_ -#include "typedefs.h" #include #include +#include "typedefs.h" // Contains video quality metrics result for a single frame. struct FrameResult { @@ -47,23 +46,35 @@ struct QualityMetricsResult { // If the result is std::numerical_limits::max() the videos were // equal. Otherwise, PSNR values are in decibel (higher is better). This // algorithm only compares up to the point when the shortest video ends. -WebRtc_Word32 -PsnrFromFiles(const WebRtc_Word8 *refFileName, - const WebRtc_Word8 *testFileName, WebRtc_Word32 width, - WebRtc_Word32 height, QualityMetricsResult *result); - +// By definition of PSNR, the result value is undefined if the reference file +// and the test file are identical. In that case the max value for double +// will be set in the result struct. +// +// Returns 0 if successful, negative on errors: +// -1 if the source file cannot be opened +// -2 if the test file cannot be opened +// -3 if any of the files are empty +int PsnrFromFiles(const WebRtc_Word8 *refFileName, + const WebRtc_Word8 *testFileName, WebRtc_Word32 width, + WebRtc_Word32 height, QualityMetricsResult *result); // SSIM values are filled into the QualityMetricsResult struct. // Values range between -1 and 1, where 1 means the files were identical. This // algorithm only compares up to the point when the shortest video ends. -WebRtc_Word32 -SsimFromFiles(const WebRtc_Word8 *refFileName, - const WebRtc_Word8 *testFileName, WebRtc_Word32 width, - WebRtc_Word32 height, QualityMetricsResult *result); - -double -SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2, WebRtc_Word32 stride_img1, - WebRtc_Word32 stride_img2, WebRtc_Word32 width, WebRtc_Word32 height); +// By definition, SSIM values varies from -1.0, when everything is different +// between the reference file and the test file, up to 1.0 for two identical +// files. +// +// Returns 0 if successful, negative on errors: +// -1 if the source file cannot be opened +// -2 if the test file cannot be opened +// -3 if any of the files are empty +int SsimFromFiles(const WebRtc_Word8 *refFileName, + const WebRtc_Word8 *testFileName, WebRtc_Word32 width, + WebRtc_Word32 height, QualityMetricsResult *result); +double SsimFrame(WebRtc_UWord8 *img1, WebRtc_UWord8 *img2, + WebRtc_Word32 stride_img1, WebRtc_Word32 stride_img2, + WebRtc_Word32 width, WebRtc_Word32 height); #endif // WEBRTC_MODULES_VIDEO_CODING_TEST_VIDEO_METRICS_H_ diff --git a/test/testsupport/metrics/video_metrics_unittest.cc b/test/testsupport/metrics/video_metrics_unittest.cc new file mode 100644 index 0000000000..ae9188f8ad --- /dev/null +++ b/test/testsupport/metrics/video_metrics_unittest.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video_metrics.h" + +#include + +#include "gtest/gtest.h" +#include "testsupport/fileutils.h" + +namespace webrtc { + +static const char* kEmptyFileName = "video_metrics_unittest_empty_file.tmp"; +static const char* kNonExistingFileName = "video_metrics_unittest_non_existing"; +static const int kWidth = 352; +static const int kHeight = 288; + +static const int kMissingReferenceFileReturnCode = -1; +static const int kMissingTestFileReturnCode = -2; +static const int kEmptyFileReturnCode = -3; +static const double kPsnrPerfectResult = std::numeric_limits::max(); +static const double kSsimPerfectResult = 1.0; + +class VideoMetricsTest: public testing::Test { + protected: + VideoMetricsTest() { + video_file = webrtc::test::ProjectRootPath() + "resources/foreman_cif.yuv"; + } + virtual ~VideoMetricsTest() {} + void SetUp() { + // Create an empty file: + FILE* dummy = fopen(kEmptyFileName, "wb"); + fclose(dummy); + } + void TearDown() { + std::remove(kEmptyFileName); + } + QualityMetricsResult result_; + std::string video_file; +}; + +// Tests that it is possible to run with the same reference as test file +TEST_F(VideoMetricsTest, ReturnsPerfectResultForIdenticalFiles) { + EXPECT_EQ(0, PsnrFromFiles(video_file.c_str(), video_file.c_str(), + kWidth, kHeight, &result_)); + EXPECT_EQ(kPsnrPerfectResult, result_.average); + EXPECT_EQ(SsimFromFiles(video_file.c_str(), video_file.c_str(), kWidth, kHeight, + &result_), 0); + EXPECT_EQ(kSsimPerfectResult, result_.average); +} + +// Tests that the right return code is given when the reference file is missing. +TEST_F(VideoMetricsTest, MissingReferenceFile) { + EXPECT_EQ(kMissingReferenceFileReturnCode, + PsnrFromFiles(kNonExistingFileName, video_file.c_str(), kWidth, + kHeight, &result_)); + EXPECT_EQ(kMissingReferenceFileReturnCode, + SsimFromFiles(kNonExistingFileName, video_file.c_str(), kWidth, + kHeight, &result_)); +} + +// Tests that the right return code is given when the test file is missing. +TEST_F(VideoMetricsTest, MissingTestFile) { + EXPECT_EQ(kMissingTestFileReturnCode, + PsnrFromFiles(video_file.c_str(), kNonExistingFileName, kWidth, + kHeight, &result_)); + EXPECT_EQ(kMissingTestFileReturnCode, + SsimFromFiles(video_file.c_str(), kNonExistingFileName, kWidth, + kHeight, &result_)); +} + +// Tests that the method can be executed with empty files. +TEST_F(VideoMetricsTest, EmptyFiles) { + EXPECT_EQ(kEmptyFileReturnCode, + PsnrFromFiles(kEmptyFileName, video_file.c_str(), kWidth, kHeight, + &result_)); + EXPECT_EQ(kEmptyFileReturnCode, + SsimFromFiles(kEmptyFileName, video_file.c_str(), kWidth, kHeight, + &result_)); + // Run the same again with the empty file switched. + EXPECT_EQ(kEmptyFileReturnCode, + PsnrFromFiles(video_file.c_str(), kEmptyFileName, kWidth, kHeight, + &result_)); + EXPECT_EQ(kEmptyFileReturnCode, + SsimFromFiles(video_file.c_str(), kEmptyFileName, kWidth, kHeight, + &result_)); +} + +} // namespace webrtc + diff --git a/webrtc.gyp b/webrtc.gyp index 04833e73d0..90ca0b736b 100644 --- a/webrtc.gyp +++ b/webrtc.gyp @@ -21,6 +21,7 @@ 'src/system_wrappers/source/system_wrappers.gyp:*', 'src/video_engine/video_engine.gyp:*', 'src/voice_engine/voice_engine.gyp:*', + 'test/test.gyp:*', ], }, # TODO(andrew): move peerconnection to its own gyp.