From c216b9aeaf33347d068ac5d8bb2e57d62753e1f0 Mon Sep 17 00:00:00 2001 From: "stefan@webrtc.org" Date: Tue, 14 Oct 2014 10:38:49 +0000 Subject: [PATCH] Add a packet loss full stack test to the new API. Remove all full stack tests for the old API. BUG=3750 R=pbos@webrtc.org Review URL: https://webrtc-codereview.appspot.com/23029004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@7442 4adac7df-926f-26a2-2b94-8c16560cd09d --- webrtc/video/full_stack.cc | 16 ++ .../automated/vie_video_verification_test.cc | 194 ------------------ 2 files changed, 16 insertions(+), 194 deletions(-) diff --git a/webrtc/video/full_stack.cc b/webrtc/video/full_stack.cc index f2d71a97d9..0c3ea789c4 100644 --- a/webrtc/video/full_stack.cc +++ b/webrtc/video/full_stack.cc @@ -392,6 +392,7 @@ void FullStackTest::RunTest(const FullStackTestParams& params) { send_config_.encoder_settings.encoder = encoder.get(); send_config_.encoder_settings.payload_name = "VP8"; send_config_.encoder_settings.payload_type = 124; + send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; VideoStream* stream = &encoder_config_.streams[0]; stream->width = params.clip.width; @@ -403,6 +404,7 @@ void FullStackTest::RunTest(const FullStackTestParams& params) { CreateMatchingReceiveConfigs(); receive_configs_[0].renderer = &analyzer; + receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs; CreateStreams(); analyzer.input_ = send_stream_->Input(); @@ -457,6 +459,20 @@ TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) { RunTest(foreman_cif); } +TEST_F(FullStackTest, ForemanCifPlr5) { + FullStackTestParams foreman_cif = {"foreman_cif_delay_50_0_plr_5", + {"foreman_cif", 352, 288, 30}, + 30000, + 500000, + 2000000, + 0.0, + 0.0 + }; + foreman_cif.link.loss_percent = 5; + foreman_cif.link.queue_delay_ms = 50; + RunTest(foreman_cif); +} + TEST_F(FullStackTest, ForemanCif500kbps) { FullStackTestParams foreman_cif = {"foreman_cif_500kbps", {"foreman_cif", 352, 288, 30}, diff --git a/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc b/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc index 66628e9ede..1653959470 100644 --- a/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc +++ b/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc @@ -115,105 +115,6 @@ class ViEVideoVerificationTest : public testing::Test { } }; -class ParameterizedFullStackTest : public ViEVideoVerificationTest, - public ::testing::WithParamInterface { - public: - static const int kNumFullStackInstances = 4; - - protected: - struct TestParameters { - NetworkParameters network; - std::string file_name; - int width; - int height; - int bitrate; - double avg_psnr_threshold; - double avg_ssim_threshold; - ProtectionMethod protection_method; - std::string test_label; - }; - - void SetUp() { - for (int i = 0; i < kNumFullStackInstances; ++i) { - parameter_table_[i].file_name = webrtc::test::ResourcePath("foreman_cif", - "yuv"); - parameter_table_[i].width = 352; - parameter_table_[i].height = 288; - } - int i = 0; - parameter_table_[i].protection_method = kNack; - // Uniform loss => Setting burst length to -1. - parameter_table_[i].network.loss_model = kUniformLoss; - parameter_table_[i].network.packet_loss_rate = 0; - parameter_table_[i].network.burst_length = -1; - parameter_table_[i].network.mean_one_way_delay = 0; - parameter_table_[i].network.std_dev_one_way_delay = 0; - parameter_table_[i].bitrate = 300; - // TODO(holmer): Enable for Win and Mac when the file rendering has been - // moved to a separate thread. -#ifdef WEBRTC_LINUX - parameter_table_[i].avg_psnr_threshold = 35; - parameter_table_[i].avg_ssim_threshold = 0.96; -#else - parameter_table_[i].avg_psnr_threshold = 0; - parameter_table_[i].avg_ssim_threshold = 0.0; -#endif - parameter_table_[i].test_label = "net_delay_0_0_plr_0"; - ++i; - parameter_table_[i].protection_method = kNack; - parameter_table_[i].network.loss_model = kUniformLoss; - parameter_table_[i].network.packet_loss_rate = 5; - parameter_table_[i].network.burst_length = -1; - parameter_table_[i].network.mean_one_way_delay = 50; - parameter_table_[i].network.std_dev_one_way_delay = 5; - parameter_table_[i].bitrate = 300; - // TODO(holmer): Enable for Win and Mac when the file rendering has been - // moved to a separate thread. -#ifdef WEBRTC_LINUX - parameter_table_[i].avg_psnr_threshold = 35; - parameter_table_[i].avg_ssim_threshold = 0.96; -#else - parameter_table_[i].avg_psnr_threshold = 0; - parameter_table_[i].avg_ssim_threshold = 0.0; -#endif - parameter_table_[i].test_label = "net_delay_50_5_plr_5"; - ++i; - parameter_table_[i].protection_method = kNack; - parameter_table_[i].network.loss_model = kUniformLoss; - parameter_table_[i].network.packet_loss_rate = 0; - parameter_table_[i].network.burst_length = -1; - parameter_table_[i].network.mean_one_way_delay = 100; - parameter_table_[i].network.std_dev_one_way_delay = 10; - parameter_table_[i].bitrate = 300; - // TODO(holmer): Enable for Win and Mac when the file rendering has been - // moved to a separate thread. -#ifdef WEBRTC_LINUX - parameter_table_[i].avg_psnr_threshold = 35; - parameter_table_[i].avg_ssim_threshold = 0.96; -#else - parameter_table_[i].avg_psnr_threshold = 0; - parameter_table_[i].avg_ssim_threshold = 0.0; -#endif - parameter_table_[i].test_label = "net_delay_100_10_plr_0"; - ++i; - parameter_table_[i].protection_method = kNack; - parameter_table_[i].network.loss_model = kGilbertElliotLoss; - parameter_table_[i].network.packet_loss_rate = 5; - parameter_table_[i].network.burst_length = 3; - parameter_table_[i].network.mean_one_way_delay = 100; - parameter_table_[i].network.std_dev_one_way_delay = 10; - parameter_table_[i].bitrate = 300; - // Thresholds disabled for now. This is being run mainly to get a graph. - parameter_table_[i].avg_psnr_threshold = 0; - parameter_table_[i].avg_ssim_threshold = 0.0; - parameter_table_[i].test_label = "net_delay_100_10_plr_5_gilbert_elliot"; - - ASSERT_EQ(kNumFullStackInstances - 1, i); - } - - TestParameters parameter_table_[kNumFullStackInstances]; -}; - TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) { // I420 is lossless, so the I420 test should obviously get perfect results - // the local preview and remote output files should be bit-exact. This test @@ -251,99 +152,4 @@ TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) { kNumAttempts << " attempts."; } -// Runs a whole stack processing with tracking of which frames are dropped -// in the encoder. Tests show that they start at the same frame, which is -// the important thing when doing frame-to-frame comparison with PSNR/SSIM. -TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors) { - // Using CIF here since it's a more common resolution than QCIF, and higher - // resolutions shouldn't be a problem for a test using VP8. - input_file_ = parameter_table_[GetParam()].file_name; - FrameDropDetector detector; - local_file_renderer_ = new ViEToFileRenderer(); - remote_file_renderer_ = new FrameDropMonitoringRemoteFileRenderer(&detector); - SetUpLocalFileRenderer(local_file_renderer_); - SetUpRemoteFileRenderer(remote_file_renderer_); - - // Set a low bit rate so the encoder budget will be tight, causing it to drop - // frames every now and then. - const int kBitRateKbps = parameter_table_[GetParam()].bitrate; - const NetworkParameters network = parameter_table_[GetParam()].network; - int width = parameter_table_[GetParam()].width; - int height = parameter_table_[GetParam()].height; - ProtectionMethod protection_method = - parameter_table_[GetParam()].protection_method; - ViETest::Log("Bit rate : %5d kbps", kBitRateKbps); - ViETest::Log("Packet loss : %5d %%", network.packet_loss_rate); - ViETest::Log("Network delay: mean=%dms std dev=%d ms", - network.mean_one_way_delay, network.std_dev_one_way_delay); - tests_.TestFullStack(input_file_, width, height, kBitRateKbps, - protection_method, network, local_file_renderer_, - remote_file_renderer_, &detector); - const std::string reference_file = local_file_renderer_->GetFullOutputPath(); - const std::string output_file = remote_file_renderer_->GetFullOutputPath(); - StopRenderers(); - - detector.CalculateResults(); - detector.PrintReport(parameter_table_[GetParam()].test_label); - - if (detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered) > - detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) { - detector.PrintDebugDump(); - } - - ASSERT_GE(detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered), - detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) - << "The number of dropped frames on the decode and render steps are not " - "equal. This may be because we have a major problem in the buffers of " - "the ViEToFileRenderer?"; - - // We may have dropped frames during the processing, which means the output - // file does not contain all the frames that are present in the input file. - // To make the quality measurement correct, we must adjust the output file to - // that by copying the last successful frame into the place where the dropped - // frame would be, for all dropped frames. - const int frame_length_in_bytes = 3 * width * height / 2; - ViETest::Log("Frame length: %d bytes", frame_length_in_bytes); - std::vector all_frames = detector.GetAllFrames(); - FixOutputFileForComparison(output_file, frame_length_in_bytes, all_frames); - - // Verify all sent frames are present in the output file. - size_t output_file_size = webrtc::test::GetFileSize(output_file); - EXPECT_EQ(all_frames.size(), output_file_size / frame_length_in_bytes) - << "The output file size is incorrect. It should be equal to the number " - "of frames multiplied by the frame size. This will likely affect " - "PSNR/SSIM calculations in a bad way."; - - TearDownFileRenderers(); - - // We are running on a lower bitrate here so we need to settle for somewhat - // lower PSNR and SSIM values. - double actual_psnr = 0; - double actual_ssim = 0; - CompareFiles(reference_file, output_file, &actual_psnr, &actual_ssim); - - const double kExpectedMinimumPSNR = - parameter_table_[GetParam()].avg_psnr_threshold; - const double kExpectedMinimumSSIM = - parameter_table_[GetParam()].avg_ssim_threshold; - - EXPECT_GE(actual_psnr, kExpectedMinimumPSNR); - EXPECT_GE(actual_ssim, kExpectedMinimumSSIM); - - std::stringstream ss; - ss << std::setprecision(3) << std::fixed << actual_psnr; - webrtc::test::PrintResult( - "psnr", "", parameter_table_[GetParam()].test_label, - ss.str(), "dB", false); - - ss.str(""); - ss << std::setprecision(3) << std::fixed << actual_ssim; - webrtc::test::PrintResult( - "ssim", "", parameter_table_[GetParam()].test_label, - ss.str(), "", false); -} - -INSTANTIATE_TEST_CASE_P(FullStackTests, ParameterizedFullStackTest, - ::testing::Range(0, ParameterizedFullStackTest::kNumFullStackInstances)); - } // namespace