zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a packet loss full stack test to the new API.

Browse Source 
Remove all full stack tests for the old API.

BUG=3750
R=pbos@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/23029004

git-svn-id: http://webrtc.googlecode.com/svn/trunk@7442 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
stefan@webrtc.org
2014-10-14 10:38:49 +00:00
parent a57678a70e
commit c216b9aeaf
2 changed files with 16 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
webrtc/video/full_stack.cc
View File

@ -392,6 +392,7 @@ void FullStackTest::RunTest(const FullStackTestParams& params) {
send_config_.encoder_settings.encoder = encoder.get(); send_config_.encoder_settings.encoder = encoder.get();
send_config_.encoder_settings.payload_name = "VP8"; send_config_.encoder_settings.payload_name = "VP8";
send_config_.encoder_settings.payload_type = 124; send_config_.encoder_settings.payload_type = 124;
send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
VideoStream* stream = &encoder_config_.streams[0]; VideoStream* stream = &encoder_config_.streams[0];
stream->width = params.clip.width; stream->width = params.clip.width;
@ -403,6 +404,7 @@ void FullStackTest::RunTest(const FullStackTestParams& params) {
CreateMatchingReceiveConfigs(); CreateMatchingReceiveConfigs();
receive_configs_[0].renderer = &analyzer; receive_configs_[0].renderer = &analyzer;
receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
CreateStreams(); CreateStreams();
analyzer.input_ = send_stream_->Input(); analyzer.input_ = send_stream_->Input();
@ -457,6 +459,20 @@ TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) {
RunTest(foreman_cif); RunTest(foreman_cif);
} }
TEST_F(FullStackTest, ForemanCifPlr5) {
FullStackTestParams foreman_cif = {"foreman_cif_delay_50_0_plr_5",
{"foreman_cif", 352, 288, 30},
30000,
500000,
2000000,
0.0,
0.0
};
foreman_cif.link.loss_percent = 5;
foreman_cif.link.queue_delay_ms = 50;
RunTest(foreman_cif);
}
TEST_F(FullStackTest, ForemanCif500kbps) { TEST_F(FullStackTest, ForemanCif500kbps) {
FullStackTestParams foreman_cif = {"foreman_cif_500kbps", FullStackTestParams foreman_cif = {"foreman_cif_500kbps",
{"foreman_cif", 352, 288, 30}, {"foreman_cif", 352, 288, 30},

194
webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc
View File

@ -115,105 +115,6 @@ class ViEVideoVerificationTest : public testing::Test {
} }
}; };
class ParameterizedFullStackTest : public ViEVideoVerificationTest,
public ::testing::WithParamInterface<int> {
public:
static const int kNumFullStackInstances = 4;
protected:
struct TestParameters {
NetworkParameters network;
std::string file_name;
int width;
int height;
int bitrate;
double avg_psnr_threshold;
double avg_ssim_threshold;
ProtectionMethod protection_method;
std::string test_label;
};
void SetUp() {
for (int i = 0; i < kNumFullStackInstances; ++i) {
parameter_table_[i].file_name = webrtc::test::ResourcePath("foreman_cif",
"yuv");
parameter_table_[i].width = 352;
parameter_table_[i].height = 288;
}
int i = 0;
parameter_table_[i].protection_method = kNack;
// Uniform loss => Setting burst length to -1.
parameter_table_[i].network.loss_model = kUniformLoss;
parameter_table_[i].network.packet_loss_rate = 0;
parameter_table_[i].network.burst_length = -1;
parameter_table_[i].network.mean_one_way_delay = 0;
parameter_table_[i].network.std_dev_one_way_delay = 0;
parameter_table_[i].bitrate = 300;
// TODO(holmer): Enable for Win and Mac when the file rendering has been
// moved to a separate thread.
#ifdef WEBRTC_LINUX
parameter_table_[i].avg_psnr_threshold = 35;
parameter_table_[i].avg_ssim_threshold = 0.96;
#else
parameter_table_[i].avg_psnr_threshold = 0;
parameter_table_[i].avg_ssim_threshold = 0.0;
#endif
parameter_table_[i].test_label = "net_delay_0_0_plr_0";
++i;
parameter_table_[i].protection_method = kNack;
parameter_table_[i].network.loss_model = kUniformLoss;
parameter_table_[i].network.packet_loss_rate = 5;
parameter_table_[i].network.burst_length = -1;
parameter_table_[i].network.mean_one_way_delay = 50;
parameter_table_[i].network.std_dev_one_way_delay = 5;
parameter_table_[i].bitrate = 300;
// TODO(holmer): Enable for Win and Mac when the file rendering has been
// moved to a separate thread.
#ifdef WEBRTC_LINUX
parameter_table_[i].avg_psnr_threshold = 35;
parameter_table_[i].avg_ssim_threshold = 0.96;
#else
parameter_table_[i].avg_psnr_threshold = 0;
parameter_table_[i].avg_ssim_threshold = 0.0;
#endif
parameter_table_[i].test_label = "net_delay_50_5_plr_5";
++i;
parameter_table_[i].protection_method = kNack;
parameter_table_[i].network.loss_model = kUniformLoss;
parameter_table_[i].network.packet_loss_rate = 0;
parameter_table_[i].network.burst_length = -1;
parameter_table_[i].network.mean_one_way_delay = 100;
parameter_table_[i].network.std_dev_one_way_delay = 10;
parameter_table_[i].bitrate = 300;
// TODO(holmer): Enable for Win and Mac when the file rendering has been
// moved to a separate thread.
#ifdef WEBRTC_LINUX
parameter_table_[i].avg_psnr_threshold = 35;
parameter_table_[i].avg_ssim_threshold = 0.96;
#else
parameter_table_[i].avg_psnr_threshold = 0;
parameter_table_[i].avg_ssim_threshold = 0.0;
#endif
parameter_table_[i].test_label = "net_delay_100_10_plr_0";
++i;
parameter_table_[i].protection_method = kNack;
parameter_table_[i].network.loss_model = kGilbertElliotLoss;
parameter_table_[i].network.packet_loss_rate = 5;
parameter_table_[i].network.burst_length = 3;
parameter_table_[i].network.mean_one_way_delay = 100;
parameter_table_[i].network.std_dev_one_way_delay = 10;
parameter_table_[i].bitrate = 300;
// Thresholds disabled for now. This is being run mainly to get a graph.
parameter_table_[i].avg_psnr_threshold = 0;
parameter_table_[i].avg_ssim_threshold = 0.0;
parameter_table_[i].test_label = "net_delay_100_10_plr_5_gilbert_elliot";
ASSERT_EQ(kNumFullStackInstances - 1, i);
}
TestParameters parameter_table_[kNumFullStackInstances];
};
TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) { TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
// I420 is lossless, so the I420 test should obviously get perfect results - // I420 is lossless, so the I420 test should obviously get perfect results -
// the local preview and remote output files should be bit-exact. This test // the local preview and remote output files should be bit-exact. This test
@ -251,99 +152,4 @@ TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
kNumAttempts << " attempts."; kNumAttempts << " attempts.";
} }
// Runs a whole stack processing with tracking of which frames are dropped
// in the encoder. Tests show that they start at the same frame, which is
// the important thing when doing frame-to-frame comparison with PSNR/SSIM.
TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors) {
// Using CIF here since it's a more common resolution than QCIF, and higher
// resolutions shouldn't be a problem for a test using VP8.
input_file_ = parameter_table_[GetParam()].file_name;
FrameDropDetector detector;
local_file_renderer_ = new ViEToFileRenderer();
remote_file_renderer_ = new FrameDropMonitoringRemoteFileRenderer(&detector);
SetUpLocalFileRenderer(local_file_renderer_);
SetUpRemoteFileRenderer(remote_file_renderer_);
// Set a low bit rate so the encoder budget will be tight, causing it to drop
// frames every now and then.
const int kBitRateKbps = parameter_table_[GetParam()].bitrate;
const NetworkParameters network = parameter_table_[GetParam()].network;
int width = parameter_table_[GetParam()].width;
int height = parameter_table_[GetParam()].height;
ProtectionMethod protection_method =
parameter_table_[GetParam()].protection_method;
ViETest::Log("Bit rate : %5d kbps", kBitRateKbps);
ViETest::Log("Packet loss : %5d %%", network.packet_loss_rate);
ViETest::Log("Network delay: mean=%dms std dev=%d ms",
network.mean_one_way_delay, network.std_dev_one_way_delay);
tests_.TestFullStack(input_file_, width, height, kBitRateKbps,
protection_method, network, local_file_renderer_,
remote_file_renderer_, &detector);
const std::string reference_file = local_file_renderer_->GetFullOutputPath();
const std::string output_file = remote_file_renderer_->GetFullOutputPath();
StopRenderers();
detector.CalculateResults();
detector.PrintReport(parameter_table_[GetParam()].test_label);
if (detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered) >
detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) {
detector.PrintDebugDump();
}
ASSERT_GE(detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered),
detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded))
<< "The number of dropped frames on the decode and render steps are not "
"equal. This may be because we have a major problem in the buffers of "
"the ViEToFileRenderer?";
// We may have dropped frames during the processing, which means the output
// file does not contain all the frames that are present in the input file.
// To make the quality measurement correct, we must adjust the output file to
// that by copying the last successful frame into the place where the dropped
// frame would be, for all dropped frames.
const int frame_length_in_bytes = 3 * width * height / 2;
ViETest::Log("Frame length: %d bytes", frame_length_in_bytes);
std::vector<Frame*> all_frames = detector.GetAllFrames();
FixOutputFileForComparison(output_file, frame_length_in_bytes, all_frames);
// Verify all sent frames are present in the output file.
size_t output_file_size = webrtc::test::GetFileSize(output_file);
EXPECT_EQ(all_frames.size(), output_file_size / frame_length_in_bytes)
<< "The output file size is incorrect. It should be equal to the number "
"of frames multiplied by the frame size. This will likely affect "
"PSNR/SSIM calculations in a bad way.";
TearDownFileRenderers();
// We are running on a lower bitrate here so we need to settle for somewhat
// lower PSNR and SSIM values.
double actual_psnr = 0;
double actual_ssim = 0;
CompareFiles(reference_file, output_file, &actual_psnr, &actual_ssim);
const double kExpectedMinimumPSNR =
parameter_table_[GetParam()].avg_psnr_threshold;
const double kExpectedMinimumSSIM =
parameter_table_[GetParam()].avg_ssim_threshold;
EXPECT_GE(actual_psnr, kExpectedMinimumPSNR);
EXPECT_GE(actual_ssim, kExpectedMinimumSSIM);
std::stringstream ss;
ss << std::setprecision(3) << std::fixed << actual_psnr;
webrtc::test::PrintResult(
"psnr", "", parameter_table_[GetParam()].test_label,
ss.str(), "dB", false);
ss.str("");
ss << std::setprecision(3) << std::fixed << actual_ssim;
webrtc::test::PrintResult(
"ssim", "", parameter_table_[GetParam()].test_label,
ss.str(), "", false);
}
INSTANTIATE_TEST_CASE_P(FullStackTests, ParameterizedFullStackTest,
::testing::Range(0, ParameterizedFullStackTest::kNumFullStackInstances));
} // namespace } // namespace