From f4c2de9e2fe9e5c8f83d790915d46f2d885e1e76 Mon Sep 17 00:00:00 2001 From: "marpan@webrtc.org" Date: Tue, 5 Jun 2012 21:07:28 +0000 Subject: [PATCH] Added some tests to videoprocessor_integrationtest, for testing: -encooder response to changing bit rate and frame rate -frame dropper and spatial resize -temporal layers Review URL: https://webrtc-codereview.appspot.com/613006 git-svn-id: http://webrtc.googlecode.com/svn/trunk@2370 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../codecs/test/videoprocessor.cc | 122 +++- .../video_coding/codecs/test/videoprocessor.h | 33 +- .../test/videoprocessor_integrationtest.cc | 578 +++++++++++++++++- 3 files changed, 697 insertions(+), 36 deletions(-) diff --git a/src/modules/video_coding/codecs/test/videoprocessor.cc b/src/modules/video_coding/codecs/test/videoprocessor.cc index f12ad4de9a..4773f3bb01 100644 --- a/src/modules/video_coding/codecs/test/videoprocessor.cc +++ b/src/modules/video_coding/codecs/test/videoprocessor.cc @@ -38,7 +38,14 @@ VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder, source_buffer_(NULL), first_key_frame_has_been_excluded_(false), last_frame_missing_(false), - initialized_(false) { + initialized_(false), + encoded_frame_size_(0), + prev_time_stamp_(0), + num_dropped_frames_(0), + num_spatial_resizes_(0), + last_encoder_frame_width_(0), + last_encoder_frame_height_(0), + scaler_() { assert(encoder); assert(decoder); assert(frame_reader); @@ -63,6 +70,10 @@ bool VideoProcessorImpl::Init() { source_frame_._length = frame_length_in_bytes; source_frame_._size = frame_length_in_bytes; + // To keep track of spatial resize actions by encoder. + last_encoder_frame_width_ = config_.codec_settings->width; + last_encoder_frame_height_ = config_.codec_settings->height; + // Setup required callbacks for the encoder/decoder: encode_callback_ = new VideoProcessorEncodeCompleteCallback(this); decode_callback_ = new VideoProcessorDecodeCompleteCallback(this); @@ -122,12 +133,40 @@ VideoProcessorImpl::~VideoProcessorImpl() { delete decode_callback_; } + +void VideoProcessorImpl::SetRates(int bit_rate, int frame_rate) { + int set_rates_result = encoder_->SetRates(bit_rate, frame_rate); + assert(set_rates_result >= 0); + if (set_rates_result < 0) { + fprintf(stderr, "Failed to update encoder with new rate %d, " + "return code: %d\n", bit_rate, set_rates_result); + } + num_dropped_frames_ = 0; + num_spatial_resizes_ = 0; +} + +int VideoProcessorImpl::EncodedFrameSize() { + return encoded_frame_size_; +} + +int VideoProcessorImpl::NumberDroppedFrames() { + return num_dropped_frames_; +} + +int VideoProcessorImpl::NumberSpatialResizes() { + return num_spatial_resizes_; +} + bool VideoProcessorImpl::ProcessFrame(int frame_number) { assert(frame_number >=0); if (!initialized_) { fprintf(stderr, "Attempting to use uninitialized VideoProcessor!\n"); return false; } + // |prev_time_stamp_| is used for getting number of dropped frames. + if (frame_number == 0) { + prev_time_stamp_ = -1; + } if (frame_reader_->ReadFrame(source_buffer_)) { // point the source frame buffer to the newly read frame data: source_frame_._buffer = source_buffer_; @@ -145,8 +184,13 @@ bool VideoProcessorImpl::ProcessFrame(int frame_number) { frame_number % config_.keyframe_interval == 0) { frame_type = kKeyFrame; } + + // For dropped frames, we regard them as zero size encoded frames. + encoded_frame_size_ = 0; + WebRtc_Word32 encode_result = encoder_->Encode(source_frame_, NULL, frame_type); + if (encode_result != WEBRTC_VIDEO_CODEC_OK) { fprintf(stderr, "Failed to encode frame %d, return code: %d\n", frame_number, encode_result); @@ -159,6 +203,22 @@ bool VideoProcessorImpl::ProcessFrame(int frame_number) { } void VideoProcessorImpl::FrameEncoded(EncodedImage* encoded_image) { + // Timestamp is frame number, so this gives us #dropped frames. + int num_dropped_from_prev_encode = encoded_image->_timeStamp - + prev_time_stamp_ - 1; + num_dropped_frames_ += num_dropped_from_prev_encode; + prev_time_stamp_ = encoded_image->_timeStamp; + if (num_dropped_from_prev_encode > 0) { + // For dropped frames, we write out the last decoded frame to avoid getting + // out of sync for the computation of PSNR and SSIM. + for (int i = 0; i < num_dropped_from_prev_encode; i++) { + frame_writer_->WriteFrame(last_successful_frame_buffer_); + } + } + // Frame is not dropped, so update the encoded frame size + // (encoder callback is only called for non-zero length frames). + encoded_frame_size_ = encoded_image->_length; + TickTime encode_stop = TickTime::Now(); int frame_number = encoded_image->_timeStamp; FrameStatistic& stat = stats_->stats_[frame_number]; @@ -220,12 +280,60 @@ void VideoProcessorImpl::FrameDecoded(const RawImage& image) { stat.decode_time_in_us = GetElapsedTimeMicroseconds(decode_start_, decode_stop); stat.decoding_successful = true; - // Update our copy of the last successful frame: - memcpy(last_successful_frame_buffer_, image._buffer, image._length); - bool write_success = frame_writer_->WriteFrame(image._buffer); - if (!write_success) { - fprintf(stderr, "Failed to write frame %d to disk!", frame_number); + // Check for resize action (either down or up): + if (static_cast(image._width) != last_encoder_frame_width_ || + static_cast(image._height) != last_encoder_frame_height_ ) { + ++num_spatial_resizes_; + last_encoder_frame_width_ = image._width; + last_encoder_frame_height_ = image._height; + } + // Check if codec size is different from native/original size, and if so, + // upsample back to original size: needed for PSNR and SSIM computations. + if (image._width != config_.codec_settings->width || + image._height != config_.codec_settings->height) { + int required_size = static_cast + (config_.codec_settings->width * config_.codec_settings->height * 3 / 2); + RawImage up_image = image; + up_image._buffer = new uint8_t[required_size]; + up_image._length = required_size; + up_image._width = config_.codec_settings->width; + up_image._height = config_.codec_settings->height; + + int ret_val = scaler_.Set(image._width, image._height, + config_.codec_settings->width, + config_.codec_settings->height, + kI420, kI420, kScaleBilinear); + assert(ret_val >= 0); + if (ret_val < 0) { + fprintf(stderr, "Failed to set scalar for frame: %d, return code: %d\n", + frame_number, ret_val); + } + ret_val = scaler_.Scale(image._buffer, up_image._buffer, + required_size); + assert(ret_val >= 0); + if (ret_val < 0) { + fprintf(stderr, "Failed to scale frame: %d, return code: %d\n", + frame_number, ret_val); + } + // Update our copy of the last successful frame: + memcpy(last_successful_frame_buffer_, up_image._buffer, up_image._length); + + bool write_success = frame_writer_->WriteFrame(up_image._buffer); + assert(write_success); + if (!write_success) { + fprintf(stderr, "Failed to write frame %d to disk!", frame_number); + } + delete [] up_image._buffer; + } else { // No resize. + // Update our copy of the last successful frame: + memcpy(last_successful_frame_buffer_, image._buffer, image._length); + + bool write_success = frame_writer_->WriteFrame(image._buffer); + assert(write_success); + if (!write_success) { + fprintf(stderr, "Failed to write frame %d to disk!", frame_number); + } } } @@ -273,7 +381,7 @@ VideoProcessorImpl::VideoProcessorEncodeCompleteCallback::Encoded( EncodedImage& encoded_image, const webrtc::CodecSpecificInfo* codec_specific_info, const webrtc::RTPFragmentationHeader* fragmentation) { - video_processor_->FrameEncoded(&encoded_image); // forward to parent class + video_processor_->FrameEncoded(&encoded_image); // Forward to parent class. return 0; } WebRtc_Word32 diff --git a/src/modules/video_coding/codecs/test/videoprocessor.h b/src/modules/video_coding/codecs/test/videoprocessor.h index 7c2c14f52f..3e017867a6 100644 --- a/src/modules/video_coding/codecs/test/videoprocessor.h +++ b/src/modules/video_coding/codecs/test/videoprocessor.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -13,6 +13,9 @@ #include +#include "common_video/libyuv/include/libyuv.h" +#include "common_video/libyuv/include/scaler.h" +#include "modules/interface/module_common_types.h" #include "modules/video_coding/codecs/interface/video_codec_interface.h" #include "modules/video_coding/codecs/test/packet_manipulator.h" #include "modules/video_coding/codecs/test/stats.h" @@ -140,6 +143,19 @@ class VideoProcessor { // available in the source clip. // Frame number must be an integer >=0. virtual bool ProcessFrame(int frame_number) = 0; + + // Updates the encoder with the target bit rate and the frame rate. + virtual void SetRates(int bit_rate, int frame_rate) = 0; + + // Return the size of the encoded frame in bytes. Dropped frames by the + // encoder are regarded as zero size. + virtual int EncodedFrameSize() = 0; + + // Return the number of dropped frames. + virtual int NumberDroppedFrames() = 0; + + // Return the number of spatial resizes. + virtual int NumberSpatialResizes() = 0; }; class VideoProcessorImpl : public VideoProcessor { @@ -164,6 +180,14 @@ class VideoProcessorImpl : public VideoProcessor { // (checks the size is within signed 32-bit bounds before casting it) int GetElapsedTimeMicroseconds(const webrtc::TickTime& start, const webrtc::TickTime& stop); + // Updates the encoder with the target bit rate and the frame rate. + void SetRates(int bit_rate, int frame_rate); + // Return the size of the encoded frame in bytes. + int EncodedFrameSize(); + // Return the number of dropped frames. + int NumberDroppedFrames(); + // Return the number of spatial resizes. + int NumberSpatialResizes(); webrtc::VideoEncoder* encoder_; webrtc::VideoDecoder* decoder_; @@ -187,6 +211,13 @@ class VideoProcessorImpl : public VideoProcessor { bool last_frame_missing_; // If Init() has executed successfully. bool initialized_; + int encoded_frame_size_; + int prev_time_stamp_; + int num_dropped_frames_; + int num_spatial_resizes_; + int last_encoder_frame_width_; + int last_encoder_frame_height_; + Scaler scaler_; // Statistics double bit_rate_factor_; // multiply frame length with this to get bit rate diff --git a/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index 1988064b00..6707cd4e7b 100644 --- a/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -9,10 +9,14 @@ */ #include "gtest/gtest.h" + +#include + #include "modules/video_coding/codecs/interface/video_codec_interface.h" #include "modules/video_coding/codecs/test/packet_manipulator.h" #include "modules/video_coding/codecs/test/videoprocessor.h" #include "modules/video_coding/codecs/vp8/main/interface/vp8.h" +#include "modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h" #include "modules/video_coding/main/interface/video_coding.h" #include "testsupport/fileutils.h" #include "testsupport/frame_reader.h" @@ -23,17 +27,31 @@ namespace webrtc { -const int kNbrFrames = 61; // foreman_cif_short.yuv +struct CodecConfigPars { + float packet_loss; + int num_temporal_layers; + int key_frame_interval; + bool error_concealment_on; + bool denoising_on; +}; + +// Sequence used is foreman (CIF): may be better to use VGA for resize test. const int kCIFWidth = 352; const int kCIFHeight = 288; -const int kBitRateKbps = 500; +const int kFrameRate = 30; +const int kStartBitRateKbps = 300; +const int kNbrFramesShort = 100; // Some tests are run for shorter sequence. +const int kNbrFramesLong = 299; +const int kPercTargetvsActualMismatch = 20; -// Integration test for video processor. Encodes+decodes a small clip and -// writes it to the output directory. After completion, PSNR and SSIM -// measurements are performed on the original and the processed clip to verify -// the quality is acceptable. -// The limits for the PSNR and SSIM values must be set quite low, since we have -// no control over the random function used for packet loss in this test. +// Integration test for video processor. Encodes+decodes a clip and +// writes it to the output directory. After completion, quality metrics +// (PSNR and SSIM) and rate control metrics are computed to verify that the +// quality and encoder response is acceptable. The rate control tests allow us +// to verify the behavior for changing bitrate, changing frame rate, frame +// dropping/spatial resize, and temporal layers. The limits for the rate +// control metrics are set to be fairly conservative, so failure should only +// happen when some significant regression or breakdown occurs. class VideoProcessorIntegrationTest: public testing::Test { protected: VideoEncoder* encoder_; @@ -47,30 +65,67 @@ class VideoProcessorIntegrationTest: public testing::Test { VideoCodec codec_settings_; webrtc::test::VideoProcessor* processor_; + // Quantities defined/updated for every encoder rate update. + // Some quantities defined per temporal layer (at most 3 layers in this test). + int num_frames_per_update_[3]; + float sum_frame_size_mismatch_[3]; + float sum_encoded_frame_size_[3]; + float encoding_bitrate_[3]; + float per_frame_bandwidth_[3]; + float bit_rate_layer_[3]; + float frame_rate_layer_[3]; + int num_frames_total_; + float sum_encoded_frame_size_total_; + float encoding_bitrate_total_; + float perc_encoding_rate_mismatch_; + int num_frames_to_hit_target_; + bool encoding_rate_within_target_; + int bit_rate_; + int frame_rate_; + int layer_; + + // Codec and network settings. + float packet_loss_; + int num_temporal_layers_; + int key_frame_interval_; + bool error_concealment_on_; + bool denoising_on_; + + VideoProcessorIntegrationTest() {} virtual ~VideoProcessorIntegrationTest() {} - void SetUp() { + void SetUpCodecConfig() { encoder_ = VP8Encoder::Create(); decoder_ = VP8Decoder::Create(); + // CIF is currently used for all tests below. // Setup the TestConfig struct for processing of a clip in CIF resolution. config_.input_filename = - webrtc::test::ResourcePath("foreman_cif_short", "yuv"); + webrtc::test::ResourcePath("foreman_cif", "yuv"); config_.output_filename = webrtc::test::OutputPath() + - "foreman_cif_short_video_codecs_test_framework_integrationtests.yuv"; + "foreman_cif_short_video_codecs_test_framework_integrationtests.yuv"; config_.frame_length_in_bytes = 3 * kCIFWidth * kCIFHeight / 2; config_.verbose = false; // Only allow encoder/decoder to use single core, for predictability. config_.use_single_core = true; + // Key frame interval and packet loss are set for each test. + config_.keyframe_interval = key_frame_interval_; + config_.networking_config.packet_loss_probability = packet_loss_; // Get a codec configuration struct and configure it. VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_); config_.codec_settings = &codec_settings_; - config_.codec_settings->startBitrate = kBitRateKbps; + config_.codec_settings->startBitrate = kStartBitRateKbps; config_.codec_settings->width = kCIFWidth; config_.codec_settings->height = kCIFHeight; - config_.codec_settings->codecSpecific.VP8.errorConcealmentOn = true; + // These features may be set depending on the test. + config_.codec_settings->codecSpecific.VP8.errorConcealmentOn = + error_concealment_on_; + config_.codec_settings->codecSpecific.VP8.denoisingOn = + denoising_on_; + config_.codec_settings->codecSpecific.VP8.numberOfTemporalLayers = + num_temporal_layers_; frame_reader_ = new webrtc::test::FrameReaderImpl(config_.input_filename, @@ -91,6 +146,160 @@ class VideoProcessorIntegrationTest: public testing::Test { ASSERT_TRUE(processor_->Init()); } + // Reset quantities after each encoder update, update the target + // per-frame bandwidth. + void ResetRateControlMetrics(int num_frames) { + for (int i = 0; i < num_temporal_layers_; i++) { + num_frames_per_update_[i] = 0; + sum_frame_size_mismatch_[i] = 0.0f; + sum_encoded_frame_size_[i] = 0.0f; + encoding_bitrate_[i] = 0.0f; + // Update layer per-frame-bandwidth. + per_frame_bandwidth_[i] = static_cast(bit_rate_layer_[i]) / + static_cast(frame_rate_layer_[i]); + } + num_frames_total_ = 0; + sum_encoded_frame_size_total_ = 0.0f; + encoding_bitrate_total_ = 0.0f; + perc_encoding_rate_mismatch_ = 0.0f; + num_frames_to_hit_target_ = num_frames; + encoding_rate_within_target_ = false; + } + + // For every encoded frame, update the rate control metrics. + void UpdateRateControlMetrics(int frame_num, + int max_encoding_rate_mismatch) { + int encoded_frame_size = processor_->EncodedFrameSize(); + float encoded_size_kbits = encoded_frame_size * 8.0f / 1000.0f; + // Update layer data. + // Ignore first frame (key frame), and any other key frames in the run, + // for rate mismatch relative to per-frame bandwidth. + // Note |frame_num| = 1 for the first frame in the run. + if (frame_num > 1 && ((frame_num - 1) % key_frame_interval_ != 0 || + key_frame_interval_ < 0)) { + sum_frame_size_mismatch_[layer_] += fabs(encoded_size_kbits - + per_frame_bandwidth_[layer_]) / + per_frame_bandwidth_[layer_]; + } + sum_encoded_frame_size_[layer_] += encoded_size_kbits; + // Encoding bitrate per layer: from the start of the update/run to the + // current frame. + encoding_bitrate_[layer_] = sum_encoded_frame_size_[layer_] * + frame_rate_layer_[layer_] / + num_frames_per_update_[layer_]; + // Total encoding rate: from the start of the update/run to current frame. + sum_encoded_frame_size_total_ += encoded_size_kbits; + encoding_bitrate_total_ = sum_encoded_frame_size_total_ * frame_rate_ / + num_frames_total_; + perc_encoding_rate_mismatch_ = 100 * fabs(encoding_bitrate_total_ - + bit_rate_) / bit_rate_; + if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch && + !encoding_rate_within_target_) { + num_frames_to_hit_target_ = num_frames_total_; + encoding_rate_within_target_ = true; + } + } + + // Verify expected behavior of rate control and print out data. + void VerifyRateControl(int update_index, + int max_frame_size_mismatch, + int max_encoding_rate_mismatch, + int max_time_hit_target, + int max_num_dropped_frames, + int num_spatial_resizes) { + int num_dropped_frames = processor_->NumberDroppedFrames(); + int num_resize_actions = processor_->NumberSpatialResizes(); + printf("For update #: %d,\n " + " Target Bitrate: %d,\n" + " Encoding bitrate: %f,\n" + " Frame rate: %d \n", + update_index, bit_rate_, encoding_bitrate_total_, frame_rate_); + printf(" Number of frames to approach target rate = %d, \n" + " Number of dropped frames = %d, \n" + " Number of spatial resizes = %d, \n", + num_frames_to_hit_target_, num_dropped_frames, num_resize_actions); + EXPECT_LE(perc_encoding_rate_mismatch_, max_encoding_rate_mismatch); + printf("\n"); + printf("Rates statistics for Layer data \n"); + for (int i = 0; i < num_temporal_layers_ ; i++) { + printf("Layer #%d \n", i); + int perc_frame_size_mismatch = 100 * sum_frame_size_mismatch_[i] / + num_frames_per_update_[i]; + int perc_encoding_rate_mismatch = 100 * fabs(encoding_bitrate_[i] - + bit_rate_layer_[i]) / + bit_rate_layer_[i]; + printf(" Target Layer Bit rate: %f \n" + " Layer frame rate: %f, \n" + " Layer per frame bandwidth: %f, \n" + " Layer Encoding bit rate: %f, \n" + " Layer Percent frame size mismatch: %d, \n" + " Layer Percent encoding rate mismatch = %d, \n" + " Number of frame processed per layer = %d \n", + bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i], + encoding_bitrate_[i], perc_frame_size_mismatch, + perc_encoding_rate_mismatch, num_frames_per_update_[i]); + EXPECT_LE(perc_frame_size_mismatch, max_frame_size_mismatch); + EXPECT_LE(perc_encoding_rate_mismatch, max_encoding_rate_mismatch); + } + printf("\n"); + EXPECT_LE(num_frames_to_hit_target_, max_time_hit_target); + EXPECT_LE(num_dropped_frames, max_num_dropped_frames); + // Only if the spatial resizer is on in the codec wrapper do we expect to + // get |num_spatial_resizes| resizes, otherwise we should not get any. + EXPECT_TRUE(num_resize_actions == 0 || + num_resize_actions == num_spatial_resizes); + } + + // Layer index corresponding to frame number, for up to 3 layers. + void LayerIndexForFrame(int frame_number) { + if (num_temporal_layers_ == 1) { + layer_ = 0; + } else if (num_temporal_layers_ == 2) { + // layer 0: 0 2 4 ... + // layer 1: 1 3 + if (frame_number % 2 == 0) { + layer_ = 0; + } else { + layer_ = 1; + } + } else if (num_temporal_layers_ == 3) { + // layer 0: 0 4 8 ... + // layer 1: 2 6 + // layer 2: 1 3 5 7 + if (frame_number % 4 == 0) { + layer_ = 0; + } else if ((frame_number + 2) % 4 == 0) { + layer_ = 1; + } else if ((frame_number + 1) % 2 == 0) { + layer_ = 2; + } + } else { + assert(false); // Only up to 3 layers. + } + } + + // Set the bitrate and frame rate per layer, for up to 3 layers. + void SetLayerRates() { + assert(num_temporal_layers_<= 3); + for (int i = 0; i < num_temporal_layers_; i++) { + float bit_rate_ratio = + kVp8LayerRateAlloction[num_temporal_layers_ - 1][i]; + if (i > 0) { + float bit_rate_delta_ratio = kVp8LayerRateAlloction + [num_temporal_layers_ - 1][i] - + kVp8LayerRateAlloction[num_temporal_layers_ - 1][i - 1]; + bit_rate_layer_[i] = bit_rate_ * bit_rate_delta_ratio; + } else { + bit_rate_layer_[i] = bit_rate_ * bit_rate_ratio; + } + frame_rate_layer_[i] = frame_rate_ / static_cast( + 1 << (num_temporal_layers_ - 1)); + } + if (num_temporal_layers_ == 3) { + frame_rate_layer_[2] = frame_rate_ / 2.0f; + } + } + void TearDown() { delete processor_; delete packet_manipulator_; @@ -104,13 +313,70 @@ class VideoProcessorIntegrationTest: public testing::Test { void ProcessFramesAndVerify(double minimum_avg_psnr, double minimum_min_psnr, double minimum_avg_ssim, - double minimum_min_ssim) { + double minimum_min_ssim, + int* target_bit_rate, + int* input_frame_rate, + int* frame_index_rate_update, + int num_frames, + CodecConfigPars process, + int* max_frame_size_mismatch, + int* max_encoding_rate_mismatch, + int* max_time_hit_target, + int* max_num_dropped_frames, + int* num_spatial_resizes) { + // Codec/config settings. + packet_loss_ = process.packet_loss; + key_frame_interval_ = process.key_frame_interval; + num_temporal_layers_ = process.num_temporal_layers; + error_concealment_on_ = process.error_concealment_on; + denoising_on_ = process.denoising_on; + SetUpCodecConfig(); + // Update the layers and the codec with the initial rates. + bit_rate_ = target_bit_rate[0]; + frame_rate_ = input_frame_rate[0]; + SetLayerRates(); + processor_->SetRates(bit_rate_, frame_rate_); + // Process each frame, up to |num_frames|. + int update_index = 0; + ResetRateControlMetrics(frame_index_rate_update[update_index + 1]); int frame_number = 0; - while (processor_->ProcessFrame(frame_number)) { - frame_number++; + while (processor_->ProcessFrame(frame_number) && + frame_number < num_frames) { + // Get the layer index for the frame |frame_number|. + LayerIndexForFrame(frame_number); + // Counter for whole sequence run. + ++frame_number; + // Counters for each rate update. + ++num_frames_per_update_[layer_]; + ++num_frames_total_; + UpdateRateControlMetrics(frame_number, + max_encoding_rate_mismatch[update_index]); + // If we hit another/next update, verify stats for current state and + // update layers and codec with new rates. + if (frame_number == frame_index_rate_update[update_index + 1]) { + VerifyRateControl(update_index, + max_frame_size_mismatch[update_index], + max_encoding_rate_mismatch[update_index], + max_time_hit_target[update_index], + max_num_dropped_frames[update_index], + num_spatial_resizes[update_index]); + // Update layer rates and the codec with new rates. + ++update_index; + bit_rate_ = target_bit_rate[update_index]; + frame_rate_ = input_frame_rate[update_index]; + SetLayerRates(); + ResetRateControlMetrics(frame_index_rate_update[update_index + 1]); + processor_->SetRates(bit_rate_, frame_rate_); + } } - EXPECT_EQ(kNbrFrames, frame_number); - EXPECT_EQ(kNbrFrames, static_cast(stats_.stats_.size())); + VerifyRateControl(update_index, + max_frame_size_mismatch[update_index], + max_encoding_rate_mismatch[update_index], + max_time_hit_target[update_index], + max_num_dropped_frames[update_index], + num_spatial_resizes[update_index]); + EXPECT_EQ(num_frames, frame_number); + EXPECT_EQ(num_frames + 1, static_cast(stats_.stats_.size())); // Release encoder and decoder to make sure they have finished processing: EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); @@ -119,6 +385,7 @@ class VideoProcessorIntegrationTest: public testing::Test { frame_reader_->Close(); frame_writer_->Close(); + // TODO(marpan): should compute these quality metrics per SetRates update. webrtc::test::QualityMetricsResult psnr_result, ssim_result; EXPECT_EQ(0, webrtc::test::I420MetricsFromFiles( config_.input_filename.c_str(), @@ -138,37 +405,292 @@ class VideoProcessorIntegrationTest: public testing::Test { } }; -// Run with no packet loss. Quality should be very high. +// Run with no packet loss and fixed bitrate. Quality should be very high. +// One key frame (first frame only) in sequence. Setting |key_frame_interval| +// to -1 below means no periodic key frames in test. TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) { - config_.networking_config.packet_loss_probability = 0; + // Bitrate and frame rate profile. + int target_bit_rate[] = {500}; // kbps + int input_frame_rate[] = {kFrameRate}; + int frame_index_rate_update[] = {0, kNbrFramesShort + 1}; + int num_frames = kNbrFramesShort; + // Codec/network settings. + CodecConfigPars process_settings; + process_settings.packet_loss = 0.0f; + process_settings.key_frame_interval = -1; + process_settings.num_temporal_layers = 1; + process_settings.error_concealment_on = true; + process_settings.denoising_on = true; + // Metrics for expected quality. double minimum_avg_psnr = 36; - double minimum_min_psnr = 34; + double minimum_min_psnr = 32; double minimum_avg_ssim = 0.9; double minimum_min_ssim = 0.9; + // Metrics for rate control: rate mismatch metrics are defined as percentages. + // |max_time_hit_target| is defined as number of frames, after a rate update + // is made to the encoder, for the encoder to reach within + // |kPercTargetvsActualMismatch| of new target rate. + int max_num_dropped_frames[] = {0}; + int max_frame_size_mismatch[] = {20}; + int max_encoding_rate_mismatch[] = {15}; + int max_time_hit_target[] = {15}; + int num_spatial_resizes[] = {0}; ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr, - minimum_avg_ssim, minimum_min_ssim); + minimum_avg_ssim, minimum_min_ssim, + target_bit_rate, input_frame_rate, + frame_index_rate_update, num_frames, process_settings, + max_frame_size_mismatch, max_encoding_rate_mismatch, + max_time_hit_target, max_num_dropped_frames, + num_spatial_resizes); } -// Run with 5% packet loss. Quality should be a bit lower. +// Run with 5% packet loss and fixed bitrate. Quality should be a bit lower. +// One key frame (first frame only) in sequence. TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) { - config_.networking_config.packet_loss_probability = 0.05; + // Bitrate and frame rate profile, and codec settings. + int target_bit_rate[] = {500}; // kbps + int input_frame_rate[] = {kFrameRate}; + int frame_index_rate_update[] = {0, kNbrFramesShort + 1}; + int num_frames = kNbrFramesShort; + // Codec/network settings. + CodecConfigPars process_settings; + process_settings.packet_loss = 0.05f; + process_settings.key_frame_interval = -1; + process_settings.num_temporal_layers = 1; + process_settings.error_concealment_on = true; + process_settings.denoising_on = true; + // Metrics for expected quality. double minimum_avg_psnr = 21; double minimum_min_psnr = 16; double minimum_avg_ssim = 0.6; double minimum_min_ssim = 0.4; + // Metrics for rate control: rate mismatch metrics are defined as percentages. + // |max_time_hit_target| is defined as number of frames, after a rate update + // is made to the encoder, for the encoder to reach within + // |kPercTargetvsActualMismatch| of new target rate. + int max_num_dropped_frames[] = {0}; + int max_frame_size_mismatch[] = {25}; + int max_encoding_rate_mismatch[] = {15}; + int max_time_hit_target[] = {15}; + int num_spatial_resizes[] = {0}; ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr, - minimum_avg_ssim, minimum_min_ssim); + minimum_avg_ssim, minimum_min_ssim, + target_bit_rate, input_frame_rate, + frame_index_rate_update, num_frames, process_settings, + max_frame_size_mismatch, max_encoding_rate_mismatch, + max_time_hit_target, max_num_dropped_frames, + num_spatial_resizes); } -// Run with 10% packet loss. Quality should be even lower. +// Run with 10% packet loss and fixed bitrate. Quality should be even lower. +// One key frame (first frame only) in sequence. TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) { - config_.networking_config.packet_loss_probability = 0.10; + // Bitrate and frame rate profile, and codec settings. + int target_bit_rate[] = {500}; // kbps + int input_frame_rate[] = {kFrameRate}; + int frame_index_rate_update[] = {0, kNbrFramesShort + 1}; + int num_frames = kNbrFramesShort; + // Codec/network settings. + CodecConfigPars process_settings; + process_settings.packet_loss = 0.1f; + process_settings.key_frame_interval = -1; + process_settings.num_temporal_layers = 1; + process_settings.error_concealment_on = true; + process_settings.denoising_on = true; + // Metrics for expected quality. double minimum_avg_psnr = 19; double minimum_min_psnr = 16; double minimum_avg_ssim = 0.5; double minimum_min_ssim = 0.35; + // Metrics for rate control: rate mismatch metrics are defined as percentages. + // |max_time_hit_target| is defined as number of frames, after a rate update + // is made to the encoder, for the encoder to reach within + // |kPercTargetvsActualMismatch| of new target rate. + int max_num_dropped_frames[] = {0}; + int max_frame_size_mismatch[] = {25}; + int max_encoding_rate_mismatch[] = {15}; + int max_time_hit_target[] = {15}; + int num_spatial_resizes[] = {0}; ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr, - minimum_avg_ssim, minimum_min_ssim); + minimum_avg_ssim, minimum_min_ssim, + target_bit_rate, input_frame_rate, + frame_index_rate_update, num_frames, process_settings, + max_frame_size_mismatch, max_encoding_rate_mismatch, + max_time_hit_target, max_num_dropped_frames, + num_spatial_resizes); } +// Run with no packet loss, with varying bitrate (3 rate updates): +// low to high to medium. Check that quality and encoder response to the new +// target rate/per-frame bandwidth (for each rate update) is within limits. +// One key frame (first frame only) in sequence. +TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeBitRate) { + // Bitrate and frame rate profile, and codec settings. + int target_bit_rate[] = {200, 800, 500}; // kbps + int input_frame_rate[] = {30, 30, 30}; + int frame_index_rate_update[] = {0, 100, 200, kNbrFramesLong + 1}; + int num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + process_settings.packet_loss = 0.0f; + process_settings.key_frame_interval = -1; + process_settings.num_temporal_layers = 1; + process_settings.error_concealment_on = true; + process_settings.denoising_on = true; + // Metrics for expected quality. + double minimum_avg_psnr = 34; + double minimum_min_psnr = 32; + double minimum_avg_ssim = 0.85; + double minimum_min_ssim = 0.8; + // Metrics for rate control: rate mismatch metrics are defined as percentages. + // |max_time_hit_target| is defined as number of frames, after a rate update + // is made to the encoder, for the encoder to reach within + // |kPercTargetvsActualMismatch| of new target rate. + int max_num_dropped_frames[] = {0, 0, 0}; + int max_frame_size_mismatch[] = {20, 25, 25}; + int max_encoding_rate_mismatch[] = {10, 20, 15}; + int max_time_hit_target[] = {15, 10, 10}; + int num_spatial_resizes[] = {0, 0, 0}; + ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr, + minimum_avg_ssim, minimum_min_ssim, + target_bit_rate, input_frame_rate, + frame_index_rate_update, num_frames, process_settings, + max_frame_size_mismatch, max_encoding_rate_mismatch, + max_time_hit_target, max_num_dropped_frames, + num_spatial_resizes); +} + +// Run with no packet loss, with an update (decrease) in frame rate. +// Lower frame rate means higher per-frame-bandwidth, so easier to encode. +// At the bitrate in this test, this means better rate control after the +// update(s) to lower frame rate. So expect less frame drops, and max values +// for the rate control metrics can be lower. One key frame (first frame only). +// Note: quality after update should be higher but we currently compute quality +// metrics avergaed over whole sequence run. +TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeFrameRateFrameDrop) { + config_.networking_config.packet_loss_probability = 0; + // Bitrate and frame rate profile, and codec settings. + int target_bit_rate[] = {80, 80, 80}; // kbps + int input_frame_rate[] = {30, 15, 10}; + int frame_index_rate_update[] = {0, 100, 200, kNbrFramesLong + 1}; + int num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + process_settings.packet_loss = 0.0f; + process_settings.key_frame_interval = -1; + process_settings.num_temporal_layers = 1; + process_settings.error_concealment_on = true; + process_settings.denoising_on = true; + // Metrics for expected quality. + double minimum_avg_psnr = 31; + double minimum_min_psnr = 24; + double minimum_avg_ssim = 0.8; + double minimum_min_ssim = 0.7; + // Metrics for rate control: rate mismatch metrics are defined as percentages. + // |max_time_hit_target| is defined as number of frames, after a rate update + // is made to the encoder, for the encoder to reach within + // |kPercTargetvsActualMismatch| of new target rate. + int max_num_dropped_frames[] = {25, 10, 0}; + int max_frame_size_mismatch[] = {60, 25, 20}; + int max_encoding_rate_mismatch[] = {40, 10, 10}; + // At the low per-frame bandwidth for this scene, encoder can't hit target + // rate before first update. + int max_time_hit_target[] = {100, 40, 10}; + int num_spatial_resizes[] = {0, 0, 0}; + ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr, + minimum_avg_ssim, minimum_min_ssim, + target_bit_rate, input_frame_rate, + frame_index_rate_update, num_frames, process_settings, + max_frame_size_mismatch, max_encoding_rate_mismatch, + max_time_hit_target, max_num_dropped_frames, + num_spatial_resizes); +} + +// Run with no packet loss, at low bitrate, then increase rate somewhat. +// Key frame is thrown in every 120 frames. Can expect some frame drops after +// key frame, even at high rate. If resizer is on, expect spatial resize down +// at first key frame, and back up at second key frame. Expected values for +// quality and rate control in this test are such that the test should pass +// with resizing on or off. Error_concealment is off in this test since there +// is a memory leak with resizing and error concealment. +TEST_F(VideoProcessorIntegrationTest, ProcessNoLossSpatialResizeFrameDrop) { + config_.networking_config.packet_loss_probability = 0; + // Bitrate and frame rate profile, and codec settings. + int target_bit_rate[] = {80, 200, 200}; // kbps + int input_frame_rate[] = {30, 30, 30}; + int frame_index_rate_update[] = {0, 120, 240, kNbrFramesLong + 1}; + int num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + process_settings.packet_loss = 0.0f; + process_settings.key_frame_interval = 120; + process_settings.num_temporal_layers = 1; + process_settings.error_concealment_on = false; + process_settings.denoising_on = true; + // Metrics for expected quality.: lower quality on average from up-sampling + // the down-sampled portion of the run, in case resizer is on. + double minimum_avg_psnr = 29; + double minimum_min_psnr = 20; + double minimum_avg_ssim = 0.75; + double minimum_min_ssim = 0.6; + // Metrics for rate control: rate mismatch metrics are defined as percentages. + // |max_time_hit_target| is defined as number of frames, after a rate update + // is made to the encoder, for the encoder to reach within + // |kPercTargetvsActualMismatch| of new target rate. + int max_num_dropped_frames[] = {25, 15, 0}; + int max_frame_size_mismatch[] = {60, 30, 30}; + int max_encoding_rate_mismatch[] = {30, 20, 15}; + // At this low rate for this scene, can't hit target rate before first update. + int max_time_hit_target[] = {120, 15, 25}; + int num_spatial_resizes[] = {0, 1, 1}; + ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr, + minimum_avg_ssim, minimum_min_ssim, + target_bit_rate, input_frame_rate, + frame_index_rate_update, num_frames, process_settings, + max_frame_size_mismatch, max_encoding_rate_mismatch, + max_time_hit_target, max_num_dropped_frames, + num_spatial_resizes); +} + +// Run with no packet loss, with 3 temporal layers, with a rate update in the +// middle of the sequence. The max values for the frame size mismatch and +// encoding rate mismatch are applied to each layer. +// No dropped frames in this test, and the denoiser is off for temporal layers. +// One key frame (first frame only) in sequence, so no spatial resizing. +TEST_F(VideoProcessorIntegrationTest, ProcessNoLossTemporalLayers) { + config_.networking_config.packet_loss_probability = 0; + // Bitrate and frame rate profile, and codec settings. + int target_bit_rate[] = {200, 400}; // kbps + int input_frame_rate[] = {30, 30}; + int frame_index_rate_update[] = {0, 150, kNbrFramesLong + 1}; + int num_frames = kNbrFramesLong; + // Codec/network settings. + CodecConfigPars process_settings; + process_settings.packet_loss = 0.0f; + process_settings.key_frame_interval = -1; + process_settings.num_temporal_layers = 3; + process_settings.error_concealment_on = true; + process_settings.denoising_on = false; + // Metrics for expected quality. + double minimum_avg_psnr = 33; + double minimum_min_psnr = 30; + double minimum_avg_ssim = 0.85; + double minimum_min_ssim = 0.80; + // Metrics for rate control: rate mismatch metrics are defined as percentages. + // |max_time_hit_target| is defined as number of frames, after a rate update + // is made to the encoder, for the encoder to reach within + // |kPercTargetvsActualMismatch| of new target rate. + int max_num_dropped_frames[] = {0, 0}; + int max_frame_size_mismatch[] = {30, 30}; + int max_encoding_rate_mismatch[] = {10, 12}; + int max_time_hit_target[] = {15, 15}; + int num_spatial_resizes[] = {0, 0}; + ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr, + minimum_avg_ssim, minimum_min_ssim, + target_bit_rate, input_frame_rate, + frame_index_rate_update, num_frames, process_settings, + max_frame_size_mismatch, max_encoding_rate_mismatch, + max_time_hit_target, max_num_dropped_frames, + num_spatial_resizes); +} } // namespace webrtc