zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added some tests to videoprocessor_integrationtest, for testing:

Browse Source 
-encooder response to changing bit rate and frame rate
   -frame dropper and spatial resize
   -temporal layers
Review URL: https://webrtc-codereview.appspot.com/613006

git-svn-id: http://webrtc.googlecode.com/svn/trunk@2370 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
marpan@webrtc.org
2012-06-05 21:07:28 +00:00
parent 8866bb1132
commit f4c2de9e2f
3 changed files with 697 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
src/modules/video_coding/codecs/test/videoprocessor.cc
View File

@ -38,7 +38,14 @@ VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder,
source_buffer_(NULL),
first_key_frame_has_been_excluded_(false),
last_frame_missing_(false),
initialized_(false) {
initialized_(false),
encoded_frame_size_(0),
prev_time_stamp_(0),
num_dropped_frames_(0),
num_spatial_resizes_(0),
last_encoder_frame_width_(0),
last_encoder_frame_height_(0),
scaler_() {
assert(encoder);
assert(decoder);
assert(frame_reader);
@ -63,6 +70,10 @@ bool VideoProcessorImpl::Init() {
source_frame_._length = frame_length_in_bytes;
source_frame_._size = frame_length_in_bytes;
// To keep track of spatial resize actions by encoder.
last_encoder_frame_width_ = config_.codec_settings->width;
last_encoder_frame_height_ = config_.codec_settings->height;
// Setup required callbacks for the encoder/decoder:
encode_callback_ = new VideoProcessorEncodeCompleteCallback(this);
decode_callback_ = new VideoProcessorDecodeCompleteCallback(this);
@ -122,12 +133,40 @@ VideoProcessorImpl::~VideoProcessorImpl() {
delete decode_callback_;
}
void VideoProcessorImpl::SetRates(int bit_rate, int frame_rate) {
int set_rates_result = encoder_->SetRates(bit_rate, frame_rate);
assert(set_rates_result >= 0);
if (set_rates_result < 0) {
fprintf(stderr, "Failed to update encoder with new rate %d, "
"return code: %d\n", bit_rate, set_rates_result);
}
num_dropped_frames_ = 0;
num_spatial_resizes_ = 0;
}
int VideoProcessorImpl::EncodedFrameSize() {
return encoded_frame_size_;
}
int VideoProcessorImpl::NumberDroppedFrames() {
return num_dropped_frames_;
}
int VideoProcessorImpl::NumberSpatialResizes() {
return num_spatial_resizes_;
}
bool VideoProcessorImpl::ProcessFrame(int frame_number) {
assert(frame_number >=0);
if (!initialized_) {
fprintf(stderr, "Attempting to use uninitialized VideoProcessor!\n");
return false;
}
// |prev_time_stamp_| is used for getting number of dropped frames.
if (frame_number == 0) {
prev_time_stamp_ = -1;
}
if (frame_reader_->ReadFrame(source_buffer_)) {
// point the source frame buffer to the newly read frame data:
source_frame_._buffer = source_buffer_;
@ -145,8 +184,13 @@ bool VideoProcessorImpl::ProcessFrame(int frame_number) {
frame_number % config_.keyframe_interval == 0) {
frame_type = kKeyFrame;
}
// For dropped frames, we regard them as zero size encoded frames.
encoded_frame_size_ = 0;
WebRtc_Word32 encode_result = encoder_->Encode(source_frame_, NULL,
frame_type);
if (encode_result != WEBRTC_VIDEO_CODEC_OK) {
fprintf(stderr, "Failed to encode frame %d, return code: %d\n",
frame_number, encode_result);
@ -159,6 +203,22 @@ bool VideoProcessorImpl::ProcessFrame(int frame_number) {
}
void VideoProcessorImpl::FrameEncoded(EncodedImage* encoded_image) {
// Timestamp is frame number, so this gives us #dropped frames.
int num_dropped_from_prev_encode = encoded_image->_timeStamp -
prev_time_stamp_ - 1;
num_dropped_frames_ += num_dropped_from_prev_encode;
prev_time_stamp_ = encoded_image->_timeStamp;
if (num_dropped_from_prev_encode > 0) {
// For dropped frames, we write out the last decoded frame to avoid getting
// out of sync for the computation of PSNR and SSIM.
for (int i = 0; i < num_dropped_from_prev_encode; i++) {
frame_writer_->WriteFrame(last_successful_frame_buffer_);
}
}
// Frame is not dropped, so update the encoded frame size
// (encoder callback is only called for non-zero length frames).
encoded_frame_size_ = encoded_image->_length;
TickTime encode_stop = TickTime::Now();
int frame_number = encoded_image->_timeStamp;
FrameStatistic& stat = stats_->stats_[frame_number];
@ -220,13 +280,61 @@ void VideoProcessorImpl::FrameDecoded(const RawImage& image) {
stat.decode_time_in_us = GetElapsedTimeMicroseconds(decode_start_,
decode_stop);
stat.decoding_successful = true;
// Check for resize action (either down or up):
if (static_cast<int>(image._width) != last_encoder_frame_width_ ||
static_cast<int>(image._height) != last_encoder_frame_height_ ) {
++num_spatial_resizes_;
last_encoder_frame_width_ = image._width;
last_encoder_frame_height_ = image._height;
}
// Check if codec size is different from native/original size, and if so,
// upsample back to original size: needed for PSNR and SSIM computations.
if (image._width != config_.codec_settings->width ||
image._height != config_.codec_settings->height) {
int required_size = static_cast<WebRtc_UWord32>
(config_.codec_settings->width * config_.codec_settings->height * 3 / 2);
RawImage up_image = image;
up_image._buffer = new uint8_t[required_size];
up_image._length = required_size;
up_image._width = config_.codec_settings->width;
up_image._height = config_.codec_settings->height;
int ret_val = scaler_.Set(image._width, image._height,
config_.codec_settings->width,
config_.codec_settings->height,
kI420, kI420, kScaleBilinear);
assert(ret_val >= 0);
if (ret_val < 0) {
fprintf(stderr, "Failed to set scalar for frame: %d, return code: %d\n",
frame_number, ret_val);
}
ret_val = scaler_.Scale(image._buffer, up_image._buffer,
required_size);
assert(ret_val >= 0);
if (ret_val < 0) {
fprintf(stderr, "Failed to scale frame: %d, return code: %d\n",
frame_number, ret_val);
}
// Update our copy of the last successful frame:
memcpy(last_successful_frame_buffer_, up_image._buffer, up_image._length);
bool write_success = frame_writer_->WriteFrame(up_image._buffer);
assert(write_success);
if (!write_success) {
fprintf(stderr, "Failed to write frame %d to disk!", frame_number);
}
delete [] up_image._buffer;
} else { // No resize.
// Update our copy of the last successful frame:
memcpy(last_successful_frame_buffer_, image._buffer, image._length);
bool write_success = frame_writer_->WriteFrame(image._buffer);
assert(write_success);
if (!write_success) {
fprintf(stderr, "Failed to write frame %d to disk!", frame_number);
}
}
}
int VideoProcessorImpl::GetElapsedTimeMicroseconds(
@ -273,7 +381,7 @@ VideoProcessorImpl::VideoProcessorEncodeCompleteCallback::Encoded(
EncodedImage& encoded_image,
const webrtc::CodecSpecificInfo* codec_specific_info,
const webrtc::RTPFragmentationHeader* fragmentation) {
video_processor_->FrameEncoded(&encoded_image); // forward to parent class
video_processor_->FrameEncoded(&encoded_image); // Forward to parent class.
return 0;
}
WebRtc_Word32

33
src/modules/video_coding/codecs/test/videoprocessor.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@ -13,6 +13,9 @@
#include <string>
#include "common_video/libyuv/include/libyuv.h"
#include "common_video/libyuv/include/scaler.h"
#include "modules/interface/module_common_types.h"
#include "modules/video_coding/codecs/interface/video_codec_interface.h"
#include "modules/video_coding/codecs/test/packet_manipulator.h"
#include "modules/video_coding/codecs/test/stats.h"
@ -140,6 +143,19 @@ class VideoProcessor {
// available in the source clip.
// Frame number must be an integer >=0.
virtual bool ProcessFrame(int frame_number) = 0;
// Updates the encoder with the target bit rate and the frame rate.
virtual void SetRates(int bit_rate, int frame_rate) = 0;
// Return the size of the encoded frame in bytes. Dropped frames by the
// encoder are regarded as zero size.
virtual int EncodedFrameSize() = 0;
// Return the number of dropped frames.
virtual int NumberDroppedFrames() = 0;
// Return the number of spatial resizes.
virtual int NumberSpatialResizes() = 0;
};
class VideoProcessorImpl : public VideoProcessor {
@ -164,6 +180,14 @@ class VideoProcessorImpl : public VideoProcessor {
// (checks the size is within signed 32-bit bounds before casting it)
int GetElapsedTimeMicroseconds(const webrtc::TickTime& start,
const webrtc::TickTime& stop);
// Updates the encoder with the target bit rate and the frame rate.
void SetRates(int bit_rate, int frame_rate);
// Return the size of the encoded frame in bytes.
int EncodedFrameSize();
// Return the number of dropped frames.
int NumberDroppedFrames();
// Return the number of spatial resizes.
int NumberSpatialResizes();
webrtc::VideoEncoder* encoder_;
webrtc::VideoDecoder* decoder_;
@ -187,6 +211,13 @@ class VideoProcessorImpl : public VideoProcessor {
bool last_frame_missing_;
// If Init() has executed successfully.
bool initialized_;
int encoded_frame_size_;
int prev_time_stamp_;
int num_dropped_frames_;
int num_spatial_resizes_;
int last_encoder_frame_width_;
int last_encoder_frame_height_;
Scaler scaler_;
// Statistics
double bit_rate_factor_; // multiply frame length with this to get bit rate

576
src/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc
View File

@ -9,10 +9,14 @@
*/
#include "gtest/gtest.h"
#include <math.h>
#include "modules/video_coding/codecs/interface/video_codec_interface.h"
#include "modules/video_coding/codecs/test/packet_manipulator.h"
#include "modules/video_coding/codecs/test/videoprocessor.h"
#include "modules/video_coding/codecs/vp8/main/interface/vp8.h"
#include "modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h"
#include "modules/video_coding/main/interface/video_coding.h"
#include "testsupport/fileutils.h"
#include "testsupport/frame_reader.h"
@ -23,17 +27,31 @@
namespace webrtc {
const int kNbrFrames = 61; // foreman_cif_short.yuv
struct CodecConfigPars {
float packet_loss;
int num_temporal_layers;
int key_frame_interval;
bool error_concealment_on;
bool denoising_on;
};
// Sequence used is foreman (CIF): may be better to use VGA for resize test.
const int kCIFWidth = 352;
const int kCIFHeight = 288;
const int kBitRateKbps = 500;
const int kFrameRate = 30;
const int kStartBitRateKbps = 300;
const int kNbrFramesShort = 100; // Some tests are run for shorter sequence.
const int kNbrFramesLong = 299;
const int kPercTargetvsActualMismatch = 20;
// Integration test for video processor. Encodes+decodes a small clip and
// writes it to the output directory. After completion, PSNR and SSIM
// measurements are performed on the original and the processed clip to verify
// the quality is acceptable.
// The limits for the PSNR and SSIM values must be set quite low, since we have
// no control over the random function used for packet loss in this test.
// Integration test for video processor. Encodes+decodes a clip and
// writes it to the output directory. After completion, quality metrics
// (PSNR and SSIM) and rate control metrics are computed to verify that the
// quality and encoder response is acceptable. The rate control tests allow us
// to verify the behavior for changing bitrate, changing frame rate, frame
// dropping/spatial resize, and temporal layers. The limits for the rate
// control metrics are set to be fairly conservative, so failure should only
// happen when some significant regression or breakdown occurs.
class VideoProcessorIntegrationTest: public testing::Test {
protected:
VideoEncoder* encoder_;
@ -47,30 +65,67 @@ class VideoProcessorIntegrationTest: public testing::Test {
VideoCodec codec_settings_;
webrtc::test::VideoProcessor* processor_;
// Quantities defined/updated for every encoder rate update.
// Some quantities defined per temporal layer (at most 3 layers in this test).
int num_frames_per_update_[3];
float sum_frame_size_mismatch_[3];
float sum_encoded_frame_size_[3];
float encoding_bitrate_[3];
float per_frame_bandwidth_[3];
float bit_rate_layer_[3];
float frame_rate_layer_[3];
int num_frames_total_;
float sum_encoded_frame_size_total_;
float encoding_bitrate_total_;
float perc_encoding_rate_mismatch_;
int num_frames_to_hit_target_;
bool encoding_rate_within_target_;
int bit_rate_;
int frame_rate_;
int layer_;
// Codec and network settings.
float packet_loss_;
int num_temporal_layers_;
int key_frame_interval_;
bool error_concealment_on_;
bool denoising_on_;
VideoProcessorIntegrationTest() {}
virtual ~VideoProcessorIntegrationTest() {}
void SetUp() {
void SetUpCodecConfig() {
encoder_ = VP8Encoder::Create();
decoder_ = VP8Decoder::Create();
// CIF is currently used for all tests below.
// Setup the TestConfig struct for processing of a clip in CIF resolution.
config_.input_filename =
webrtc::test::ResourcePath("foreman_cif_short", "yuv");
webrtc::test::ResourcePath("foreman_cif", "yuv");
config_.output_filename = webrtc::test::OutputPath() +
"foreman_cif_short_video_codecs_test_framework_integrationtests.yuv";
config_.frame_length_in_bytes = 3 * kCIFWidth * kCIFHeight / 2;
config_.verbose = false;
// Only allow encoder/decoder to use single core, for predictability.
config_.use_single_core = true;
// Key frame interval and packet loss are set for each test.
config_.keyframe_interval = key_frame_interval_;
config_.networking_config.packet_loss_probability = packet_loss_;
// Get a codec configuration struct and configure it.
VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_);
config_.codec_settings = &codec_settings_;
config_.codec_settings->startBitrate = kBitRateKbps;
config_.codec_settings->startBitrate = kStartBitRateKbps;
config_.codec_settings->width = kCIFWidth;
config_.codec_settings->height = kCIFHeight;
config_.codec_settings->codecSpecific.VP8.errorConcealmentOn = true;
// These features may be set depending on the test.
config_.codec_settings->codecSpecific.VP8.errorConcealmentOn =
error_concealment_on_;
config_.codec_settings->codecSpecific.VP8.denoisingOn =
denoising_on_;
config_.codec_settings->codecSpecific.VP8.numberOfTemporalLayers =
num_temporal_layers_;
frame_reader_ =
new webrtc::test::FrameReaderImpl(config_.input_filename,
@ -91,6 +146,160 @@ class VideoProcessorIntegrationTest: public testing::Test {
ASSERT_TRUE(processor_->Init());
}
// Reset quantities after each encoder update, update the target
// per-frame bandwidth.
void ResetRateControlMetrics(int num_frames) {
for (int i = 0; i < num_temporal_layers_; i++) {
num_frames_per_update_[i] = 0;
sum_frame_size_mismatch_[i] = 0.0f;
sum_encoded_frame_size_[i] = 0.0f;
encoding_bitrate_[i] = 0.0f;
// Update layer per-frame-bandwidth.
per_frame_bandwidth_[i] = static_cast<float>(bit_rate_layer_[i]) /
static_cast<float>(frame_rate_layer_[i]);
}
num_frames_total_ = 0;
sum_encoded_frame_size_total_ = 0.0f;
encoding_bitrate_total_ = 0.0f;
perc_encoding_rate_mismatch_ = 0.0f;
num_frames_to_hit_target_ = num_frames;
encoding_rate_within_target_ = false;
}
// For every encoded frame, update the rate control metrics.
void UpdateRateControlMetrics(int frame_num,
int max_encoding_rate_mismatch) {
int encoded_frame_size = processor_->EncodedFrameSize();
float encoded_size_kbits = encoded_frame_size * 8.0f / 1000.0f;
// Update layer data.
// Ignore first frame (key frame), and any other key frames in the run,
// for rate mismatch relative to per-frame bandwidth.
// Note |frame_num| = 1 for the first frame in the run.
if (frame_num > 1 && ((frame_num - 1) % key_frame_interval_ != 0 ||
key_frame_interval_ < 0)) {
sum_frame_size_mismatch_[layer_] += fabs(encoded_size_kbits -
per_frame_bandwidth_[layer_]) /
per_frame_bandwidth_[layer_];
}
sum_encoded_frame_size_[layer_] += encoded_size_kbits;
// Encoding bitrate per layer: from the start of the update/run to the
// current frame.
encoding_bitrate_[layer_] = sum_encoded_frame_size_[layer_] *
frame_rate_layer_[layer_] /
num_frames_per_update_[layer_];
// Total encoding rate: from the start of the update/run to current frame.
sum_encoded_frame_size_total_ += encoded_size_kbits;
encoding_bitrate_total_ = sum_encoded_frame_size_total_ * frame_rate_ /
num_frames_total_;
perc_encoding_rate_mismatch_ = 100 * fabs(encoding_bitrate_total_ -
bit_rate_) / bit_rate_;
if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch &&
!encoding_rate_within_target_) {
num_frames_to_hit_target_ = num_frames_total_;
encoding_rate_within_target_ = true;
}
}
// Verify expected behavior of rate control and print out data.
void VerifyRateControl(int update_index,
int max_frame_size_mismatch,
int max_encoding_rate_mismatch,
int max_time_hit_target,
int max_num_dropped_frames,
int num_spatial_resizes) {
int num_dropped_frames = processor_->NumberDroppedFrames();
int num_resize_actions = processor_->NumberSpatialResizes();
printf("For update #: %d,\n "
" Target Bitrate: %d,\n"
" Encoding bitrate: %f,\n"
" Frame rate: %d \n",
update_index, bit_rate_, encoding_bitrate_total_, frame_rate_);
printf(" Number of frames to approach target rate = %d, \n"
" Number of dropped frames = %d, \n"
" Number of spatial resizes = %d, \n",
num_frames_to_hit_target_, num_dropped_frames, num_resize_actions);
EXPECT_LE(perc_encoding_rate_mismatch_, max_encoding_rate_mismatch);
printf("\n");
printf("Rates statistics for Layer data \n");
for (int i = 0; i < num_temporal_layers_ ; i++) {
printf("Layer #%d \n", i);
int perc_frame_size_mismatch = 100 * sum_frame_size_mismatch_[i] /
num_frames_per_update_[i];
int perc_encoding_rate_mismatch = 100 * fabs(encoding_bitrate_[i] -
bit_rate_layer_[i]) /
bit_rate_layer_[i];
printf(" Target Layer Bit rate: %f \n"
" Layer frame rate: %f, \n"
" Layer per frame bandwidth: %f, \n"
" Layer Encoding bit rate: %f, \n"
" Layer Percent frame size mismatch: %d, \n"
" Layer Percent encoding rate mismatch = %d, \n"
" Number of frame processed per layer = %d \n",
bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i],
encoding_bitrate_[i], perc_frame_size_mismatch,
perc_encoding_rate_mismatch, num_frames_per_update_[i]);
EXPECT_LE(perc_frame_size_mismatch, max_frame_size_mismatch);
EXPECT_LE(perc_encoding_rate_mismatch, max_encoding_rate_mismatch);
}
printf("\n");
EXPECT_LE(num_frames_to_hit_target_, max_time_hit_target);
EXPECT_LE(num_dropped_frames, max_num_dropped_frames);
// Only if the spatial resizer is on in the codec wrapper do we expect to
// get |num_spatial_resizes| resizes, otherwise we should not get any.
EXPECT_TRUE(num_resize_actions == 0 ||
num_resize_actions == num_spatial_resizes);
}
// Layer index corresponding to frame number, for up to 3 layers.
void LayerIndexForFrame(int frame_number) {
if (num_temporal_layers_ == 1) {
layer_ = 0;
} else if (num_temporal_layers_ == 2) {
// layer 0: 0 2 4 ...
// layer 1: 1 3
if (frame_number % 2 == 0) {
layer_ = 0;
} else {
layer_ = 1;
}
} else if (num_temporal_layers_ == 3) {
// layer 0: 0 4 8 ...
// layer 1: 2 6
// layer 2: 1 3 5 7
if (frame_number % 4 == 0) {
layer_ = 0;
} else if ((frame_number + 2) % 4 == 0) {
layer_ = 1;
} else if ((frame_number + 1) % 2 == 0) {
layer_ = 2;
}
} else {
assert(false); // Only up to 3 layers.
}
}
// Set the bitrate and frame rate per layer, for up to 3 layers.
void SetLayerRates() {
assert(num_temporal_layers_<= 3);
for (int i = 0; i < num_temporal_layers_; i++) {
float bit_rate_ratio =
kVp8LayerRateAlloction[num_temporal_layers_ - 1][i];
if (i > 0) {
float bit_rate_delta_ratio = kVp8LayerRateAlloction
[num_temporal_layers_ - 1][i] -
kVp8LayerRateAlloction[num_temporal_layers_ - 1][i - 1];
bit_rate_layer_[i] = bit_rate_ * bit_rate_delta_ratio;
} else {
bit_rate_layer_[i] = bit_rate_ * bit_rate_ratio;
}
frame_rate_layer_[i] = frame_rate_ / static_cast<float>(
1 << (num_temporal_layers_ - 1));
}
if (num_temporal_layers_ == 3) {
frame_rate_layer_[2] = frame_rate_ / 2.0f;
}
}
void TearDown() {
delete processor_;
delete packet_manipulator_;
@ -104,13 +313,70 @@ class VideoProcessorIntegrationTest: public testing::Test {
void ProcessFramesAndVerify(double minimum_avg_psnr,
double minimum_min_psnr,
double minimum_avg_ssim,
double minimum_min_ssim) {
double minimum_min_ssim,
int* target_bit_rate,
int* input_frame_rate,
int* frame_index_rate_update,
int num_frames,
CodecConfigPars process,
int* max_frame_size_mismatch,
int* max_encoding_rate_mismatch,
int* max_time_hit_target,
int* max_num_dropped_frames,
int* num_spatial_resizes) {
// Codec/config settings.
packet_loss_ = process.packet_loss;
key_frame_interval_ = process.key_frame_interval;
num_temporal_layers_ = process.num_temporal_layers;
error_concealment_on_ = process.error_concealment_on;
denoising_on_ = process.denoising_on;
SetUpCodecConfig();
// Update the layers and the codec with the initial rates.
bit_rate_ = target_bit_rate[0];
frame_rate_ = input_frame_rate[0];
SetLayerRates();
processor_->SetRates(bit_rate_, frame_rate_);
// Process each frame, up to |num_frames|.
int update_index = 0;
ResetRateControlMetrics(frame_index_rate_update[update_index + 1]);
int frame_number = 0;
while (processor_->ProcessFrame(frame_number)) {
frame_number++;
while (processor_->ProcessFrame(frame_number) &&
frame_number < num_frames) {
// Get the layer index for the frame |frame_number|.
LayerIndexForFrame(frame_number);
// Counter for whole sequence run.
++frame_number;
// Counters for each rate update.
++num_frames_per_update_[layer_];
++num_frames_total_;
UpdateRateControlMetrics(frame_number,
max_encoding_rate_mismatch[update_index]);
// If we hit another/next update, verify stats for current state and
// update layers and codec with new rates.
if (frame_number == frame_index_rate_update[update_index + 1]) {
VerifyRateControl(update_index,
max_frame_size_mismatch[update_index],
max_encoding_rate_mismatch[update_index],
max_time_hit_target[update_index],
max_num_dropped_frames[update_index],
num_spatial_resizes[update_index]);
// Update layer rates and the codec with new rates.
++update_index;
bit_rate_ = target_bit_rate[update_index];
frame_rate_ = input_frame_rate[update_index];
SetLayerRates();
ResetRateControlMetrics(frame_index_rate_update[update_index + 1]);
processor_->SetRates(bit_rate_, frame_rate_);
}
EXPECT_EQ(kNbrFrames, frame_number);
EXPECT_EQ(kNbrFrames, static_cast<int>(stats_.stats_.size()));
}
VerifyRateControl(update_index,
max_frame_size_mismatch[update_index],
max_encoding_rate_mismatch[update_index],
max_time_hit_target[update_index],
max_num_dropped_frames[update_index],
num_spatial_resizes[update_index]);
EXPECT_EQ(num_frames, frame_number);
EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size()));
// Release encoder and decoder to make sure they have finished processing:
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
@ -119,6 +385,7 @@ class VideoProcessorIntegrationTest: public testing::Test {
frame_reader_->Close();
frame_writer_->Close();
// TODO(marpan): should compute these quality metrics per SetRates update.
webrtc::test::QualityMetricsResult psnr_result, ssim_result;
EXPECT_EQ(0, webrtc::test::I420MetricsFromFiles(
config_.input_filename.c_str(),
@ -138,37 +405,292 @@ class VideoProcessorIntegrationTest: public testing::Test {
}
};
// Run with no packet loss. Quality should be very high.
// Run with no packet loss and fixed bitrate. Quality should be very high.
// One key frame (first frame only) in sequence. Setting |key_frame_interval|
// to -1 below means no periodic key frames in test.
TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) {
config_.networking_config.packet_loss_probability = 0;
// Bitrate and frame rate profile.
int target_bit_rate[] = {500}; // kbps
int input_frame_rate[] = {kFrameRate};
int frame_index_rate_update[] = {0, kNbrFramesShort + 1};
int num_frames = kNbrFramesShort;
// Codec/network settings.
CodecConfigPars process_settings;
process_settings.packet_loss = 0.0f;
process_settings.key_frame_interval = -1;
process_settings.num_temporal_layers = 1;
process_settings.error_concealment_on = true;
process_settings.denoising_on = true;
// Metrics for expected quality.
double minimum_avg_psnr = 36;
double minimum_min_psnr = 34;
double minimum_min_psnr = 32;
double minimum_avg_ssim = 0.9;
double minimum_min_ssim = 0.9;
// Metrics for rate control: rate mismatch metrics are defined as percentages.
// |max_time_hit_target| is defined as number of frames, after a rate update
// is made to the encoder, for the encoder to reach within
// |kPercTargetvsActualMismatch| of new target rate.
int max_num_dropped_frames[] = {0};
int max_frame_size_mismatch[] = {20};
int max_encoding_rate_mismatch[] = {15};
int max_time_hit_target[] = {15};
int num_spatial_resizes[] = {0};
ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr,
minimum_avg_ssim, minimum_min_ssim);
minimum_avg_ssim, minimum_min_ssim,
target_bit_rate, input_frame_rate,
frame_index_rate_update, num_frames, process_settings,
max_frame_size_mismatch, max_encoding_rate_mismatch,
max_time_hit_target, max_num_dropped_frames,
num_spatial_resizes);
}
// Run with 5% packet loss. Quality should be a bit lower.
// Run with 5% packet loss and fixed bitrate. Quality should be a bit lower.
// One key frame (first frame only) in sequence.
TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) {
config_.networking_config.packet_loss_probability = 0.05;
// Bitrate and frame rate profile, and codec settings.
int target_bit_rate[] = {500}; // kbps
int input_frame_rate[] = {kFrameRate};
int frame_index_rate_update[] = {0, kNbrFramesShort + 1};
int num_frames = kNbrFramesShort;
// Codec/network settings.
CodecConfigPars process_settings;
process_settings.packet_loss = 0.05f;
process_settings.key_frame_interval = -1;
process_settings.num_temporal_layers = 1;
process_settings.error_concealment_on = true;
process_settings.denoising_on = true;
// Metrics for expected quality.
double minimum_avg_psnr = 21;
double minimum_min_psnr = 16;
double minimum_avg_ssim = 0.6;
double minimum_min_ssim = 0.4;
// Metrics for rate control: rate mismatch metrics are defined as percentages.
// |max_time_hit_target| is defined as number of frames, after a rate update
// is made to the encoder, for the encoder to reach within
// |kPercTargetvsActualMismatch| of new target rate.
int max_num_dropped_frames[] = {0};
int max_frame_size_mismatch[] = {25};
int max_encoding_rate_mismatch[] = {15};
int max_time_hit_target[] = {15};
int num_spatial_resizes[] = {0};
ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr,
minimum_avg_ssim, minimum_min_ssim);
minimum_avg_ssim, minimum_min_ssim,
target_bit_rate, input_frame_rate,
frame_index_rate_update, num_frames, process_settings,
max_frame_size_mismatch, max_encoding_rate_mismatch,
max_time_hit_target, max_num_dropped_frames,
num_spatial_resizes);
}
// Run with 10% packet loss. Quality should be even lower.
// Run with 10% packet loss and fixed bitrate. Quality should be even lower.
// One key frame (first frame only) in sequence.
TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) {
config_.networking_config.packet_loss_probability = 0.10;
// Bitrate and frame rate profile, and codec settings.
int target_bit_rate[] = {500}; // kbps
int input_frame_rate[] = {kFrameRate};
int frame_index_rate_update[] = {0, kNbrFramesShort + 1};
int num_frames = kNbrFramesShort;
// Codec/network settings.
CodecConfigPars process_settings;
process_settings.packet_loss = 0.1f;
process_settings.key_frame_interval = -1;
process_settings.num_temporal_layers = 1;
process_settings.error_concealment_on = true;
process_settings.denoising_on = true;
// Metrics for expected quality.
double minimum_avg_psnr = 19;
double minimum_min_psnr = 16;
double minimum_avg_ssim = 0.5;
double minimum_min_ssim = 0.35;
// Metrics for rate control: rate mismatch metrics are defined as percentages.
// |max_time_hit_target| is defined as number of frames, after a rate update
// is made to the encoder, for the encoder to reach within
// |kPercTargetvsActualMismatch| of new target rate.
int max_num_dropped_frames[] = {0};
int max_frame_size_mismatch[] = {25};
int max_encoding_rate_mismatch[] = {15};
int max_time_hit_target[] = {15};
int num_spatial_resizes[] = {0};
ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr,
minimum_avg_ssim, minimum_min_ssim);
minimum_avg_ssim, minimum_min_ssim,
target_bit_rate, input_frame_rate,
frame_index_rate_update, num_frames, process_settings,
max_frame_size_mismatch, max_encoding_rate_mismatch,
max_time_hit_target, max_num_dropped_frames,
num_spatial_resizes);
}
// Run with no packet loss, with varying bitrate (3 rate updates):
// low to high to medium. Check that quality and encoder response to the new
// target rate/per-frame bandwidth (for each rate update) is within limits.
// One key frame (first frame only) in sequence.
TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeBitRate) {
// Bitrate and frame rate profile, and codec settings.
int target_bit_rate[] = {200, 800, 500}; // kbps
int input_frame_rate[] = {30, 30, 30};
int frame_index_rate_update[] = {0, 100, 200, kNbrFramesLong + 1};
int num_frames = kNbrFramesLong;
// Codec/network settings.
CodecConfigPars process_settings;
process_settings.packet_loss = 0.0f;
process_settings.key_frame_interval = -1;
process_settings.num_temporal_layers = 1;
process_settings.error_concealment_on = true;
process_settings.denoising_on = true;
// Metrics for expected quality.
double minimum_avg_psnr = 34;
double minimum_min_psnr = 32;
double minimum_avg_ssim = 0.85;
double minimum_min_ssim = 0.8;
// Metrics for rate control: rate mismatch metrics are defined as percentages.
// |max_time_hit_target| is defined as number of frames, after a rate update
// is made to the encoder, for the encoder to reach within
// |kPercTargetvsActualMismatch| of new target rate.
int max_num_dropped_frames[] = {0, 0, 0};
int max_frame_size_mismatch[] = {20, 25, 25};
int max_encoding_rate_mismatch[] = {10, 20, 15};
int max_time_hit_target[] = {15, 10, 10};
int num_spatial_resizes[] = {0, 0, 0};
ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr,
minimum_avg_ssim, minimum_min_ssim,
target_bit_rate, input_frame_rate,
frame_index_rate_update, num_frames, process_settings,
max_frame_size_mismatch, max_encoding_rate_mismatch,
max_time_hit_target, max_num_dropped_frames,
num_spatial_resizes);
}
// Run with no packet loss, with an update (decrease) in frame rate.
// Lower frame rate means higher per-frame-bandwidth, so easier to encode.
// At the bitrate in this test, this means better rate control after the
// update(s) to lower frame rate. So expect less frame drops, and max values
// for the rate control metrics can be lower. One key frame (first frame only).
// Note: quality after update should be higher but we currently compute quality
// metrics avergaed over whole sequence run.
TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeFrameRateFrameDrop) {
config_.networking_config.packet_loss_probability = 0;
// Bitrate and frame rate profile, and codec settings.
int target_bit_rate[] = {80, 80, 80}; // kbps
int input_frame_rate[] = {30, 15, 10};
int frame_index_rate_update[] = {0, 100, 200, kNbrFramesLong + 1};
int num_frames = kNbrFramesLong;
// Codec/network settings.
CodecConfigPars process_settings;
process_settings.packet_loss = 0.0f;
process_settings.key_frame_interval = -1;
process_settings.num_temporal_layers = 1;
process_settings.error_concealment_on = true;
process_settings.denoising_on = true;
// Metrics for expected quality.
double minimum_avg_psnr = 31;
double minimum_min_psnr = 24;
double minimum_avg_ssim = 0.8;
double minimum_min_ssim = 0.7;
// Metrics for rate control: rate mismatch metrics are defined as percentages.
// |max_time_hit_target| is defined as number of frames, after a rate update
// is made to the encoder, for the encoder to reach within
// |kPercTargetvsActualMismatch| of new target rate.
int max_num_dropped_frames[] = {25, 10, 0};
int max_frame_size_mismatch[] = {60, 25, 20};
int max_encoding_rate_mismatch[] = {40, 10, 10};
// At the low per-frame bandwidth for this scene, encoder can't hit target
// rate before first update.
int max_time_hit_target[] = {100, 40, 10};
int num_spatial_resizes[] = {0, 0, 0};
ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr,
minimum_avg_ssim, minimum_min_ssim,
target_bit_rate, input_frame_rate,
frame_index_rate_update, num_frames, process_settings,
max_frame_size_mismatch, max_encoding_rate_mismatch,
max_time_hit_target, max_num_dropped_frames,
num_spatial_resizes);
}
// Run with no packet loss, at low bitrate, then increase rate somewhat.
// Key frame is thrown in every 120 frames. Can expect some frame drops after
// key frame, even at high rate. If resizer is on, expect spatial resize down
// at first key frame, and back up at second key frame. Expected values for
// quality and rate control in this test are such that the test should pass
// with resizing on or off. Error_concealment is off in this test since there
// is a memory leak with resizing and error concealment.
TEST_F(VideoProcessorIntegrationTest, ProcessNoLossSpatialResizeFrameDrop) {
config_.networking_config.packet_loss_probability = 0;
// Bitrate and frame rate profile, and codec settings.
int target_bit_rate[] = {80, 200, 200}; // kbps
int input_frame_rate[] = {30, 30, 30};
int frame_index_rate_update[] = {0, 120, 240, kNbrFramesLong + 1};
int num_frames = kNbrFramesLong;
// Codec/network settings.
CodecConfigPars process_settings;
process_settings.packet_loss = 0.0f;
process_settings.key_frame_interval = 120;
process_settings.num_temporal_layers = 1;
process_settings.error_concealment_on = false;
process_settings.denoising_on = true;
// Metrics for expected quality.: lower quality on average from up-sampling
// the down-sampled portion of the run, in case resizer is on.
double minimum_avg_psnr = 29;
double minimum_min_psnr = 20;
double minimum_avg_ssim = 0.75;
double minimum_min_ssim = 0.6;
// Metrics for rate control: rate mismatch metrics are defined as percentages.
// |max_time_hit_target| is defined as number of frames, after a rate update
// is made to the encoder, for the encoder to reach within
// |kPercTargetvsActualMismatch| of new target rate.
int max_num_dropped_frames[] = {25, 15, 0};
int max_frame_size_mismatch[] = {60, 30, 30};
int max_encoding_rate_mismatch[] = {30, 20, 15};
// At this low rate for this scene, can't hit target rate before first update.
int max_time_hit_target[] = {120, 15, 25};
int num_spatial_resizes[] = {0, 1, 1};
ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr,
minimum_avg_ssim, minimum_min_ssim,
target_bit_rate, input_frame_rate,
frame_index_rate_update, num_frames, process_settings,
max_frame_size_mismatch, max_encoding_rate_mismatch,
max_time_hit_target, max_num_dropped_frames,
num_spatial_resizes);
}
// Run with no packet loss, with 3 temporal layers, with a rate update in the
// middle of the sequence. The max values for the frame size mismatch and
// encoding rate mismatch are applied to each layer.
// No dropped frames in this test, and the denoiser is off for temporal layers.
// One key frame (first frame only) in sequence, so no spatial resizing.
TEST_F(VideoProcessorIntegrationTest, ProcessNoLossTemporalLayers) {
config_.networking_config.packet_loss_probability = 0;
// Bitrate and frame rate profile, and codec settings.
int target_bit_rate[] = {200, 400}; // kbps
int input_frame_rate[] = {30, 30};
int frame_index_rate_update[] = {0, 150, kNbrFramesLong + 1};
int num_frames = kNbrFramesLong;
// Codec/network settings.
CodecConfigPars process_settings;
process_settings.packet_loss = 0.0f;
process_settings.key_frame_interval = -1;
process_settings.num_temporal_layers = 3;
process_settings.error_concealment_on = true;
process_settings.denoising_on = false;
// Metrics for expected quality.
double minimum_avg_psnr = 33;
double minimum_min_psnr = 30;
double minimum_avg_ssim = 0.85;
double minimum_min_ssim = 0.80;
// Metrics for rate control: rate mismatch metrics are defined as percentages.
// |max_time_hit_target| is defined as number of frames, after a rate update
// is made to the encoder, for the encoder to reach within
// |kPercTargetvsActualMismatch| of new target rate.
int max_num_dropped_frames[] = {0, 0};
int max_frame_size_mismatch[] = {30, 30};
int max_encoding_rate_mismatch[] = {10, 12};
int max_time_hit_target[] = {15, 15};
int num_spatial_resizes[] = {0, 0};
ProcessFramesAndVerify(minimum_avg_psnr, minimum_min_psnr,
minimum_avg_ssim, minimum_min_ssim,
target_bit_rate, input_frame_rate,
frame_index_rate_update, num_frames, process_settings,
max_frame_size_mismatch, max_encoding_rate_mismatch,
max_time_hit_target, max_num_dropped_frames,
num_spatial_resizes);
}
} // namespace webrtc