Rename spatial/temporal index variables and fields in videoprocessor.

This fixes inconsistency in names of variables and fields which
represent spatial/temporal index of layer:
simulcast_svc_idx -> spatial_idx
spatial_layer_idx -> spatial_idx
temporal_layer_idx -> temporal_idx

Also, this adds printing of spatial/temporal index and target bitrate
to RD report.

Bug: none
Change-Id: Ic4dfdadc57a1577bb3d35d1782a152a9dbef0280
Reviewed-on: https://webrtc-review.googlesource.com/69981
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22869}
This commit is contained in:
Sergey Silkin
2018-04-15 21:06:16 +02:00
committed by Commit Bot
parent eb99d84341
commit bc20fe1221
4 changed files with 126 additions and 141 deletions

View File

@ -30,8 +30,8 @@ std::string FrameStatistics::ToString() const {
ss << "frame_number " << frame_number; ss << "frame_number " << frame_number;
ss << " decoded_width " << decoded_width; ss << " decoded_width " << decoded_width;
ss << " decoded_height " << decoded_height; ss << " decoded_height " << decoded_height;
ss << " simulcast_svc_idx " << simulcast_svc_idx; ss << " spatial_idx " << spatial_idx;
ss << " temporal_layer_idx " << temporal_layer_idx; ss << " temporal_idx " << temporal_idx;
ss << " inter_layer_predicted " << inter_layer_predicted; ss << " inter_layer_predicted " << inter_layer_predicted;
ss << " frame_type " << frame_type; ss << " frame_type " << frame_type;
ss << " length_bytes " << length_bytes; ss << " length_bytes " << length_bytes;
@ -52,8 +52,8 @@ std::string VideoStatistics::ToString(std::string prefix) const {
std::stringstream ss; std::stringstream ss;
ss << prefix << "target_bitrate_kbps: " << target_bitrate_kbps; ss << prefix << "target_bitrate_kbps: " << target_bitrate_kbps;
ss << "\n" << prefix << "input_framerate_fps: " << input_framerate_fps; ss << "\n" << prefix << "input_framerate_fps: " << input_framerate_fps;
ss << "\n" << prefix << "spatial_layer_idx: " << spatial_layer_idx; ss << "\n" << prefix << "spatial_idx: " << spatial_idx;
ss << "\n" << prefix << "temporal_layer_idx: " << temporal_layer_idx; ss << "\n" << prefix << "temporal_idx: " << temporal_idx;
ss << "\n" << prefix << "width: " << width; ss << "\n" << prefix << "width: " << width;
ss << "\n" << prefix << "height: " << height; ss << "\n" << prefix << "height: " << height;
ss << "\n" << prefix << "length_bytes: " << length_bytes; ss << "\n" << prefix << "length_bytes: " << length_bytes;
@ -124,13 +124,12 @@ std::vector<VideoStatistics> Stats::SliceAndCalcLayerVideoStatistic(
RTC_CHECK_GT(num_spatial_layers, 0); RTC_CHECK_GT(num_spatial_layers, 0);
RTC_CHECK_GT(num_temporal_layers, 0); RTC_CHECK_GT(num_temporal_layers, 0);
for (size_t spatial_layer_idx = 0; spatial_layer_idx < num_spatial_layers; for (size_t spatial_idx = 0; spatial_idx < num_spatial_layers;
++spatial_layer_idx) { ++spatial_idx) {
for (size_t temporal_layer_idx = 0; for (size_t temporal_idx = 0; temporal_idx < num_temporal_layers;
temporal_layer_idx < num_temporal_layers; ++temporal_layer_idx) { ++temporal_idx) {
VideoStatistics layer_stat = SliceAndCalcVideoStatistic( VideoStatistics layer_stat = SliceAndCalcVideoStatistic(
first_frame_num, last_frame_num, spatial_layer_idx, first_frame_num, last_frame_num, spatial_idx, temporal_idx, false);
temporal_layer_idx, false);
layer_stats.push_back(layer_stat); layer_stats.push_back(layer_stat);
} }
} }
@ -162,8 +161,8 @@ void Stats::PrintFrameStatistics() {
} }
} }
size_t Stats::Size(size_t spatial_layer_idx) { size_t Stats::Size(size_t spatial_idx) {
return layer_stats_[spatial_layer_idx].size(); return layer_stats_[spatial_idx].size();
} }
void Stats::Clear() { void Stats::Clear() {
@ -173,13 +172,13 @@ void Stats::Clear() {
FrameStatistics Stats::AggregateFrameStatistic( FrameStatistics Stats::AggregateFrameStatistic(
size_t frame_num, size_t frame_num,
size_t spatial_layer_idx, size_t spatial_idx,
bool aggregate_independent_layers) { bool aggregate_independent_layers) {
FrameStatistics frame_stat = *GetFrame(frame_num, spatial_layer_idx); FrameStatistics frame_stat = *GetFrame(frame_num, spatial_idx);
bool inter_layer_predicted = frame_stat.inter_layer_predicted; bool inter_layer_predicted = frame_stat.inter_layer_predicted;
while (spatial_layer_idx-- > 0) { while (spatial_idx-- > 0) {
if (aggregate_independent_layers || inter_layer_predicted) { if (aggregate_independent_layers || inter_layer_predicted) {
FrameStatistics* base_frame_stat = GetFrame(frame_num, spatial_layer_idx); FrameStatistics* base_frame_stat = GetFrame(frame_num, spatial_idx);
frame_stat.length_bytes += base_frame_stat->length_bytes; frame_stat.length_bytes += base_frame_stat->length_bytes;
frame_stat.target_bitrate_kbps += base_frame_stat->target_bitrate_kbps; frame_stat.target_bitrate_kbps += base_frame_stat->target_bitrate_kbps;
@ -192,8 +191,8 @@ FrameStatistics Stats::AggregateFrameStatistic(
size_t Stats::CalcLayerTargetBitrateKbps(size_t first_frame_num, size_t Stats::CalcLayerTargetBitrateKbps(size_t first_frame_num,
size_t last_frame_num, size_t last_frame_num,
size_t spatial_layer_idx, size_t spatial_idx,
size_t temporal_layer_idx, size_t temporal_idx,
bool aggregate_independent_layers) { bool aggregate_independent_layers) {
size_t target_bitrate_kbps = 0; size_t target_bitrate_kbps = 0;
@ -204,9 +203,9 @@ size_t Stats::CalcLayerTargetBitrateKbps(size_t first_frame_num,
for (size_t frame_num = first_frame_num; frame_num <= last_frame_num; for (size_t frame_num = first_frame_num; frame_num <= last_frame_num;
++frame_num) { ++frame_num) {
FrameStatistics superframe = AggregateFrameStatistic( FrameStatistics superframe = AggregateFrameStatistic(
frame_num, spatial_layer_idx, aggregate_independent_layers); frame_num, spatial_idx, aggregate_independent_layers);
if (superframe.temporal_layer_idx <= temporal_layer_idx) { if (superframe.temporal_idx <= temporal_idx) {
target_bitrate_kbps = target_bitrate_kbps =
std::max(target_bitrate_kbps, superframe.target_bitrate_kbps); std::max(target_bitrate_kbps, superframe.target_bitrate_kbps);
} }
@ -219,8 +218,8 @@ size_t Stats::CalcLayerTargetBitrateKbps(size_t first_frame_num,
VideoStatistics Stats::SliceAndCalcVideoStatistic( VideoStatistics Stats::SliceAndCalcVideoStatistic(
size_t first_frame_num, size_t first_frame_num,
size_t last_frame_num, size_t last_frame_num,
size_t spatial_layer_idx, size_t spatial_idx,
size_t temporal_layer_idx, size_t temporal_idx,
bool aggregate_independent_layers) { bool aggregate_independent_layers) {
VideoStatistics video_stat; VideoStatistics video_stat;
@ -245,14 +244,14 @@ VideoStatistics Stats::SliceAndCalcVideoStatistic(
FrameStatistics last_successfully_decoded_frame(0, 0); FrameStatistics last_successfully_decoded_frame(0, 0);
const size_t target_bitrate_kbps = CalcLayerTargetBitrateKbps( const size_t target_bitrate_kbps =
first_frame_num, last_frame_num, spatial_layer_idx, temporal_layer_idx, CalcLayerTargetBitrateKbps(first_frame_num, last_frame_num, spatial_idx,
aggregate_independent_layers); temporal_idx, aggregate_independent_layers);
for (size_t frame_num = first_frame_num; frame_num <= last_frame_num; for (size_t frame_num = first_frame_num; frame_num <= last_frame_num;
++frame_num) { ++frame_num) {
FrameStatistics frame_stat = AggregateFrameStatistic( FrameStatistics frame_stat = AggregateFrameStatistic(
frame_num, spatial_layer_idx, aggregate_independent_layers); frame_num, spatial_idx, aggregate_independent_layers);
float time_since_first_frame_sec = float time_since_first_frame_sec =
1.0f * (frame_stat.rtp_timestamp - rtp_timestamp_first_frame) / 1.0f * (frame_stat.rtp_timestamp - rtp_timestamp_first_frame) /
@ -261,7 +260,7 @@ VideoStatistics Stats::SliceAndCalcVideoStatistic(
1.0f * (frame_stat.rtp_timestamp - rtp_timestamp_prev_frame) / 1.0f * (frame_stat.rtp_timestamp - rtp_timestamp_prev_frame) /
kVideoPayloadTypeFrequency; kVideoPayloadTypeFrequency;
if (frame_stat.temporal_layer_idx > temporal_layer_idx) { if (frame_stat.temporal_idx > temporal_idx) {
continue; continue;
} }
@ -340,8 +339,8 @@ VideoStatistics Stats::SliceAndCalcVideoStatistic(
const size_t num_frames = last_frame_num - first_frame_num + 1; const size_t num_frames = last_frame_num - first_frame_num + 1;
const size_t timestamp_delta = const size_t timestamp_delta =
GetFrame(first_frame_num + 1, spatial_layer_idx)->rtp_timestamp - GetFrame(first_frame_num + 1, spatial_idx)->rtp_timestamp -
GetFrame(first_frame_num, spatial_layer_idx)->rtp_timestamp; GetFrame(first_frame_num, spatial_idx)->rtp_timestamp;
const float input_framerate_fps = const float input_framerate_fps =
1.0 * kVideoPayloadTypeFrequency / timestamp_delta; 1.0 * kVideoPayloadTypeFrequency / timestamp_delta;
const float duration_sec = num_frames / input_framerate_fps; const float duration_sec = num_frames / input_framerate_fps;
@ -349,8 +348,8 @@ VideoStatistics Stats::SliceAndCalcVideoStatistic(
video_stat.target_bitrate_kbps = target_bitrate_kbps; video_stat.target_bitrate_kbps = target_bitrate_kbps;
video_stat.input_framerate_fps = input_framerate_fps; video_stat.input_framerate_fps = input_framerate_fps;
video_stat.spatial_layer_idx = spatial_layer_idx; video_stat.spatial_idx = spatial_idx;
video_stat.temporal_layer_idx = temporal_layer_idx; video_stat.temporal_idx = temporal_idx;
video_stat.bitrate_kbps = video_stat.bitrate_kbps =
static_cast<size_t>(8 * video_stat.length_bytes / 1000 / duration_sec); static_cast<size_t>(8 * video_stat.length_bytes / 1000 / duration_sec);
@ -391,14 +390,14 @@ void Stats::GetNumberOfEncodedLayers(size_t first_frame_num,
for (size_t frame_num = first_frame_num; frame_num <= last_frame_num; for (size_t frame_num = first_frame_num; frame_num <= last_frame_num;
++frame_num) { ++frame_num) {
for (size_t spatial_layer_idx = 0; spatial_layer_idx < num_spatial_layers; for (size_t spatial_idx = 0; spatial_idx < num_spatial_layers;
++spatial_layer_idx) { ++spatial_idx) {
FrameStatistics* frame_stat = GetFrame(frame_num, spatial_layer_idx); FrameStatistics* frame_stat = GetFrame(frame_num, spatial_idx);
if (frame_stat->encoding_successful) { if (frame_stat->encoding_successful) {
*num_encoded_spatial_layers = std::max( *num_encoded_spatial_layers =
*num_encoded_spatial_layers, frame_stat->simulcast_svc_idx + 1); std::max(*num_encoded_spatial_layers, frame_stat->spatial_idx + 1);
*num_encoded_temporal_layers = std::max( *num_encoded_temporal_layers = std::max(*num_encoded_temporal_layers,
*num_encoded_temporal_layers, frame_stat->temporal_layer_idx + 1); frame_stat->temporal_idx + 1);
} }
} }
} }

View File

@ -40,8 +40,8 @@ struct FrameStatistics {
webrtc::FrameType frame_type = kVideoFrameDelta; webrtc::FrameType frame_type = kVideoFrameDelta;
// Layering. // Layering.
size_t temporal_layer_idx = 0; size_t spatial_idx = 0;
size_t simulcast_svc_idx = 0; size_t temporal_idx = 0;
bool inter_layer_predicted = false; bool inter_layer_predicted = false;
// H264 specific. // H264 specific.
@ -72,8 +72,8 @@ struct VideoStatistics {
size_t target_bitrate_kbps = 0; size_t target_bitrate_kbps = 0;
float input_framerate_fps = 0.0f; float input_framerate_fps = 0.0f;
size_t spatial_layer_idx = 0; size_t spatial_idx = 0;
size_t temporal_layer_idx = 0; size_t temporal_idx = 0;
size_t width = 0; size_t width = 0;
size_t height = 0; size_t height = 0;
@ -117,12 +117,11 @@ class Stats {
~Stats() = default; ~Stats() = default;
// Creates a FrameStatistics for the next frame to be processed. // Creates a FrameStatistics for the next frame to be processed.
FrameStatistics* AddFrame(size_t timestamp, size_t spatial_layer_idx); FrameStatistics* AddFrame(size_t timestamp, size_t spatial_idx);
// Returns the FrameStatistics corresponding to |frame_number| or |timestamp|. // Returns the FrameStatistics corresponding to |frame_number| or |timestamp|.
FrameStatistics* GetFrame(size_t frame_number, size_t spatial_layer_idx); FrameStatistics* GetFrame(size_t frame_number, size_t spatial_idx);
FrameStatistics* GetFrameWithTimestamp(size_t timestamp, FrameStatistics* GetFrameWithTimestamp(size_t timestamp, size_t spatial_idx);
size_t spatial_layer_idx);
std::vector<VideoStatistics> SliceAndCalcLayerVideoStatistic( std::vector<VideoStatistics> SliceAndCalcLayerVideoStatistic(
size_t first_frame_num, size_t first_frame_num,
@ -133,25 +132,25 @@ class Stats {
void PrintFrameStatistics(); void PrintFrameStatistics();
size_t Size(size_t spatial_layer_idx); size_t Size(size_t spatial_idx);
void Clear(); void Clear();
private: private:
FrameStatistics AggregateFrameStatistic(size_t frame_num, FrameStatistics AggregateFrameStatistic(size_t frame_num,
size_t spatial_layer_idx, size_t spatial_idx,
bool aggregate_independent_layers); bool aggregate_independent_layers);
size_t CalcLayerTargetBitrateKbps(size_t first_frame_num, size_t CalcLayerTargetBitrateKbps(size_t first_frame_num,
size_t last_frame_num, size_t last_frame_num,
size_t spatial_layer_idx, size_t spatial_idx,
size_t temporal_layer_idx, size_t temporal_idx,
bool aggregate_independent_layers); bool aggregate_independent_layers);
VideoStatistics SliceAndCalcVideoStatistic(size_t first_frame_num, VideoStatistics SliceAndCalcVideoStatistic(size_t first_frame_num,
size_t last_frame_num, size_t last_frame_num,
size_t spatial_layer_idx, size_t spatial_idx,
size_t temporal_layer_idx, size_t temporal_idx,
bool aggregate_independent_layers); bool aggregate_independent_layers);
void GetNumberOfEncodedLayers(size_t first_frame_num, void GetNumberOfEncodedLayers(size_t first_frame_num,

View File

@ -56,17 +56,17 @@ size_t GetMaxNaluSizeBytes(const EncodedImage& encoded_frame,
} }
void GetLayerIndices(const CodecSpecificInfo& codec_specific, void GetLayerIndices(const CodecSpecificInfo& codec_specific,
size_t* simulcast_svc_idx, size_t* spatial_idx,
size_t* temporal_idx) { size_t* temporal_idx) {
if (codec_specific.codecType == kVideoCodecVP8) { if (codec_specific.codecType == kVideoCodecVP8) {
*simulcast_svc_idx = codec_specific.codecSpecific.VP8.simulcastIdx; *spatial_idx = codec_specific.codecSpecific.VP8.simulcastIdx;
*temporal_idx = codec_specific.codecSpecific.VP8.temporalIdx; *temporal_idx = codec_specific.codecSpecific.VP8.temporalIdx;
} else if (codec_specific.codecType == kVideoCodecVP9) { } else if (codec_specific.codecType == kVideoCodecVP9) {
*simulcast_svc_idx = codec_specific.codecSpecific.VP9.spatial_idx; *spatial_idx = codec_specific.codecSpecific.VP9.spatial_idx;
*temporal_idx = codec_specific.codecSpecific.VP9.temporal_idx; *temporal_idx = codec_specific.codecSpecific.VP9.temporal_idx;
} }
if (*simulcast_svc_idx == kNoSpatialIdx) { if (*spatial_idx == kNoSpatialIdx) {
*simulcast_svc_idx = 0; *spatial_idx = 0;
} }
if (*temporal_idx == kNoTemporalIdx) { if (*temporal_idx == kNoTemporalIdx) {
*temporal_idx = 0; *temporal_idx = 0;
@ -205,19 +205,15 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder,
config_.max_payload_size_bytes), config_.max_payload_size_bytes),
WEBRTC_VIDEO_CODEC_OK); WEBRTC_VIDEO_CODEC_OK);
for (size_t simulcast_svc_idx = 0; for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
simulcast_svc_idx < num_simulcast_or_spatial_layers_;
++simulcast_svc_idx) {
decode_callback_.push_back( decode_callback_.push_back(
rtc::MakeUnique<VideoProcessorDecodeCompleteCallback>( rtc::MakeUnique<VideoProcessorDecodeCompleteCallback>(this, i));
this, simulcast_svc_idx)); RTC_CHECK_EQ(
RTC_CHECK_EQ(decoders_->at(simulcast_svc_idx) decoders_->at(i)->InitDecode(&config_.codec_settings,
->InitDecode(&config_.codec_settings, static_cast<int>(config_.NumberOfCores())),
static_cast<int>(config_.NumberOfCores())), WEBRTC_VIDEO_CODEC_OK);
WEBRTC_VIDEO_CODEC_OK); RTC_CHECK_EQ(decoders_->at(i)->RegisterDecodeCompleteCallback(
RTC_CHECK_EQ(decoders_->at(simulcast_svc_idx) decode_callback_.at(i).get()),
->RegisterDecodeCompleteCallback(
decode_callback_.at(simulcast_svc_idx).get()),
WEBRTC_VIDEO_CODEC_OK); WEBRTC_VIDEO_CODEC_OK);
} }
} }
@ -238,10 +234,8 @@ VideoProcessor::~VideoProcessor() {
RTC_CHECK_LE(input_frames_.size(), kMaxBufferedInputFrames); RTC_CHECK_LE(input_frames_.size(), kMaxBufferedInputFrames);
// Deal with manual memory management of EncodedImage's. // Deal with manual memory management of EncodedImage's.
for (size_t simulcast_svc_idx = 0; for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
simulcast_svc_idx < num_simulcast_or_spatial_layers_; uint8_t* buffer = merged_encoded_frames_.at(i)._buffer;
++simulcast_svc_idx) {
uint8_t* buffer = merged_encoded_frames_.at(simulcast_svc_idx)._buffer;
if (buffer) { if (buffer) {
delete[] buffer; delete[] buffer;
} }
@ -270,20 +264,15 @@ void VideoProcessor::ProcessFrame() {
post_encode_time_ns_ = 0; post_encode_time_ns_ = 0;
// Create frame statistics object for all simulcast/spatial layers. // Create frame statistics object for all simulcast/spatial layers.
for (size_t simulcast_svc_idx = 0; for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
simulcast_svc_idx < num_simulcast_or_spatial_layers_; stats_->AddFrame(timestamp, i);
++simulcast_svc_idx) {
stats_->AddFrame(timestamp, simulcast_svc_idx);
} }
// For the highest measurement accuracy of the encode time, the start/stop // For the highest measurement accuracy of the encode time, the start/stop
// time recordings should wrap the Encode call as tightly as possible. // time recordings should wrap the Encode call as tightly as possible.
const int64_t encode_start_ns = rtc::TimeNanos(); const int64_t encode_start_ns = rtc::TimeNanos();
for (size_t simulcast_svc_idx = 0; for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
simulcast_svc_idx < num_simulcast_or_spatial_layers_; FrameStatistics* frame_stat = stats_->GetFrame(frame_number, i);
++simulcast_svc_idx) {
FrameStatistics* frame_stat =
stats_->GetFrame(frame_number, simulcast_svc_idx);
frame_stat->encode_start_ns = encode_start_ns; frame_stat->encode_start_ns = encode_start_ns;
} }
@ -292,11 +281,8 @@ void VideoProcessor::ProcessFrame() {
config_.FrameTypeForFrame(frame_number); config_.FrameTypeForFrame(frame_number);
const int encode_return_code = const int encode_return_code =
encoder_->Encode(input_frame, nullptr, &frame_types); encoder_->Encode(input_frame, nullptr, &frame_types);
for (size_t simulcast_svc_idx = 0; for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
simulcast_svc_idx < num_simulcast_or_spatial_layers_; FrameStatistics* frame_stat = stats_->GetFrame(frame_number, i);
++simulcast_svc_idx) {
FrameStatistics* frame_stat =
stats_->GetFrame(frame_number, simulcast_svc_idx);
frame_stat->encode_return_code = encode_return_code; frame_stat->encode_return_code = encode_return_code;
} }
} }
@ -327,45 +313,45 @@ void VideoProcessor::FrameEncoded(
} }
// Layer metadata. // Layer metadata.
size_t simulcast_svc_idx = 0; size_t spatial_idx = 0;
size_t temporal_idx = 0; size_t temporal_idx = 0;
GetLayerIndices(codec_specific, &simulcast_svc_idx, &temporal_idx); GetLayerIndices(codec_specific, &spatial_idx, &temporal_idx);
FrameStatistics* frame_stat = stats_->GetFrameWithTimestamp( FrameStatistics* frame_stat =
encoded_image._timeStamp, simulcast_svc_idx); stats_->GetFrameWithTimestamp(encoded_image._timeStamp, spatial_idx);
const size_t frame_number = frame_stat->frame_number; const size_t frame_number = frame_stat->frame_number;
// Ensure that the encode order is monotonically increasing, within this // Ensure that the encode order is monotonically increasing, within this
// simulcast/spatial layer. // simulcast/spatial layer.
RTC_CHECK(first_encoded_frame_[simulcast_svc_idx] || RTC_CHECK(first_encoded_frame_[spatial_idx] ||
last_encoded_frame_num_[simulcast_svc_idx] < frame_number); last_encoded_frame_num_[spatial_idx] < frame_number);
// Ensure SVC spatial layers are delivered in ascending order. // Ensure SVC spatial layers are delivered in ascending order.
if (!first_encoded_frame_[simulcast_svc_idx] && if (!first_encoded_frame_[spatial_idx] &&
config_.NumberOfSpatialLayers() > 1) { config_.NumberOfSpatialLayers() > 1) {
for (size_t i = 0; i < simulcast_svc_idx; ++i) { for (size_t i = 0; i < spatial_idx; ++i) {
RTC_CHECK_LE(last_encoded_frame_num_[i], frame_number); RTC_CHECK_LE(last_encoded_frame_num_[i], frame_number);
} }
for (size_t i = simulcast_svc_idx + 1; i < num_simulcast_or_spatial_layers_; for (size_t i = spatial_idx + 1; i < num_simulcast_or_spatial_layers_;
++i) { ++i) {
RTC_CHECK_GT(frame_number, last_encoded_frame_num_[i]); RTC_CHECK_GT(frame_number, last_encoded_frame_num_[i]);
} }
} }
first_encoded_frame_[simulcast_svc_idx] = false; first_encoded_frame_[spatial_idx] = false;
last_encoded_frame_num_[simulcast_svc_idx] = frame_number; last_encoded_frame_num_[spatial_idx] = frame_number;
// Update frame statistics. // Update frame statistics.
frame_stat->encoding_successful = true; frame_stat->encoding_successful = true;
frame_stat->encode_time_us = GetElapsedTimeMicroseconds( frame_stat->encode_time_us = GetElapsedTimeMicroseconds(
frame_stat->encode_start_ns, encode_stop_ns - post_encode_time_ns_); frame_stat->encode_start_ns, encode_stop_ns - post_encode_time_ns_);
frame_stat->target_bitrate_kbps = (bitrate_allocation_.GetTemporalLayerSum( frame_stat->target_bitrate_kbps =
simulcast_svc_idx, temporal_idx) + (bitrate_allocation_.GetTemporalLayerSum(spatial_idx, temporal_idx) +
500) / 500) /
1000; 1000;
frame_stat->length_bytes = encoded_image._length; frame_stat->length_bytes = encoded_image._length;
frame_stat->frame_type = encoded_image._frameType; frame_stat->frame_type = encoded_image._frameType;
frame_stat->temporal_layer_idx = temporal_idx; frame_stat->temporal_idx = temporal_idx;
frame_stat->simulcast_svc_idx = simulcast_svc_idx; frame_stat->spatial_idx = spatial_idx;
frame_stat->max_nalu_size_bytes = GetMaxNaluSizeBytes(encoded_image, config_); frame_stat->max_nalu_size_bytes = GetMaxNaluSizeBytes(encoded_image, config_);
frame_stat->qp = encoded_image.qp_; frame_stat->qp = encoded_image.qp_;
@ -384,31 +370,29 @@ void VideoProcessor::FrameEncoded(
if (config_.decode || encoded_frame_writers_) { if (config_.decode || encoded_frame_writers_) {
if (num_spatial_layers > 1) { if (num_spatial_layers > 1) {
encoded_image_for_decode = BuildAndStoreSuperframe( encoded_image_for_decode = BuildAndStoreSuperframe(
encoded_image, codec_type, frame_number, simulcast_svc_idx, encoded_image, codec_type, frame_number, spatial_idx,
frame_stat->inter_layer_predicted); frame_stat->inter_layer_predicted);
} }
} }
if (config_.decode) { if (config_.decode) {
DecodeFrame(*encoded_image_for_decode, simulcast_svc_idx); DecodeFrame(*encoded_image_for_decode, spatial_idx);
if (end_of_superframe && inter_layer_prediction) { if (end_of_superframe && inter_layer_prediction) {
// If inter-layer prediction is enabled and upper layer was dropped then // If inter-layer prediction is enabled and upper layer was dropped then
// base layer should be passed to upper layer decoder. Otherwise decoder // base layer should be passed to upper layer decoder. Otherwise decoder
// won't be able to decode next superframe. // won't be able to decode next superframe.
const EncodedImage* base_image = nullptr; const EncodedImage* base_image = nullptr;
for (size_t spatial_idx = 0; spatial_idx < num_spatial_layers; for (size_t i = 0; i < num_spatial_layers; ++i) {
++spatial_idx) { const bool layer_dropped = last_decoded_frame_num_[i] < frame_number;
const bool layer_dropped =
last_decoded_frame_num_[spatial_idx] < frame_number;
// Ensure current layer was decoded. // Ensure current layer was decoded.
RTC_CHECK(layer_dropped == false || spatial_idx != simulcast_svc_idx); RTC_CHECK(layer_dropped == false || i != spatial_idx);
if (!layer_dropped) { if (!layer_dropped) {
base_image = &merged_encoded_frames_[spatial_idx]; base_image = &merged_encoded_frames_[i];
} else if (base_image) { } else if (base_image) {
DecodeFrame(*base_image, spatial_idx); DecodeFrame(*base_image, i);
} }
} }
} }
@ -417,7 +401,7 @@ void VideoProcessor::FrameEncoded(
} }
if (encoded_frame_writers_) { if (encoded_frame_writers_) {
RTC_CHECK(encoded_frame_writers_->at(simulcast_svc_idx) RTC_CHECK(encoded_frame_writers_->at(spatial_idx)
->WriteFrame(*encoded_image_for_decode, ->WriteFrame(*encoded_image_for_decode,
config_.codec_settings.codecType)); config_.codec_settings.codecType));
} }
@ -430,23 +414,23 @@ void VideoProcessor::FrameEncoded(
} }
void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame, void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame,
size_t simulcast_svc_idx) { size_t spatial_idx) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
// For the highest measurement accuracy of the decode time, the start/stop // For the highest measurement accuracy of the decode time, the start/stop
// time recordings should wrap the Decode call as tightly as possible. // time recordings should wrap the Decode call as tightly as possible.
const int64_t decode_stop_ns = rtc::TimeNanos(); const int64_t decode_stop_ns = rtc::TimeNanos();
FrameStatistics* frame_stat = stats_->GetFrameWithTimestamp( FrameStatistics* frame_stat =
decoded_frame.timestamp(), simulcast_svc_idx); stats_->GetFrameWithTimestamp(decoded_frame.timestamp(), spatial_idx);
const size_t frame_number = frame_stat->frame_number; const size_t frame_number = frame_stat->frame_number;
// Ensure that the decode order is monotonically increasing, within this // Ensure that the decode order is monotonically increasing, within this
// simulcast/spatial layer. // simulcast/spatial layer.
RTC_CHECK(first_decoded_frame_[simulcast_svc_idx] || RTC_CHECK(first_decoded_frame_[spatial_idx] ||
last_decoded_frame_num_[simulcast_svc_idx] < frame_number); last_decoded_frame_num_[spatial_idx] < frame_number);
first_decoded_frame_[simulcast_svc_idx] = false; first_decoded_frame_[spatial_idx] = false;
last_decoded_frame_num_[simulcast_svc_idx] = frame_number; last_decoded_frame_num_[spatial_idx] = frame_number;
// Update frame statistics. // Update frame statistics.
frame_stat->decoding_successful = true; frame_stat->decoding_successful = true;
@ -483,28 +467,28 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame,
ExtractI420BufferWithSize(decoded_frame, config_.codec_settings.width, ExtractI420BufferWithSize(decoded_frame, config_.codec_settings.width,
config_.codec_settings.height, &tmp_i420_buffer_); config_.codec_settings.height, &tmp_i420_buffer_);
RTC_CHECK_EQ(tmp_i420_buffer_.size(), RTC_CHECK_EQ(tmp_i420_buffer_.size(),
decoded_frame_writers_->at(simulcast_svc_idx)->FrameLength()); decoded_frame_writers_->at(spatial_idx)->FrameLength());
RTC_CHECK(decoded_frame_writers_->at(simulcast_svc_idx) RTC_CHECK(decoded_frame_writers_->at(spatial_idx)
->WriteFrame(tmp_i420_buffer_.data())); ->WriteFrame(tmp_i420_buffer_.data()));
} }
} }
void VideoProcessor::DecodeFrame(const EncodedImage& encoded_image, void VideoProcessor::DecodeFrame(const EncodedImage& encoded_image,
size_t simulcast_svc_idx) { size_t spatial_idx) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
FrameStatistics* frame_stat = stats_->GetFrameWithTimestamp( FrameStatistics* frame_stat =
encoded_image._timeStamp, simulcast_svc_idx); stats_->GetFrameWithTimestamp(encoded_image._timeStamp, spatial_idx);
frame_stat->decode_start_ns = rtc::TimeNanos(); frame_stat->decode_start_ns = rtc::TimeNanos();
frame_stat->decode_return_code = frame_stat->decode_return_code =
decoders_->at(simulcast_svc_idx)->Decode(encoded_image, false, nullptr); decoders_->at(spatial_idx)->Decode(encoded_image, false, nullptr);
} }
const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe( const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe(
const EncodedImage& encoded_image, const EncodedImage& encoded_image,
const VideoCodecType codec, const VideoCodecType codec,
size_t frame_number, size_t frame_number,
size_t simulcast_svc_idx, size_t spatial_idx,
bool inter_layer_predicted) { bool inter_layer_predicted) {
// Should only be called for SVC. // Should only be called for SVC.
RTC_CHECK_GT(config_.NumberOfSpatialLayers(), 1); RTC_CHECK_GT(config_.NumberOfSpatialLayers(), 1);
@ -515,7 +499,7 @@ const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe(
// Each SVC layer is decoded with dedicated decoder. Find the nearest // Each SVC layer is decoded with dedicated decoder. Find the nearest
// non-dropped base frame and merge it and current frame into superframe. // non-dropped base frame and merge it and current frame into superframe.
if (inter_layer_predicted) { if (inter_layer_predicted) {
for (int base_idx = static_cast<int>(simulcast_svc_idx) - 1; base_idx >= 0; for (int base_idx = static_cast<int>(spatial_idx) - 1; base_idx >= 0;
--base_idx) { --base_idx) {
EncodedImage lower_layer = merged_encoded_frames_.at(base_idx); EncodedImage lower_layer = merged_encoded_frames_.at(base_idx);
if (lower_layer._timeStamp == encoded_image._timeStamp) { if (lower_layer._timeStamp == encoded_image._timeStamp) {
@ -545,13 +529,13 @@ const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe(
copied_image._size = buffer_size_bytes; copied_image._size = buffer_size_bytes;
// Replace previous EncodedImage for this spatial layer. // Replace previous EncodedImage for this spatial layer.
uint8_t* old_buffer = merged_encoded_frames_.at(simulcast_svc_idx)._buffer; uint8_t* old_buffer = merged_encoded_frames_.at(spatial_idx)._buffer;
if (old_buffer) { if (old_buffer) {
delete[] old_buffer; delete[] old_buffer;
} }
merged_encoded_frames_.at(simulcast_svc_idx) = copied_image; merged_encoded_frames_.at(spatial_idx) = copied_image;
return &merged_encoded_frames_.at(simulcast_svc_idx); return &merged_encoded_frames_.at(spatial_idx);
} }
} // namespace test } // namespace test

View File

@ -53,20 +53,23 @@ class VideoProcessorIntegrationTestLibvpx
void PrintRdPerf(std::map<size_t, std::vector<VideoStatistics>> rd_stats) { void PrintRdPerf(std::map<size_t, std::vector<VideoStatistics>> rd_stats) {
printf("--> Summary\n"); printf("--> Summary\n");
printf("%11s %5s %6s %13s %13s %5s %7s %7s %7s %13s %13s\n", "uplink_kbps", printf("%11s %5s %6s %11s %12s %11s %13s %13s %5s %7s %7s %7s %13s %13s\n",
"width", "height", "downlink_kbps", "framerate_fps", "psnr", "uplink_kbps", "width", "height", "spatial_idx", "temporal_idx",
"psnr_y", "psnr_u", "psnr_v", "enc_speed_fps", "dec_speed_fps"); "target_kbps", "downlink_kbps", "framerate_fps", "psnr", "psnr_y",
"psnr_u", "psnr_v", "enc_speed_fps", "dec_speed_fps");
for (const auto& rd_stat : rd_stats) { for (const auto& rd_stat : rd_stats) {
const size_t bitrate_kbps = rd_stat.first; const size_t bitrate_kbps = rd_stat.first;
for (const auto& layer_stat : rd_stat.second) { for (const auto& layer_stat : rd_stat.second) {
printf( printf(
"%11zu %5zu %6zu %13zu %13.2f %5.2f %7.2f %7.2f %7.2f %13.2f " "%11zu %5zu %6zu %11zu %12zu %11zu %13zu %13.2f %5.2f %7.2f %7.2f "
"%13.2f\n", "%7.2f"
"%13.2f %13.2f\n",
bitrate_kbps, layer_stat.width, layer_stat.height, bitrate_kbps, layer_stat.width, layer_stat.height,
layer_stat.bitrate_kbps, layer_stat.framerate_fps, layer_stat.spatial_idx, layer_stat.temporal_idx,
layer_stat.avg_psnr, layer_stat.avg_psnr_y, layer_stat.avg_psnr_u, layer_stat.target_bitrate_kbps, layer_stat.bitrate_kbps,
layer_stat.avg_psnr_v, layer_stat.enc_speed_fps, layer_stat.framerate_fps, layer_stat.avg_psnr,
layer_stat.dec_speed_fps); layer_stat.avg_psnr_y, layer_stat.avg_psnr_u, layer_stat.avg_psnr_v,
layer_stat.enc_speed_fps, layer_stat.dec_speed_fps);
} }
} }
} }