Rename spatial/temporal index variables and fields in videoprocessor.
This fixes inconsistency in names of variables and fields which represent spatial/temporal index of layer: simulcast_svc_idx -> spatial_idx spatial_layer_idx -> spatial_idx temporal_layer_idx -> temporal_idx Also, this adds printing of spatial/temporal index and target bitrate to RD report. Bug: none Change-Id: Ic4dfdadc57a1577bb3d35d1782a152a9dbef0280 Reviewed-on: https://webrtc-review.googlesource.com/69981 Reviewed-by: Rasmus Brandt <brandtr@webrtc.org> Commit-Queue: Sergey Silkin <ssilkin@webrtc.org> Cr-Commit-Position: refs/heads/master@{#22869}
This commit is contained in:

committed by
Commit Bot

parent
eb99d84341
commit
bc20fe1221
@ -30,8 +30,8 @@ std::string FrameStatistics::ToString() const {
|
|||||||
ss << "frame_number " << frame_number;
|
ss << "frame_number " << frame_number;
|
||||||
ss << " decoded_width " << decoded_width;
|
ss << " decoded_width " << decoded_width;
|
||||||
ss << " decoded_height " << decoded_height;
|
ss << " decoded_height " << decoded_height;
|
||||||
ss << " simulcast_svc_idx " << simulcast_svc_idx;
|
ss << " spatial_idx " << spatial_idx;
|
||||||
ss << " temporal_layer_idx " << temporal_layer_idx;
|
ss << " temporal_idx " << temporal_idx;
|
||||||
ss << " inter_layer_predicted " << inter_layer_predicted;
|
ss << " inter_layer_predicted " << inter_layer_predicted;
|
||||||
ss << " frame_type " << frame_type;
|
ss << " frame_type " << frame_type;
|
||||||
ss << " length_bytes " << length_bytes;
|
ss << " length_bytes " << length_bytes;
|
||||||
@ -52,8 +52,8 @@ std::string VideoStatistics::ToString(std::string prefix) const {
|
|||||||
std::stringstream ss;
|
std::stringstream ss;
|
||||||
ss << prefix << "target_bitrate_kbps: " << target_bitrate_kbps;
|
ss << prefix << "target_bitrate_kbps: " << target_bitrate_kbps;
|
||||||
ss << "\n" << prefix << "input_framerate_fps: " << input_framerate_fps;
|
ss << "\n" << prefix << "input_framerate_fps: " << input_framerate_fps;
|
||||||
ss << "\n" << prefix << "spatial_layer_idx: " << spatial_layer_idx;
|
ss << "\n" << prefix << "spatial_idx: " << spatial_idx;
|
||||||
ss << "\n" << prefix << "temporal_layer_idx: " << temporal_layer_idx;
|
ss << "\n" << prefix << "temporal_idx: " << temporal_idx;
|
||||||
ss << "\n" << prefix << "width: " << width;
|
ss << "\n" << prefix << "width: " << width;
|
||||||
ss << "\n" << prefix << "height: " << height;
|
ss << "\n" << prefix << "height: " << height;
|
||||||
ss << "\n" << prefix << "length_bytes: " << length_bytes;
|
ss << "\n" << prefix << "length_bytes: " << length_bytes;
|
||||||
@ -124,13 +124,12 @@ std::vector<VideoStatistics> Stats::SliceAndCalcLayerVideoStatistic(
|
|||||||
RTC_CHECK_GT(num_spatial_layers, 0);
|
RTC_CHECK_GT(num_spatial_layers, 0);
|
||||||
RTC_CHECK_GT(num_temporal_layers, 0);
|
RTC_CHECK_GT(num_temporal_layers, 0);
|
||||||
|
|
||||||
for (size_t spatial_layer_idx = 0; spatial_layer_idx < num_spatial_layers;
|
for (size_t spatial_idx = 0; spatial_idx < num_spatial_layers;
|
||||||
++spatial_layer_idx) {
|
++spatial_idx) {
|
||||||
for (size_t temporal_layer_idx = 0;
|
for (size_t temporal_idx = 0; temporal_idx < num_temporal_layers;
|
||||||
temporal_layer_idx < num_temporal_layers; ++temporal_layer_idx) {
|
++temporal_idx) {
|
||||||
VideoStatistics layer_stat = SliceAndCalcVideoStatistic(
|
VideoStatistics layer_stat = SliceAndCalcVideoStatistic(
|
||||||
first_frame_num, last_frame_num, spatial_layer_idx,
|
first_frame_num, last_frame_num, spatial_idx, temporal_idx, false);
|
||||||
temporal_layer_idx, false);
|
|
||||||
layer_stats.push_back(layer_stat);
|
layer_stats.push_back(layer_stat);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -162,8 +161,8 @@ void Stats::PrintFrameStatistics() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t Stats::Size(size_t spatial_layer_idx) {
|
size_t Stats::Size(size_t spatial_idx) {
|
||||||
return layer_stats_[spatial_layer_idx].size();
|
return layer_stats_[spatial_idx].size();
|
||||||
}
|
}
|
||||||
|
|
||||||
void Stats::Clear() {
|
void Stats::Clear() {
|
||||||
@ -173,13 +172,13 @@ void Stats::Clear() {
|
|||||||
|
|
||||||
FrameStatistics Stats::AggregateFrameStatistic(
|
FrameStatistics Stats::AggregateFrameStatistic(
|
||||||
size_t frame_num,
|
size_t frame_num,
|
||||||
size_t spatial_layer_idx,
|
size_t spatial_idx,
|
||||||
bool aggregate_independent_layers) {
|
bool aggregate_independent_layers) {
|
||||||
FrameStatistics frame_stat = *GetFrame(frame_num, spatial_layer_idx);
|
FrameStatistics frame_stat = *GetFrame(frame_num, spatial_idx);
|
||||||
bool inter_layer_predicted = frame_stat.inter_layer_predicted;
|
bool inter_layer_predicted = frame_stat.inter_layer_predicted;
|
||||||
while (spatial_layer_idx-- > 0) {
|
while (spatial_idx-- > 0) {
|
||||||
if (aggregate_independent_layers || inter_layer_predicted) {
|
if (aggregate_independent_layers || inter_layer_predicted) {
|
||||||
FrameStatistics* base_frame_stat = GetFrame(frame_num, spatial_layer_idx);
|
FrameStatistics* base_frame_stat = GetFrame(frame_num, spatial_idx);
|
||||||
frame_stat.length_bytes += base_frame_stat->length_bytes;
|
frame_stat.length_bytes += base_frame_stat->length_bytes;
|
||||||
frame_stat.target_bitrate_kbps += base_frame_stat->target_bitrate_kbps;
|
frame_stat.target_bitrate_kbps += base_frame_stat->target_bitrate_kbps;
|
||||||
|
|
||||||
@ -192,8 +191,8 @@ FrameStatistics Stats::AggregateFrameStatistic(
|
|||||||
|
|
||||||
size_t Stats::CalcLayerTargetBitrateKbps(size_t first_frame_num,
|
size_t Stats::CalcLayerTargetBitrateKbps(size_t first_frame_num,
|
||||||
size_t last_frame_num,
|
size_t last_frame_num,
|
||||||
size_t spatial_layer_idx,
|
size_t spatial_idx,
|
||||||
size_t temporal_layer_idx,
|
size_t temporal_idx,
|
||||||
bool aggregate_independent_layers) {
|
bool aggregate_independent_layers) {
|
||||||
size_t target_bitrate_kbps = 0;
|
size_t target_bitrate_kbps = 0;
|
||||||
|
|
||||||
@ -204,9 +203,9 @@ size_t Stats::CalcLayerTargetBitrateKbps(size_t first_frame_num,
|
|||||||
for (size_t frame_num = first_frame_num; frame_num <= last_frame_num;
|
for (size_t frame_num = first_frame_num; frame_num <= last_frame_num;
|
||||||
++frame_num) {
|
++frame_num) {
|
||||||
FrameStatistics superframe = AggregateFrameStatistic(
|
FrameStatistics superframe = AggregateFrameStatistic(
|
||||||
frame_num, spatial_layer_idx, aggregate_independent_layers);
|
frame_num, spatial_idx, aggregate_independent_layers);
|
||||||
|
|
||||||
if (superframe.temporal_layer_idx <= temporal_layer_idx) {
|
if (superframe.temporal_idx <= temporal_idx) {
|
||||||
target_bitrate_kbps =
|
target_bitrate_kbps =
|
||||||
std::max(target_bitrate_kbps, superframe.target_bitrate_kbps);
|
std::max(target_bitrate_kbps, superframe.target_bitrate_kbps);
|
||||||
}
|
}
|
||||||
@ -219,8 +218,8 @@ size_t Stats::CalcLayerTargetBitrateKbps(size_t first_frame_num,
|
|||||||
VideoStatistics Stats::SliceAndCalcVideoStatistic(
|
VideoStatistics Stats::SliceAndCalcVideoStatistic(
|
||||||
size_t first_frame_num,
|
size_t first_frame_num,
|
||||||
size_t last_frame_num,
|
size_t last_frame_num,
|
||||||
size_t spatial_layer_idx,
|
size_t spatial_idx,
|
||||||
size_t temporal_layer_idx,
|
size_t temporal_idx,
|
||||||
bool aggregate_independent_layers) {
|
bool aggregate_independent_layers) {
|
||||||
VideoStatistics video_stat;
|
VideoStatistics video_stat;
|
||||||
|
|
||||||
@ -245,14 +244,14 @@ VideoStatistics Stats::SliceAndCalcVideoStatistic(
|
|||||||
|
|
||||||
FrameStatistics last_successfully_decoded_frame(0, 0);
|
FrameStatistics last_successfully_decoded_frame(0, 0);
|
||||||
|
|
||||||
const size_t target_bitrate_kbps = CalcLayerTargetBitrateKbps(
|
const size_t target_bitrate_kbps =
|
||||||
first_frame_num, last_frame_num, spatial_layer_idx, temporal_layer_idx,
|
CalcLayerTargetBitrateKbps(first_frame_num, last_frame_num, spatial_idx,
|
||||||
aggregate_independent_layers);
|
temporal_idx, aggregate_independent_layers);
|
||||||
|
|
||||||
for (size_t frame_num = first_frame_num; frame_num <= last_frame_num;
|
for (size_t frame_num = first_frame_num; frame_num <= last_frame_num;
|
||||||
++frame_num) {
|
++frame_num) {
|
||||||
FrameStatistics frame_stat = AggregateFrameStatistic(
|
FrameStatistics frame_stat = AggregateFrameStatistic(
|
||||||
frame_num, spatial_layer_idx, aggregate_independent_layers);
|
frame_num, spatial_idx, aggregate_independent_layers);
|
||||||
|
|
||||||
float time_since_first_frame_sec =
|
float time_since_first_frame_sec =
|
||||||
1.0f * (frame_stat.rtp_timestamp - rtp_timestamp_first_frame) /
|
1.0f * (frame_stat.rtp_timestamp - rtp_timestamp_first_frame) /
|
||||||
@ -261,7 +260,7 @@ VideoStatistics Stats::SliceAndCalcVideoStatistic(
|
|||||||
1.0f * (frame_stat.rtp_timestamp - rtp_timestamp_prev_frame) /
|
1.0f * (frame_stat.rtp_timestamp - rtp_timestamp_prev_frame) /
|
||||||
kVideoPayloadTypeFrequency;
|
kVideoPayloadTypeFrequency;
|
||||||
|
|
||||||
if (frame_stat.temporal_layer_idx > temporal_layer_idx) {
|
if (frame_stat.temporal_idx > temporal_idx) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -340,8 +339,8 @@ VideoStatistics Stats::SliceAndCalcVideoStatistic(
|
|||||||
|
|
||||||
const size_t num_frames = last_frame_num - first_frame_num + 1;
|
const size_t num_frames = last_frame_num - first_frame_num + 1;
|
||||||
const size_t timestamp_delta =
|
const size_t timestamp_delta =
|
||||||
GetFrame(first_frame_num + 1, spatial_layer_idx)->rtp_timestamp -
|
GetFrame(first_frame_num + 1, spatial_idx)->rtp_timestamp -
|
||||||
GetFrame(first_frame_num, spatial_layer_idx)->rtp_timestamp;
|
GetFrame(first_frame_num, spatial_idx)->rtp_timestamp;
|
||||||
const float input_framerate_fps =
|
const float input_framerate_fps =
|
||||||
1.0 * kVideoPayloadTypeFrequency / timestamp_delta;
|
1.0 * kVideoPayloadTypeFrequency / timestamp_delta;
|
||||||
const float duration_sec = num_frames / input_framerate_fps;
|
const float duration_sec = num_frames / input_framerate_fps;
|
||||||
@ -349,8 +348,8 @@ VideoStatistics Stats::SliceAndCalcVideoStatistic(
|
|||||||
video_stat.target_bitrate_kbps = target_bitrate_kbps;
|
video_stat.target_bitrate_kbps = target_bitrate_kbps;
|
||||||
video_stat.input_framerate_fps = input_framerate_fps;
|
video_stat.input_framerate_fps = input_framerate_fps;
|
||||||
|
|
||||||
video_stat.spatial_layer_idx = spatial_layer_idx;
|
video_stat.spatial_idx = spatial_idx;
|
||||||
video_stat.temporal_layer_idx = temporal_layer_idx;
|
video_stat.temporal_idx = temporal_idx;
|
||||||
|
|
||||||
video_stat.bitrate_kbps =
|
video_stat.bitrate_kbps =
|
||||||
static_cast<size_t>(8 * video_stat.length_bytes / 1000 / duration_sec);
|
static_cast<size_t>(8 * video_stat.length_bytes / 1000 / duration_sec);
|
||||||
@ -391,14 +390,14 @@ void Stats::GetNumberOfEncodedLayers(size_t first_frame_num,
|
|||||||
|
|
||||||
for (size_t frame_num = first_frame_num; frame_num <= last_frame_num;
|
for (size_t frame_num = first_frame_num; frame_num <= last_frame_num;
|
||||||
++frame_num) {
|
++frame_num) {
|
||||||
for (size_t spatial_layer_idx = 0; spatial_layer_idx < num_spatial_layers;
|
for (size_t spatial_idx = 0; spatial_idx < num_spatial_layers;
|
||||||
++spatial_layer_idx) {
|
++spatial_idx) {
|
||||||
FrameStatistics* frame_stat = GetFrame(frame_num, spatial_layer_idx);
|
FrameStatistics* frame_stat = GetFrame(frame_num, spatial_idx);
|
||||||
if (frame_stat->encoding_successful) {
|
if (frame_stat->encoding_successful) {
|
||||||
*num_encoded_spatial_layers = std::max(
|
*num_encoded_spatial_layers =
|
||||||
*num_encoded_spatial_layers, frame_stat->simulcast_svc_idx + 1);
|
std::max(*num_encoded_spatial_layers, frame_stat->spatial_idx + 1);
|
||||||
*num_encoded_temporal_layers = std::max(
|
*num_encoded_temporal_layers = std::max(*num_encoded_temporal_layers,
|
||||||
*num_encoded_temporal_layers, frame_stat->temporal_layer_idx + 1);
|
frame_stat->temporal_idx + 1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -40,8 +40,8 @@ struct FrameStatistics {
|
|||||||
webrtc::FrameType frame_type = kVideoFrameDelta;
|
webrtc::FrameType frame_type = kVideoFrameDelta;
|
||||||
|
|
||||||
// Layering.
|
// Layering.
|
||||||
size_t temporal_layer_idx = 0;
|
size_t spatial_idx = 0;
|
||||||
size_t simulcast_svc_idx = 0;
|
size_t temporal_idx = 0;
|
||||||
bool inter_layer_predicted = false;
|
bool inter_layer_predicted = false;
|
||||||
|
|
||||||
// H264 specific.
|
// H264 specific.
|
||||||
@ -72,8 +72,8 @@ struct VideoStatistics {
|
|||||||
size_t target_bitrate_kbps = 0;
|
size_t target_bitrate_kbps = 0;
|
||||||
float input_framerate_fps = 0.0f;
|
float input_framerate_fps = 0.0f;
|
||||||
|
|
||||||
size_t spatial_layer_idx = 0;
|
size_t spatial_idx = 0;
|
||||||
size_t temporal_layer_idx = 0;
|
size_t temporal_idx = 0;
|
||||||
|
|
||||||
size_t width = 0;
|
size_t width = 0;
|
||||||
size_t height = 0;
|
size_t height = 0;
|
||||||
@ -117,12 +117,11 @@ class Stats {
|
|||||||
~Stats() = default;
|
~Stats() = default;
|
||||||
|
|
||||||
// Creates a FrameStatistics for the next frame to be processed.
|
// Creates a FrameStatistics for the next frame to be processed.
|
||||||
FrameStatistics* AddFrame(size_t timestamp, size_t spatial_layer_idx);
|
FrameStatistics* AddFrame(size_t timestamp, size_t spatial_idx);
|
||||||
|
|
||||||
// Returns the FrameStatistics corresponding to |frame_number| or |timestamp|.
|
// Returns the FrameStatistics corresponding to |frame_number| or |timestamp|.
|
||||||
FrameStatistics* GetFrame(size_t frame_number, size_t spatial_layer_idx);
|
FrameStatistics* GetFrame(size_t frame_number, size_t spatial_idx);
|
||||||
FrameStatistics* GetFrameWithTimestamp(size_t timestamp,
|
FrameStatistics* GetFrameWithTimestamp(size_t timestamp, size_t spatial_idx);
|
||||||
size_t spatial_layer_idx);
|
|
||||||
|
|
||||||
std::vector<VideoStatistics> SliceAndCalcLayerVideoStatistic(
|
std::vector<VideoStatistics> SliceAndCalcLayerVideoStatistic(
|
||||||
size_t first_frame_num,
|
size_t first_frame_num,
|
||||||
@ -133,25 +132,25 @@ class Stats {
|
|||||||
|
|
||||||
void PrintFrameStatistics();
|
void PrintFrameStatistics();
|
||||||
|
|
||||||
size_t Size(size_t spatial_layer_idx);
|
size_t Size(size_t spatial_idx);
|
||||||
|
|
||||||
void Clear();
|
void Clear();
|
||||||
|
|
||||||
private:
|
private:
|
||||||
FrameStatistics AggregateFrameStatistic(size_t frame_num,
|
FrameStatistics AggregateFrameStatistic(size_t frame_num,
|
||||||
size_t spatial_layer_idx,
|
size_t spatial_idx,
|
||||||
bool aggregate_independent_layers);
|
bool aggregate_independent_layers);
|
||||||
|
|
||||||
size_t CalcLayerTargetBitrateKbps(size_t first_frame_num,
|
size_t CalcLayerTargetBitrateKbps(size_t first_frame_num,
|
||||||
size_t last_frame_num,
|
size_t last_frame_num,
|
||||||
size_t spatial_layer_idx,
|
size_t spatial_idx,
|
||||||
size_t temporal_layer_idx,
|
size_t temporal_idx,
|
||||||
bool aggregate_independent_layers);
|
bool aggregate_independent_layers);
|
||||||
|
|
||||||
VideoStatistics SliceAndCalcVideoStatistic(size_t first_frame_num,
|
VideoStatistics SliceAndCalcVideoStatistic(size_t first_frame_num,
|
||||||
size_t last_frame_num,
|
size_t last_frame_num,
|
||||||
size_t spatial_layer_idx,
|
size_t spatial_idx,
|
||||||
size_t temporal_layer_idx,
|
size_t temporal_idx,
|
||||||
bool aggregate_independent_layers);
|
bool aggregate_independent_layers);
|
||||||
|
|
||||||
void GetNumberOfEncodedLayers(size_t first_frame_num,
|
void GetNumberOfEncodedLayers(size_t first_frame_num,
|
||||||
|
@ -56,17 +56,17 @@ size_t GetMaxNaluSizeBytes(const EncodedImage& encoded_frame,
|
|||||||
}
|
}
|
||||||
|
|
||||||
void GetLayerIndices(const CodecSpecificInfo& codec_specific,
|
void GetLayerIndices(const CodecSpecificInfo& codec_specific,
|
||||||
size_t* simulcast_svc_idx,
|
size_t* spatial_idx,
|
||||||
size_t* temporal_idx) {
|
size_t* temporal_idx) {
|
||||||
if (codec_specific.codecType == kVideoCodecVP8) {
|
if (codec_specific.codecType == kVideoCodecVP8) {
|
||||||
*simulcast_svc_idx = codec_specific.codecSpecific.VP8.simulcastIdx;
|
*spatial_idx = codec_specific.codecSpecific.VP8.simulcastIdx;
|
||||||
*temporal_idx = codec_specific.codecSpecific.VP8.temporalIdx;
|
*temporal_idx = codec_specific.codecSpecific.VP8.temporalIdx;
|
||||||
} else if (codec_specific.codecType == kVideoCodecVP9) {
|
} else if (codec_specific.codecType == kVideoCodecVP9) {
|
||||||
*simulcast_svc_idx = codec_specific.codecSpecific.VP9.spatial_idx;
|
*spatial_idx = codec_specific.codecSpecific.VP9.spatial_idx;
|
||||||
*temporal_idx = codec_specific.codecSpecific.VP9.temporal_idx;
|
*temporal_idx = codec_specific.codecSpecific.VP9.temporal_idx;
|
||||||
}
|
}
|
||||||
if (*simulcast_svc_idx == kNoSpatialIdx) {
|
if (*spatial_idx == kNoSpatialIdx) {
|
||||||
*simulcast_svc_idx = 0;
|
*spatial_idx = 0;
|
||||||
}
|
}
|
||||||
if (*temporal_idx == kNoTemporalIdx) {
|
if (*temporal_idx == kNoTemporalIdx) {
|
||||||
*temporal_idx = 0;
|
*temporal_idx = 0;
|
||||||
@ -205,19 +205,15 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder,
|
|||||||
config_.max_payload_size_bytes),
|
config_.max_payload_size_bytes),
|
||||||
WEBRTC_VIDEO_CODEC_OK);
|
WEBRTC_VIDEO_CODEC_OK);
|
||||||
|
|
||||||
for (size_t simulcast_svc_idx = 0;
|
for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
|
||||||
simulcast_svc_idx < num_simulcast_or_spatial_layers_;
|
|
||||||
++simulcast_svc_idx) {
|
|
||||||
decode_callback_.push_back(
|
decode_callback_.push_back(
|
||||||
rtc::MakeUnique<VideoProcessorDecodeCompleteCallback>(
|
rtc::MakeUnique<VideoProcessorDecodeCompleteCallback>(this, i));
|
||||||
this, simulcast_svc_idx));
|
RTC_CHECK_EQ(
|
||||||
RTC_CHECK_EQ(decoders_->at(simulcast_svc_idx)
|
decoders_->at(i)->InitDecode(&config_.codec_settings,
|
||||||
->InitDecode(&config_.codec_settings,
|
static_cast<int>(config_.NumberOfCores())),
|
||||||
static_cast<int>(config_.NumberOfCores())),
|
WEBRTC_VIDEO_CODEC_OK);
|
||||||
WEBRTC_VIDEO_CODEC_OK);
|
RTC_CHECK_EQ(decoders_->at(i)->RegisterDecodeCompleteCallback(
|
||||||
RTC_CHECK_EQ(decoders_->at(simulcast_svc_idx)
|
decode_callback_.at(i).get()),
|
||||||
->RegisterDecodeCompleteCallback(
|
|
||||||
decode_callback_.at(simulcast_svc_idx).get()),
|
|
||||||
WEBRTC_VIDEO_CODEC_OK);
|
WEBRTC_VIDEO_CODEC_OK);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -238,10 +234,8 @@ VideoProcessor::~VideoProcessor() {
|
|||||||
RTC_CHECK_LE(input_frames_.size(), kMaxBufferedInputFrames);
|
RTC_CHECK_LE(input_frames_.size(), kMaxBufferedInputFrames);
|
||||||
|
|
||||||
// Deal with manual memory management of EncodedImage's.
|
// Deal with manual memory management of EncodedImage's.
|
||||||
for (size_t simulcast_svc_idx = 0;
|
for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
|
||||||
simulcast_svc_idx < num_simulcast_or_spatial_layers_;
|
uint8_t* buffer = merged_encoded_frames_.at(i)._buffer;
|
||||||
++simulcast_svc_idx) {
|
|
||||||
uint8_t* buffer = merged_encoded_frames_.at(simulcast_svc_idx)._buffer;
|
|
||||||
if (buffer) {
|
if (buffer) {
|
||||||
delete[] buffer;
|
delete[] buffer;
|
||||||
}
|
}
|
||||||
@ -270,20 +264,15 @@ void VideoProcessor::ProcessFrame() {
|
|||||||
post_encode_time_ns_ = 0;
|
post_encode_time_ns_ = 0;
|
||||||
|
|
||||||
// Create frame statistics object for all simulcast/spatial layers.
|
// Create frame statistics object for all simulcast/spatial layers.
|
||||||
for (size_t simulcast_svc_idx = 0;
|
for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
|
||||||
simulcast_svc_idx < num_simulcast_or_spatial_layers_;
|
stats_->AddFrame(timestamp, i);
|
||||||
++simulcast_svc_idx) {
|
|
||||||
stats_->AddFrame(timestamp, simulcast_svc_idx);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// For the highest measurement accuracy of the encode time, the start/stop
|
// For the highest measurement accuracy of the encode time, the start/stop
|
||||||
// time recordings should wrap the Encode call as tightly as possible.
|
// time recordings should wrap the Encode call as tightly as possible.
|
||||||
const int64_t encode_start_ns = rtc::TimeNanos();
|
const int64_t encode_start_ns = rtc::TimeNanos();
|
||||||
for (size_t simulcast_svc_idx = 0;
|
for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
|
||||||
simulcast_svc_idx < num_simulcast_or_spatial_layers_;
|
FrameStatistics* frame_stat = stats_->GetFrame(frame_number, i);
|
||||||
++simulcast_svc_idx) {
|
|
||||||
FrameStatistics* frame_stat =
|
|
||||||
stats_->GetFrame(frame_number, simulcast_svc_idx);
|
|
||||||
frame_stat->encode_start_ns = encode_start_ns;
|
frame_stat->encode_start_ns = encode_start_ns;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -292,11 +281,8 @@ void VideoProcessor::ProcessFrame() {
|
|||||||
config_.FrameTypeForFrame(frame_number);
|
config_.FrameTypeForFrame(frame_number);
|
||||||
const int encode_return_code =
|
const int encode_return_code =
|
||||||
encoder_->Encode(input_frame, nullptr, &frame_types);
|
encoder_->Encode(input_frame, nullptr, &frame_types);
|
||||||
for (size_t simulcast_svc_idx = 0;
|
for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
|
||||||
simulcast_svc_idx < num_simulcast_or_spatial_layers_;
|
FrameStatistics* frame_stat = stats_->GetFrame(frame_number, i);
|
||||||
++simulcast_svc_idx) {
|
|
||||||
FrameStatistics* frame_stat =
|
|
||||||
stats_->GetFrame(frame_number, simulcast_svc_idx);
|
|
||||||
frame_stat->encode_return_code = encode_return_code;
|
frame_stat->encode_return_code = encode_return_code;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -327,45 +313,45 @@ void VideoProcessor::FrameEncoded(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Layer metadata.
|
// Layer metadata.
|
||||||
size_t simulcast_svc_idx = 0;
|
size_t spatial_idx = 0;
|
||||||
size_t temporal_idx = 0;
|
size_t temporal_idx = 0;
|
||||||
GetLayerIndices(codec_specific, &simulcast_svc_idx, &temporal_idx);
|
GetLayerIndices(codec_specific, &spatial_idx, &temporal_idx);
|
||||||
|
|
||||||
FrameStatistics* frame_stat = stats_->GetFrameWithTimestamp(
|
FrameStatistics* frame_stat =
|
||||||
encoded_image._timeStamp, simulcast_svc_idx);
|
stats_->GetFrameWithTimestamp(encoded_image._timeStamp, spatial_idx);
|
||||||
const size_t frame_number = frame_stat->frame_number;
|
const size_t frame_number = frame_stat->frame_number;
|
||||||
|
|
||||||
// Ensure that the encode order is monotonically increasing, within this
|
// Ensure that the encode order is monotonically increasing, within this
|
||||||
// simulcast/spatial layer.
|
// simulcast/spatial layer.
|
||||||
RTC_CHECK(first_encoded_frame_[simulcast_svc_idx] ||
|
RTC_CHECK(first_encoded_frame_[spatial_idx] ||
|
||||||
last_encoded_frame_num_[simulcast_svc_idx] < frame_number);
|
last_encoded_frame_num_[spatial_idx] < frame_number);
|
||||||
|
|
||||||
// Ensure SVC spatial layers are delivered in ascending order.
|
// Ensure SVC spatial layers are delivered in ascending order.
|
||||||
if (!first_encoded_frame_[simulcast_svc_idx] &&
|
if (!first_encoded_frame_[spatial_idx] &&
|
||||||
config_.NumberOfSpatialLayers() > 1) {
|
config_.NumberOfSpatialLayers() > 1) {
|
||||||
for (size_t i = 0; i < simulcast_svc_idx; ++i) {
|
for (size_t i = 0; i < spatial_idx; ++i) {
|
||||||
RTC_CHECK_LE(last_encoded_frame_num_[i], frame_number);
|
RTC_CHECK_LE(last_encoded_frame_num_[i], frame_number);
|
||||||
}
|
}
|
||||||
for (size_t i = simulcast_svc_idx + 1; i < num_simulcast_or_spatial_layers_;
|
for (size_t i = spatial_idx + 1; i < num_simulcast_or_spatial_layers_;
|
||||||
++i) {
|
++i) {
|
||||||
RTC_CHECK_GT(frame_number, last_encoded_frame_num_[i]);
|
RTC_CHECK_GT(frame_number, last_encoded_frame_num_[i]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
first_encoded_frame_[simulcast_svc_idx] = false;
|
first_encoded_frame_[spatial_idx] = false;
|
||||||
last_encoded_frame_num_[simulcast_svc_idx] = frame_number;
|
last_encoded_frame_num_[spatial_idx] = frame_number;
|
||||||
|
|
||||||
// Update frame statistics.
|
// Update frame statistics.
|
||||||
frame_stat->encoding_successful = true;
|
frame_stat->encoding_successful = true;
|
||||||
frame_stat->encode_time_us = GetElapsedTimeMicroseconds(
|
frame_stat->encode_time_us = GetElapsedTimeMicroseconds(
|
||||||
frame_stat->encode_start_ns, encode_stop_ns - post_encode_time_ns_);
|
frame_stat->encode_start_ns, encode_stop_ns - post_encode_time_ns_);
|
||||||
frame_stat->target_bitrate_kbps = (bitrate_allocation_.GetTemporalLayerSum(
|
frame_stat->target_bitrate_kbps =
|
||||||
simulcast_svc_idx, temporal_idx) +
|
(bitrate_allocation_.GetTemporalLayerSum(spatial_idx, temporal_idx) +
|
||||||
500) /
|
500) /
|
||||||
1000;
|
1000;
|
||||||
frame_stat->length_bytes = encoded_image._length;
|
frame_stat->length_bytes = encoded_image._length;
|
||||||
frame_stat->frame_type = encoded_image._frameType;
|
frame_stat->frame_type = encoded_image._frameType;
|
||||||
frame_stat->temporal_layer_idx = temporal_idx;
|
frame_stat->temporal_idx = temporal_idx;
|
||||||
frame_stat->simulcast_svc_idx = simulcast_svc_idx;
|
frame_stat->spatial_idx = spatial_idx;
|
||||||
frame_stat->max_nalu_size_bytes = GetMaxNaluSizeBytes(encoded_image, config_);
|
frame_stat->max_nalu_size_bytes = GetMaxNaluSizeBytes(encoded_image, config_);
|
||||||
frame_stat->qp = encoded_image.qp_;
|
frame_stat->qp = encoded_image.qp_;
|
||||||
|
|
||||||
@ -384,31 +370,29 @@ void VideoProcessor::FrameEncoded(
|
|||||||
if (config_.decode || encoded_frame_writers_) {
|
if (config_.decode || encoded_frame_writers_) {
|
||||||
if (num_spatial_layers > 1) {
|
if (num_spatial_layers > 1) {
|
||||||
encoded_image_for_decode = BuildAndStoreSuperframe(
|
encoded_image_for_decode = BuildAndStoreSuperframe(
|
||||||
encoded_image, codec_type, frame_number, simulcast_svc_idx,
|
encoded_image, codec_type, frame_number, spatial_idx,
|
||||||
frame_stat->inter_layer_predicted);
|
frame_stat->inter_layer_predicted);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config_.decode) {
|
if (config_.decode) {
|
||||||
DecodeFrame(*encoded_image_for_decode, simulcast_svc_idx);
|
DecodeFrame(*encoded_image_for_decode, spatial_idx);
|
||||||
|
|
||||||
if (end_of_superframe && inter_layer_prediction) {
|
if (end_of_superframe && inter_layer_prediction) {
|
||||||
// If inter-layer prediction is enabled and upper layer was dropped then
|
// If inter-layer prediction is enabled and upper layer was dropped then
|
||||||
// base layer should be passed to upper layer decoder. Otherwise decoder
|
// base layer should be passed to upper layer decoder. Otherwise decoder
|
||||||
// won't be able to decode next superframe.
|
// won't be able to decode next superframe.
|
||||||
const EncodedImage* base_image = nullptr;
|
const EncodedImage* base_image = nullptr;
|
||||||
for (size_t spatial_idx = 0; spatial_idx < num_spatial_layers;
|
for (size_t i = 0; i < num_spatial_layers; ++i) {
|
||||||
++spatial_idx) {
|
const bool layer_dropped = last_decoded_frame_num_[i] < frame_number;
|
||||||
const bool layer_dropped =
|
|
||||||
last_decoded_frame_num_[spatial_idx] < frame_number;
|
|
||||||
|
|
||||||
// Ensure current layer was decoded.
|
// Ensure current layer was decoded.
|
||||||
RTC_CHECK(layer_dropped == false || spatial_idx != simulcast_svc_idx);
|
RTC_CHECK(layer_dropped == false || i != spatial_idx);
|
||||||
|
|
||||||
if (!layer_dropped) {
|
if (!layer_dropped) {
|
||||||
base_image = &merged_encoded_frames_[spatial_idx];
|
base_image = &merged_encoded_frames_[i];
|
||||||
} else if (base_image) {
|
} else if (base_image) {
|
||||||
DecodeFrame(*base_image, spatial_idx);
|
DecodeFrame(*base_image, i);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -417,7 +401,7 @@ void VideoProcessor::FrameEncoded(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (encoded_frame_writers_) {
|
if (encoded_frame_writers_) {
|
||||||
RTC_CHECK(encoded_frame_writers_->at(simulcast_svc_idx)
|
RTC_CHECK(encoded_frame_writers_->at(spatial_idx)
|
||||||
->WriteFrame(*encoded_image_for_decode,
|
->WriteFrame(*encoded_image_for_decode,
|
||||||
config_.codec_settings.codecType));
|
config_.codec_settings.codecType));
|
||||||
}
|
}
|
||||||
@ -430,23 +414,23 @@ void VideoProcessor::FrameEncoded(
|
|||||||
}
|
}
|
||||||
|
|
||||||
void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame,
|
void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame,
|
||||||
size_t simulcast_svc_idx) {
|
size_t spatial_idx) {
|
||||||
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
|
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
|
||||||
|
|
||||||
// For the highest measurement accuracy of the decode time, the start/stop
|
// For the highest measurement accuracy of the decode time, the start/stop
|
||||||
// time recordings should wrap the Decode call as tightly as possible.
|
// time recordings should wrap the Decode call as tightly as possible.
|
||||||
const int64_t decode_stop_ns = rtc::TimeNanos();
|
const int64_t decode_stop_ns = rtc::TimeNanos();
|
||||||
|
|
||||||
FrameStatistics* frame_stat = stats_->GetFrameWithTimestamp(
|
FrameStatistics* frame_stat =
|
||||||
decoded_frame.timestamp(), simulcast_svc_idx);
|
stats_->GetFrameWithTimestamp(decoded_frame.timestamp(), spatial_idx);
|
||||||
const size_t frame_number = frame_stat->frame_number;
|
const size_t frame_number = frame_stat->frame_number;
|
||||||
|
|
||||||
// Ensure that the decode order is monotonically increasing, within this
|
// Ensure that the decode order is monotonically increasing, within this
|
||||||
// simulcast/spatial layer.
|
// simulcast/spatial layer.
|
||||||
RTC_CHECK(first_decoded_frame_[simulcast_svc_idx] ||
|
RTC_CHECK(first_decoded_frame_[spatial_idx] ||
|
||||||
last_decoded_frame_num_[simulcast_svc_idx] < frame_number);
|
last_decoded_frame_num_[spatial_idx] < frame_number);
|
||||||
first_decoded_frame_[simulcast_svc_idx] = false;
|
first_decoded_frame_[spatial_idx] = false;
|
||||||
last_decoded_frame_num_[simulcast_svc_idx] = frame_number;
|
last_decoded_frame_num_[spatial_idx] = frame_number;
|
||||||
|
|
||||||
// Update frame statistics.
|
// Update frame statistics.
|
||||||
frame_stat->decoding_successful = true;
|
frame_stat->decoding_successful = true;
|
||||||
@ -483,28 +467,28 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame,
|
|||||||
ExtractI420BufferWithSize(decoded_frame, config_.codec_settings.width,
|
ExtractI420BufferWithSize(decoded_frame, config_.codec_settings.width,
|
||||||
config_.codec_settings.height, &tmp_i420_buffer_);
|
config_.codec_settings.height, &tmp_i420_buffer_);
|
||||||
RTC_CHECK_EQ(tmp_i420_buffer_.size(),
|
RTC_CHECK_EQ(tmp_i420_buffer_.size(),
|
||||||
decoded_frame_writers_->at(simulcast_svc_idx)->FrameLength());
|
decoded_frame_writers_->at(spatial_idx)->FrameLength());
|
||||||
RTC_CHECK(decoded_frame_writers_->at(simulcast_svc_idx)
|
RTC_CHECK(decoded_frame_writers_->at(spatial_idx)
|
||||||
->WriteFrame(tmp_i420_buffer_.data()));
|
->WriteFrame(tmp_i420_buffer_.data()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void VideoProcessor::DecodeFrame(const EncodedImage& encoded_image,
|
void VideoProcessor::DecodeFrame(const EncodedImage& encoded_image,
|
||||||
size_t simulcast_svc_idx) {
|
size_t spatial_idx) {
|
||||||
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
|
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
|
||||||
FrameStatistics* frame_stat = stats_->GetFrameWithTimestamp(
|
FrameStatistics* frame_stat =
|
||||||
encoded_image._timeStamp, simulcast_svc_idx);
|
stats_->GetFrameWithTimestamp(encoded_image._timeStamp, spatial_idx);
|
||||||
|
|
||||||
frame_stat->decode_start_ns = rtc::TimeNanos();
|
frame_stat->decode_start_ns = rtc::TimeNanos();
|
||||||
frame_stat->decode_return_code =
|
frame_stat->decode_return_code =
|
||||||
decoders_->at(simulcast_svc_idx)->Decode(encoded_image, false, nullptr);
|
decoders_->at(spatial_idx)->Decode(encoded_image, false, nullptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe(
|
const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe(
|
||||||
const EncodedImage& encoded_image,
|
const EncodedImage& encoded_image,
|
||||||
const VideoCodecType codec,
|
const VideoCodecType codec,
|
||||||
size_t frame_number,
|
size_t frame_number,
|
||||||
size_t simulcast_svc_idx,
|
size_t spatial_idx,
|
||||||
bool inter_layer_predicted) {
|
bool inter_layer_predicted) {
|
||||||
// Should only be called for SVC.
|
// Should only be called for SVC.
|
||||||
RTC_CHECK_GT(config_.NumberOfSpatialLayers(), 1);
|
RTC_CHECK_GT(config_.NumberOfSpatialLayers(), 1);
|
||||||
@ -515,7 +499,7 @@ const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe(
|
|||||||
// Each SVC layer is decoded with dedicated decoder. Find the nearest
|
// Each SVC layer is decoded with dedicated decoder. Find the nearest
|
||||||
// non-dropped base frame and merge it and current frame into superframe.
|
// non-dropped base frame and merge it and current frame into superframe.
|
||||||
if (inter_layer_predicted) {
|
if (inter_layer_predicted) {
|
||||||
for (int base_idx = static_cast<int>(simulcast_svc_idx) - 1; base_idx >= 0;
|
for (int base_idx = static_cast<int>(spatial_idx) - 1; base_idx >= 0;
|
||||||
--base_idx) {
|
--base_idx) {
|
||||||
EncodedImage lower_layer = merged_encoded_frames_.at(base_idx);
|
EncodedImage lower_layer = merged_encoded_frames_.at(base_idx);
|
||||||
if (lower_layer._timeStamp == encoded_image._timeStamp) {
|
if (lower_layer._timeStamp == encoded_image._timeStamp) {
|
||||||
@ -545,13 +529,13 @@ const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe(
|
|||||||
copied_image._size = buffer_size_bytes;
|
copied_image._size = buffer_size_bytes;
|
||||||
|
|
||||||
// Replace previous EncodedImage for this spatial layer.
|
// Replace previous EncodedImage for this spatial layer.
|
||||||
uint8_t* old_buffer = merged_encoded_frames_.at(simulcast_svc_idx)._buffer;
|
uint8_t* old_buffer = merged_encoded_frames_.at(spatial_idx)._buffer;
|
||||||
if (old_buffer) {
|
if (old_buffer) {
|
||||||
delete[] old_buffer;
|
delete[] old_buffer;
|
||||||
}
|
}
|
||||||
merged_encoded_frames_.at(simulcast_svc_idx) = copied_image;
|
merged_encoded_frames_.at(spatial_idx) = copied_image;
|
||||||
|
|
||||||
return &merged_encoded_frames_.at(simulcast_svc_idx);
|
return &merged_encoded_frames_.at(spatial_idx);
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace test
|
} // namespace test
|
||||||
|
@ -53,20 +53,23 @@ class VideoProcessorIntegrationTestLibvpx
|
|||||||
|
|
||||||
void PrintRdPerf(std::map<size_t, std::vector<VideoStatistics>> rd_stats) {
|
void PrintRdPerf(std::map<size_t, std::vector<VideoStatistics>> rd_stats) {
|
||||||
printf("--> Summary\n");
|
printf("--> Summary\n");
|
||||||
printf("%11s %5s %6s %13s %13s %5s %7s %7s %7s %13s %13s\n", "uplink_kbps",
|
printf("%11s %5s %6s %11s %12s %11s %13s %13s %5s %7s %7s %7s %13s %13s\n",
|
||||||
"width", "height", "downlink_kbps", "framerate_fps", "psnr",
|
"uplink_kbps", "width", "height", "spatial_idx", "temporal_idx",
|
||||||
"psnr_y", "psnr_u", "psnr_v", "enc_speed_fps", "dec_speed_fps");
|
"target_kbps", "downlink_kbps", "framerate_fps", "psnr", "psnr_y",
|
||||||
|
"psnr_u", "psnr_v", "enc_speed_fps", "dec_speed_fps");
|
||||||
for (const auto& rd_stat : rd_stats) {
|
for (const auto& rd_stat : rd_stats) {
|
||||||
const size_t bitrate_kbps = rd_stat.first;
|
const size_t bitrate_kbps = rd_stat.first;
|
||||||
for (const auto& layer_stat : rd_stat.second) {
|
for (const auto& layer_stat : rd_stat.second) {
|
||||||
printf(
|
printf(
|
||||||
"%11zu %5zu %6zu %13zu %13.2f %5.2f %7.2f %7.2f %7.2f %13.2f "
|
"%11zu %5zu %6zu %11zu %12zu %11zu %13zu %13.2f %5.2f %7.2f %7.2f "
|
||||||
"%13.2f\n",
|
"%7.2f"
|
||||||
|
"%13.2f %13.2f\n",
|
||||||
bitrate_kbps, layer_stat.width, layer_stat.height,
|
bitrate_kbps, layer_stat.width, layer_stat.height,
|
||||||
layer_stat.bitrate_kbps, layer_stat.framerate_fps,
|
layer_stat.spatial_idx, layer_stat.temporal_idx,
|
||||||
layer_stat.avg_psnr, layer_stat.avg_psnr_y, layer_stat.avg_psnr_u,
|
layer_stat.target_bitrate_kbps, layer_stat.bitrate_kbps,
|
||||||
layer_stat.avg_psnr_v, layer_stat.enc_speed_fps,
|
layer_stat.framerate_fps, layer_stat.avg_psnr,
|
||||||
layer_stat.dec_speed_fps);
|
layer_stat.avg_psnr_y, layer_stat.avg_psnr_u, layer_stat.avg_psnr_v,
|
||||||
|
layer_stat.enc_speed_fps, layer_stat.dec_speed_fps);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user