zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generic Frame Descriptor (GFD) VP8 templates.

Browse Source 
In this CL:
 - Updated Vp8TemporalLayers::OnEncodeDone to take a CodecSpecificInfo
   instead of a CodecSpecificInfoVP8, so that both the VP8 specific and
   generic information can be populated.
 - Added structs to represent the GFD template structure.
 - Added code to generate templates for video/screensharing.

Bug: webrtc:10342
Change-Id: I978f9d708597a6f86bbdc494e62acf7a7b400db3
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/123422
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26987}
This commit is contained in:
philipel
2019-03-04 16:37:50 +01:00
committed by Commit Bot
parent 8fb1a6ad27
commit 9df335374a
17 changed files with 441 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
api/video_codecs/vp8_temporal_layers.h
View File

@ -44,7 +44,7 @@ namespace webrtc {
// the bitrate produced.
enum class Vp8TemporalLayersType { kFixedPattern, kBitrateDynamic };
struct CodecSpecificInfoVP8;
struct CodecSpecificInfo;
struct Vp8EncoderConfig {
static constexpr size_t kMaxPeriodicity = 16;
@ -115,8 +115,8 @@ class Vp8TemporalLayers {
// a keyframe.
// If the encoder decided to drop this frame, |size_bytes| must be set to 0,
// otherwise it should indicate the size in bytes of the encoded frame.
// If |size_bytes| > 0, and |vp8_info| is not null, the TemporalLayers
// instance my update |vp8_info| with codec specific data such as temporal id.
// If |size_bytes| > 0, and |info| is not null, the TemporalLayers
// instance my update |info| with codec specific data such as temporal id.
// Some fields of this struct may have already been populated by the encoder,
// check before overwriting.
// If |size_bytes| > 0, |qp| should indicate the frame-level QP this frame was
@ -126,7 +126,7 @@ class Vp8TemporalLayers {
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfoVP8* vp8_info) = 0;
CodecSpecificInfo* info) = 0;
};
} // namespace webrtc

23
common_video/generic_frame_descriptor/BUILD.gn Normal file
View File

@ -0,0 +1,23 @@
# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
import("../../webrtc.gni")
rtc_source_set("generic_frame_descriptor") {
sources = [
"generic_frame_info.cc",
"generic_frame_info.h",
]
deps = [
"../../api:array_view",
"../../rtc_base:checks",
"//third_party/abseil-cpp/absl/container:inlined_vector",
"//third_party/abseil-cpp/absl/strings:strings",
]
}

7
common_video/generic_frame_descriptor/OWNERS Normal file
View File

@ -0,0 +1,7 @@
philipel@webrtc.org
danilchap@webrtc.org
# These are for the common case of adding or renaming files. If you're doing
# structural changes, please get a review from a reviewer in this file.
per-file *.gn=*
per-file *.gni=*

70
common_video/generic_frame_descriptor/generic_frame_info.cc Normal file
View File

@ -0,0 +1,70 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "common_video/generic_frame_descriptor/generic_frame_info.h"
#include <utility>
#include "rtc_base/checks.h"
namespace webrtc {
GenericFrameInfo::GenericFrameInfo() = default;
GenericFrameInfo::GenericFrameInfo(const GenericFrameInfo&) = default;
GenericFrameInfo::~GenericFrameInfo() = default;
GenericFrameInfo::Builder::Builder() = default;
GenericFrameInfo::Builder::~Builder() = default;
GenericFrameInfo GenericFrameInfo::Builder::Build() const {
return info_;
}
GenericFrameInfo::Builder& GenericFrameInfo::Builder::T(int temporal_id) {
info_.temporal_id = temporal_id;
return *this;
}
GenericFrameInfo::Builder& GenericFrameInfo::Builder::S(int spatial_id) {
info_.spatial_id = spatial_id;
return *this;
}
GenericFrameInfo::Builder& GenericFrameInfo::Builder::Dtis(
absl::string_view indication_symbols) {
for (const auto& symbol : indication_symbols) {
OperatingPointIndication indication;
switch (symbol) {
case '-': indication = OperatingPointIndication::kNotPresent; break;
case 'D': indication = OperatingPointIndication::kDiscardable; break;
case 'R': indication = OperatingPointIndication::kRequired; break;
case 'S': indication = OperatingPointIndication::kSwitch; break;
default: RTC_NOTREACHED();
}
info_.operating_points.push_back(indication);
}
return *this;
}
GenericFrameInfo::Builder& GenericFrameInfo::Builder::Fdiffs(
std::initializer_list<int> frame_diffs) {
info_.frame_diffs.insert(info_.frame_diffs.end(), frame_diffs.begin(),
frame_diffs.end());
return *this;
}
TemplateStructure::TemplateStructure() = default;
TemplateStructure::TemplateStructure(const TemplateStructure&) = default;
TemplateStructure::TemplateStructure(TemplateStructure&&) = default;
TemplateStructure& TemplateStructure::operator=(const TemplateStructure&) =
default;
TemplateStructure::~TemplateStructure() = default;
} // namespace webrtc

70
common_video/generic_frame_descriptor/generic_frame_info.h Normal file
View File

@ -0,0 +1,70 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef COMMON_VIDEO_GENERIC_FRAME_DESCRIPTOR_GENERIC_FRAME_INFO_H_
#define COMMON_VIDEO_GENERIC_FRAME_DESCRIPTOR_GENERIC_FRAME_INFO_H_
#include <initializer_list>
#include <vector>
#include "absl/container/inlined_vector.h"
#include "absl/strings/string_view.h"
#include "api/array_view.h"
namespace webrtc {
struct GenericFrameInfo {
enum class OperatingPointIndication {
kNotPresent, // GenericFrameInfo::Builder symbol '-'
kDiscardable, // GenericFrameInfo::Builder symbol 'D'
kSwitch, // GenericFrameInfo::Builder symbol 'S'
kRequired // GenericFrameInfo::Builder symbol 'R'
};
class Builder;
GenericFrameInfo();
GenericFrameInfo(const GenericFrameInfo&);
~GenericFrameInfo();
int temporal_id = 0;
int spatial_id = 0;
absl::InlinedVector<int, 10> frame_diffs;
absl::InlinedVector<OperatingPointIndication, 10> operating_points;
};
class GenericFrameInfo::Builder {
public:
Builder();
~Builder();
GenericFrameInfo Build() const;
Builder& T(int temporal_id);
Builder& S(int spatial_id);
Builder& Dtis(absl::string_view indication_symbols);
Builder& Fdiffs(std::initializer_list<int> frame_diffs);
private:
GenericFrameInfo info_;
};
struct TemplateStructure {
TemplateStructure();
TemplateStructure(const TemplateStructure&);
TemplateStructure(TemplateStructure&&);
TemplateStructure& operator=(const TemplateStructure&);
~TemplateStructure();
int num_operating_points = 0;
std::vector<GenericFrameInfo> templates;
};
} // namespace webrtc
#endif // COMMON_VIDEO_GENERIC_FRAME_DESCRIPTOR_GENERIC_FRAME_INFO_H_

2
modules/video_coding/BUILD.gn
View File

@ -185,6 +185,7 @@ rtc_static_library("video_coding") {
rtc_source_set("video_codec_interface") {
visibility = [ "*" ]
sources = [
"include/video_codec_interface.cc",
"include/video_codec_interface.h",
"include/video_coding_defines.h",
"include/video_error_codes.h",
@ -196,6 +197,7 @@ rtc_source_set("video_codec_interface") {
"../../api/video:video_frame",
"../../api/video_codecs:video_codecs_api",
"../../common_video:common_video",
"../../common_video/generic_frame_descriptor:generic_frame_descriptor",
"//third_party/abseil-cpp/absl/types:optional",
]
}

94
modules/video_coding/codecs/vp8/default_temporal_layers.cc
View File

@ -444,7 +444,7 @@ void DefaultTemporalLayers::OnEncodeDone(uint32_t rtp_timestamp,
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfoVP8* vp8_info) {
CodecSpecificInfo* info) {
RTC_DCHECK_GT(num_layers_, 0);
auto pending_frame = pending_frames_.find(rtp_timestamp);
@ -463,15 +463,16 @@ void DefaultTemporalLayers::OnEncodeDone(uint32_t rtp_timestamp,
}
#endif
CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8;
if (num_layers_ == 1) {
vp8_info->temporalIdx = kNoTemporalIdx;
vp8_info->layerSync = false;
vp8_info.temporalIdx = kNoTemporalIdx;
vp8_info.layerSync = false;
} else {
if (is_keyframe) {
// Restart the temporal pattern on keyframes.
pattern_idx_ = 0;
vp8_info->temporalIdx = 0;
vp8_info->layerSync = true; // Keyframes are always sync frames.
vp8_info.temporalIdx = 0;
vp8_info.layerSync = true; // Keyframes are always sync frames.
for (Vp8BufferReference buffer : kAllBuffers) {
if (kf_buffers_.find(buffer) != kf_buffers_.end()) {
@ -486,29 +487,35 @@ void DefaultTemporalLayers::OnEncodeDone(uint32_t rtp_timestamp,
}
} else {
// Delta frame, update codec specifics with temporal id and sync flag.
vp8_info->temporalIdx = frame.frame_config.packetizer_temporal_idx;
vp8_info->layerSync = frame.frame_config.layer_sync;
vp8_info.temporalIdx = frame.frame_config.packetizer_temporal_idx;
vp8_info.layerSync = frame.frame_config.layer_sync;
}
}
vp8_info->useExplicitDependencies = true;
RTC_DCHECK_EQ(vp8_info->referencedBuffersCount, 0u);
RTC_DCHECK_EQ(vp8_info->updatedBuffersCount, 0u);
vp8_info.useExplicitDependencies = true;
RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u);
RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u);
for (int i = 0; i < static_cast<int>(Buffer::kCount); ++i) {
if (!is_keyframe && frame.frame_config.References(static_cast<Buffer>(i))) {
RTC_DCHECK_LT(vp8_info->referencedBuffersCount,
RTC_DCHECK_LT(vp8_info.referencedBuffersCount,
arraysize(CodecSpecificInfoVP8::referencedBuffers));
vp8_info->referencedBuffers[vp8_info->referencedBuffersCount++] = i;
vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i;
}
if (is_keyframe || frame.frame_config.Updates(static_cast<Buffer>(i))) {
RTC_DCHECK_LT(vp8_info->updatedBuffersCount,
RTC_DCHECK_LT(vp8_info.updatedBuffersCount,
arraysize(CodecSpecificInfoVP8::updatedBuffers));
vp8_info->updatedBuffers[vp8_info->updatedBuffersCount++] = i;
vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i;
}
}
// The templates are always present on keyframes, and then refered to by
// subsequent frames.
if (is_keyframe) {
info->template_structure = GetTemplateStructure(num_layers_);
}
if (!frame.expired) {
for (Vp8BufferReference buffer : kAllBuffers) {
if (frame.updated_buffer_mask & static_cast<uint8_t>(buffer)) {
@ -518,6 +525,65 @@ void DefaultTemporalLayers::OnEncodeDone(uint32_t rtp_timestamp,
}
}
TemplateStructure DefaultTemporalLayers::GetTemplateStructure(
int num_layers) const {
RTC_CHECK_LT(num_layers, 5);
RTC_CHECK_GT(num_layers, 0);
TemplateStructure template_structure;
template_structure.num_operating_points = num_layers;
using Builder = GenericFrameInfo::Builder;
switch (num_layers) {
case 1: {
template_structure.templates = {
Builder().T(0).Dtis("S").Build(),
Builder().T(0).Dtis("S").Fdiffs({1}).Build(),
};
return template_structure;
}
case 2: {
template_structure.templates = {
Builder().T(0).Dtis("SS").Build(),
Builder().T(0).Dtis("SS").Fdiffs({2}).Build(),
Builder().T(0).Dtis("SR").Fdiffs({2}).Build(),
Builder().T(1).Dtis("-S").Fdiffs({1}).Build(),
Builder().T(1).Dtis("-D").Fdiffs({1, 2}).Build(),
};
return template_structure;
}
case 3: {
template_structure.templates = {
Builder().T(0).Dtis("SSS").Build(),
Builder().T(0).Dtis("SSS").Fdiffs({4}).Build(),
Builder().T(0).Dtis("SRR").Fdiffs({4}).Build(),
Builder().T(1).Dtis("-SR").Fdiffs({2}).Build(),
Builder().T(1).Dtis("-DR").Fdiffs({2, 4}).Build(),
Builder().T(2).Dtis("--D").Fdiffs({1}).Build(),
Builder().T(2).Dtis("--D").Fdiffs({1, 3}).Build(),
};
return template_structure;
}
case 4: {
template_structure.templates = {
Builder().T(0).Dtis("SSSS").Build(),
Builder().T(0).Dtis("SSSS").Fdiffs({8}).Build(),
Builder().T(1).Dtis("-SRR").Fdiffs({4}).Build(),
Builder().T(1).Dtis("-SRR").Fdiffs({4, 8}).Build(),
Builder().T(2).Dtis("--SR").Fdiffs({2}).Build(),
Builder().T(2).Dtis("--SR").Fdiffs({2, 4}).Build(),
Builder().T(3).Dtis("---D").Fdiffs({1}).Build(),
Builder().T(3).Dtis("---D").Fdiffs({1, 3}).Build(),
};
return template_structure;
}
default:
RTC_NOTREACHED();
// To make the compiler happy!
return template_structure;
}
}
// Returns list of temporal dependencies for each frame in the temporal pattern.
// Values are lists of indecies in the pattern.
std::vector<std::set<uint8_t>> GetTemporalDependencies(

3
modules/video_coding/codecs/vp8/default_temporal_layers.h
View File

@ -49,7 +49,7 @@ class DefaultTemporalLayers : public Vp8TemporalLayers {
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfoVP8* vp8_info) override;
CodecSpecificInfo* info) override;
private:
static constexpr size_t kKeyframeBuffer = std::numeric_limits<size_t>::max();
@ -64,6 +64,7 @@ class DefaultTemporalLayers : public Vp8TemporalLayers {
const std::vector<Vp8FrameConfig> temporal_pattern_;
// Set of buffers that are never updated except by keyframes.
const std::set<Vp8FrameConfig::Vp8BufferReference> kf_buffers_;
TemplateStructure GetTemplateStructure(int num_layers) const;
uint8_t pattern_idx_;
// Updated cumulative bitrates, per temporal layer.

105
modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc
View File

@ -107,9 +107,9 @@ class TemporalLayersTest : public ::testing::Test {
public:
~TemporalLayersTest() override = default;
CodecSpecificInfoVP8* IgnoredCodecSpecificInfoVp8() {
CodecSpecificInfo* IgnoredCodecSpecificInfo() {
codec_specific_info_ = absl::make_unique<CodecSpecificInfo>();
return &codec_specific_info_->codecSpecific.VP8;
return codec_specific_info_.get();
}
private:
@ -142,17 +142,17 @@ TEST_F(TemporalLayersTest, 2Layers) {
for (size_t i = 0; i < kPatternSize * kRepetitions; ++i) {
const size_t ind = i % kPatternSize;
CodecSpecificInfo info;
CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8;
Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[ind], LibvpxVp8Encoder::EncodeFlags(tl_config))
<< i;
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp,
&vp8_info);
&info);
EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
EXPECT_EQ(expected_temporal_idx[ind], vp8_info.temporalIdx);
EXPECT_EQ(expected_temporal_idx[ind], info.codecSpecific.VP8.temporalIdx);
EXPECT_EQ(expected_temporal_idx[ind], tl_config.packetizer_temporal_idx);
EXPECT_EQ(expected_temporal_idx[ind], tl_config.encoder_layer_id);
EXPECT_EQ(i == 0 || expected_layer_sync[ind], vp8_info.layerSync);
EXPECT_EQ(i == 0 || expected_layer_sync[ind],
info.codecSpecific.VP8.layerSync);
EXPECT_EQ(expected_layer_sync[ind], tl_config.layer_sync);
timestamp += 3000;
}
@ -196,16 +196,16 @@ TEST_F(TemporalLayersTest, 3Layers) {
unsigned int timestamp = 0;
for (int i = 0; i < 16; ++i) {
CodecSpecificInfo info;
CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8;
Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i;
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp,
&vp8_info);
&info);
EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync);
EXPECT_EQ(i == 0 || expected_layer_sync[i],
info.codecSpecific.VP8.layerSync);
EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
timestamp += 3000;
}
@ -238,16 +238,16 @@ TEST_F(TemporalLayersTest, Alternative3Layers) {
unsigned int timestamp = 0;
for (int i = 0; i < 8; ++i) {
CodecSpecificInfo info;
CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8;
Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i;
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp,
&vp8_info);
&info);
EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync);
EXPECT_EQ(i == 0 || expected_layer_sync[i],
info.codecSpecific.VP8.layerSync);
EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
timestamp += 3000;
}
@ -271,19 +271,19 @@ TEST_F(TemporalLayersTest, SearchOrder) {
uint32_t timestamp = 0;
Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// TL2 frame. First one only references TL0. Updates altref.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
// TL1 frame. Can only reference TL0. Updated golden.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
@ -291,7 +291,7 @@ TEST_F(TemporalLayersTest, SearchOrder) {
// updated, the next to last was altref.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kGolden);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kAltref);
}
@ -314,12 +314,12 @@ TEST_F(TemporalLayersTest, SearchOrderWithDrop) {
uint32_t timestamp = 0;
Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// TL2 frame. First one only references TL0. Updates altref.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
@ -331,7 +331,7 @@ TEST_F(TemporalLayersTest, SearchOrderWithDrop) {
// been populated this cycle. Altref was last to be updated, before that last.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kAltref);
EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kLast);
}
@ -373,16 +373,16 @@ TEST_F(TemporalLayersTest, 4Layers) {
uint32_t timestamp = 0;
for (int i = 0; i < 16; ++i) {
CodecSpecificInfo info;
CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8;
Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i;
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp,
&vp8_info);
&info);
EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config));
EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync);
EXPECT_EQ(i == 0 || expected_layer_sync[i],
info.codecSpecific.VP8.layerSync);
EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
timestamp += 3000;
}
@ -405,7 +405,7 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
uint32_t timestamp = 0;
Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// Dropped TL2 frame.
tl_config = tl.UpdateLayerConfig(++timestamp);
@ -419,7 +419,7 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
// both contain the last keyframe.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
@ -429,23 +429,23 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
// TL0 base layer frame, updating and referencing last.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// TL2 frame, updating altref.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// TL1 frame, updating golden.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// TL2 frame. Can still reference all buffer since they have been update this
// cycle.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
@ -455,7 +455,7 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
// TL0 base layer frame, updating and referencing last.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// Dropped TL2 frame.
tl_config = tl.UpdateLayerConfig(++timestamp);
@ -469,7 +469,7 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
// and cannot be referenced.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
@ -491,24 +491,24 @@ TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) {
uint32_t timestamp = 0;
Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// Do a full cycle of the pattern.
for (int i = 0; i < 7; ++i) {
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
}
// TL0 base layer frame, starting the cycle over.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// TL2 frame.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// Encoder has a hiccup and builds a queue, so frame encoding is delayed.
// TL1 frame, updating golden.
@ -528,13 +528,13 @@ TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) {
// buffers are now OK to reference.
// Enqueued TL1 frame ready.
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// Enqueued TL2 frame.
tl.OnEncodeDone(++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// Enqueued TL0 frame.
tl.OnEncodeDone(++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// TL2 frame, all buffers are now in a known good state, OK to reference.
tl_config = tl.UpdateLayerConfig(++timestamp + 1);
@ -560,24 +560,24 @@ TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) {
uint32_t timestamp = 0;
Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// Do a full cycle of the pattern.
for (int i = 0; i < 3; ++i) {
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
}
// TL0 base layer frame, starting the cycle over.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// TL2 frame.
tl_config = tl.UpdateLayerConfig(++timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// Encoder has a hiccup and builds a queue, so frame encoding is delayed.
// Encoded, but delayed frames in TL 1, 2.
@ -592,10 +592,10 @@ TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) {
// TL1 frame from last cycle is ready.
tl.OnEncodeDone(timestamp + 1, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// TL2 frame from last cycle is ready.
tl.OnEncodeDone(timestamp + 2, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// TL2 frame, that should be referencing all buffers, but altref and golden
// haven not been updated this cycle. (Don't be fooled by the late frames from
@ -641,7 +641,7 @@ TEST_F(TemporalLayersTest, KeyFrame) {
EXPECT_EQ(expected_flags[j], LibvpxVp8Encoder::EncodeFlags(tl_config))
<< j;
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config));
EXPECT_EQ(expected_temporal_idx[j], tl_config.packetizer_temporal_idx);
EXPECT_EQ(expected_temporal_idx[j], tl_config.encoder_layer_id);
@ -650,12 +650,11 @@ TEST_F(TemporalLayersTest, KeyFrame) {
}
CodecSpecificInfo info;
CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8;
Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
&vp8_info);
EXPECT_TRUE(vp8_info.layerSync) << "Key frame should be marked layer sync.";
EXPECT_EQ(0, vp8_info.temporalIdx)
tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, &info);
EXPECT_TRUE(info.codecSpecific.VP8.layerSync)
<< "Key frame should be marked layer sync.";
EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx)
<< "Key frame should always be packetized as layer 0";
EXPECT_TRUE(checker.CheckTemporalConfig(true, tl_config));
}
@ -741,7 +740,7 @@ TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) {
for (int i = 0; i < kMaxPatternLength; ++i) {
Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp_);
tl.OnEncodeDone(timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
++timestamp_;
EXPECT_FALSE(tl_config.drop_frame);
tl_configs.push_back(tl_config);

9
modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc
View File

@ -850,15 +850,16 @@ void LibvpxVp8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
uint32_t timestamp) {
assert(codec_specific != NULL);
codec_specific->codecType = kVideoCodecVP8;
CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8);
vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this
vp8Info->nonReference = (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0;
codec_specific->codecSpecific.VP8.keyIdx =
kNoKeyIdx; // TODO(hlundin) populate this
codec_specific->codecSpecific.VP8.nonReference =
(pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0;
int qp = 0;
vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
temporal_layers_[stream_idx]->OnEncodeDone(
timestamp, encoded_images_[encoder_idx].size(),
(pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, qp, vp8Info);
(pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, qp, codec_specific);
}
int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image) {

63
modules/video_coding/codecs/vp8/screenshare_layers.cc
View File

@ -265,7 +265,7 @@ void ScreenshareLayers::OnEncodeDone(uint32_t rtp_timestamp,
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfoVP8* vp8_info) {
CodecSpecificInfo* info) {
if (size_bytes == 0) {
layers_[active_layer_].state = TemporalLayer::State::kDropped;
++stats_.num_overshoots_;
@ -283,44 +283,47 @@ void ScreenshareLayers::OnEncodeDone(uint32_t rtp_timestamp,
}
}
CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8;
if (number_of_temporal_layers_ == 1) {
vp8_info->temporalIdx = kNoTemporalIdx;
vp8_info->layerSync = false;
vp8_info.temporalIdx = kNoTemporalIdx;
vp8_info.layerSync = false;
} else {
int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(rtp_timestamp);
if (frame_config) {
vp8_info->temporalIdx = frame_config->packetizer_temporal_idx;
vp8_info->layerSync = frame_config->layer_sync;
vp8_info.temporalIdx = frame_config->packetizer_temporal_idx;
vp8_info.layerSync = frame_config->layer_sync;
} else {
RTC_DCHECK(is_keyframe);
}
if (is_keyframe) {
vp8_info->temporalIdx = 0;
vp8_info.temporalIdx = 0;
last_sync_timestamp_ = unwrapped_timestamp;
vp8_info->layerSync = true;
vp8_info.layerSync = true;
layers_[0].state = TemporalLayer::State::kKeyFrame;
layers_[1].state = TemporalLayer::State::kKeyFrame;
active_layer_ = 1;
info->template_structure =
GetTemplateStructure(number_of_temporal_layers_);
}
vp8_info->useExplicitDependencies = true;
RTC_DCHECK_EQ(vp8_info->referencedBuffersCount, 0u);
RTC_DCHECK_EQ(vp8_info->updatedBuffersCount, 0u);
vp8_info.useExplicitDependencies = true;
RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u);
RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u);
// Note that |frame_config| is not derefernced if |is_keyframe|,
// meaning it's never dereferenced if the optional may be unset.
for (int i = 0; i < static_cast<int>(Buffer::kCount); ++i) {
if (!is_keyframe && frame_config->References(static_cast<Buffer>(i))) {
RTC_DCHECK_LT(vp8_info->referencedBuffersCount,
RTC_DCHECK_LT(vp8_info.referencedBuffersCount,
arraysize(CodecSpecificInfoVP8::referencedBuffers));
vp8_info->referencedBuffers[vp8_info->referencedBuffersCount++] = i;
vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i;
}
if (is_keyframe || frame_config->Updates(static_cast<Buffer>(i))) {
RTC_DCHECK_LT(vp8_info->updatedBuffersCount,
RTC_DCHECK_LT(vp8_info.updatedBuffersCount,
arraysize(CodecSpecificInfoVP8::updatedBuffers));
vp8_info->updatedBuffers[vp8_info->updatedBuffersCount++] = i;
vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i;
}
}
}
@ -352,6 +355,38 @@ void ScreenshareLayers::OnEncodeDone(uint32_t rtp_timestamp,
}
}
TemplateStructure ScreenshareLayers::GetTemplateStructure(
int num_layers) const {
RTC_CHECK_LT(num_layers, 3);
RTC_CHECK_GT(num_layers, 0);
TemplateStructure template_structure;
template_structure.num_operating_points = num_layers;
using Builder = GenericFrameInfo::Builder;
switch (num_layers) {
case 1: {
template_structure.templates = {
Builder().T(0).Dtis("S").Build(),
Builder().T(0).Dtis("S").Fdiffs({1}).Build(),
};
return template_structure;
}
case 2: {
template_structure.templates = {
Builder().T(0).Dtis("SS").Build(),
Builder().T(0).Dtis("SS").Fdiffs({1}).Build(),
Builder().T(1).Dtis("-S").Fdiffs({1}).Build(),
};
return template_structure;
}
default:
RTC_NOTREACHED();
// To make the compiler happy!
return template_structure;
}
}
bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
RTC_DCHECK_EQ(1, active_layer_);
RTC_DCHECK_NE(-1, layers_[0].last_qp);

6
modules/video_coding/codecs/vp8/screenshare_layers.h
View File

@ -16,6 +16,7 @@
#include "api/video_codecs/vp8_frame_config.h"
#include "api/video_codecs/vp8_temporal_layers.h"
#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/utility/frame_dropper.h"
#include "rtc_base/rate_statistics.h"
#include "rtc_base/time_utils.h"
@ -52,7 +53,7 @@ class ScreenshareLayers : public Vp8TemporalLayers {
size_t size_bytes,
bool is_keyframe,
int qp,
CodecSpecificInfoVP8* vp8_info) override;
CodecSpecificInfo* info) override;
private:
enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync };
@ -77,6 +78,7 @@ class ScreenshareLayers : public Vp8TemporalLayers {
absl::optional<uint32_t> target_framerate_;
// Incoming framerate from capturer.
absl::optional<uint32_t> capture_framerate_;
// Tracks what framerate we actually encode, and drops frames on overshoot.
RateStatistics encode_framerate_;
bool bitrate_updated_;
@ -107,6 +109,8 @@ class ScreenshareLayers : public Vp8TemporalLayers {
} layers_[kMaxNumTemporalLayers];
void UpdateHistograms();
TemplateStructure GetTemplateStructure(int num_layers) const;
// Data for histogram statistics.
struct Stats {
int64_t first_frame_time_ms_ = -1;

90
modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc
View File

@ -80,8 +80,7 @@ class ScreenshareLayerTest : public ::testing::Test {
int flags = ConfigureFrame(base_sync);
if (flags != -1)
layers_->OnEncodeDone(timestamp_, frame_size_, base_sync, kDefaultQp,
&info->codecSpecific.VP8);
info);
return flags;
}
@ -133,10 +132,9 @@ class ScreenshareLayerTest : public ::testing::Test {
bool got_tl1 = false;
for (int i = 0; i < 10; ++i) {
CodecSpecificInfo info;
const CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8;
EXPECT_NE(-1, EncodeFrame(false, &info));
timestamp_ += kTimestampDelta5Fps;
if (vp8_info.temporalIdx == 0) {
if (info.codecSpecific.VP8.temporalIdx == 0) {
got_tl0 = true;
} else {
got_tl1 = true;
@ -164,9 +162,8 @@ class ScreenshareLayerTest : public ::testing::Test {
if (tl_config_.packetizer_temporal_idx != layer ||
(sync && *sync != tl_config_.layer_sync)) {
CodecSpecificInfo info;
CodecSpecificInfoVP8* vp8_info = &info.codecSpecific.VP8;
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp,
vp8_info);
&info);
timestamp_ += kTimestampDelta5Fps;
} else {
// Found frame from sought after layer.
@ -188,9 +185,9 @@ class ScreenshareLayerTest : public ::testing::Test {
Vp8EncoderConfig cfg_;
bool config_updated_;
CodecSpecificInfoVP8* IgnoredCodecSpecificInfoVp8() {
CodecSpecificInfo* IgnoredCodecSpecificInfo() {
ignored_codec_specific_info_ = absl::make_unique<CodecSpecificInfo>();
return &ignored_codec_specific_info_->codecSpecific.VP8;
return ignored_codec_specific_info_.get();
}
private:
@ -223,10 +220,10 @@ TEST_F(ScreenshareLayerTest, 2LayersPeriodicSync) {
const int kNumFrames = kSyncPeriodSeconds * kFrameRate * 2 - 1;
for (int i = 0; i < kNumFrames; ++i) {
CodecSpecificInfo info;
const CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8;
EncodeFrame(false, &info);
timestamp_ += kTimestampDelta5Fps;
if (vp8_info.temporalIdx == 1 && vp8_info.layerSync) {
if (info.codecSpecific.VP8.temporalIdx == 1 &&
info.codecSpecific.VP8.layerSync) {
sync_times.push_back(timestamp_);
}
}
@ -240,21 +237,20 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) {
const int kNumFrames = kMaxSyncPeriodSeconds * kFrameRate * 2 - 1;
for (int i = 0; i < kNumFrames; ++i) {
CodecSpecificInfo info;
CodecSpecificInfoVP8* vp8_info = &info.codecSpecific.VP8;
tl_config_ = UpdateLayerConfig(timestamp_);
config_updated_ = layers_->UpdateConfiguration(&cfg_);
// Simulate TL1 being at least 8 qp steps better.
if (tl_config_.packetizer_temporal_idx == 0) {
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp,
vp8_info);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &info);
} else {
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8,
vp8_info);
&info);
}
if (vp8_info->temporalIdx == 1 && vp8_info->layerSync)
if (info.codecSpecific.VP8.temporalIdx == 1 &&
info.codecSpecific.VP8.layerSync)
sync_times.push_back(timestamp_);
timestamp_ += kTimestampDelta5Fps;
@ -272,20 +268,19 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) {
kFrameRate;
for (int i = 0; i < kNumFrames; ++i) {
CodecSpecificInfo info;
CodecSpecificInfoVP8* vp8_info = &info.codecSpecific.VP8;
ConfigureFrame(false);
// Simulate TL1 being at least 8 qp steps better.
if (tl_config_.packetizer_temporal_idx == 0) {
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp,
vp8_info);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &info);
} else {
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8,
vp8_info);
&info);
}
if (vp8_info->temporalIdx == 1 && vp8_info->layerSync)
if (info.codecSpecific.VP8.temporalIdx == 1 &&
info.codecSpecific.VP8.layerSync)
sync_times.push_back(timestamp_);
timestamp_ += kTimestampDelta5Fps;
@ -296,17 +291,16 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) {
bool bumped_tl0_quality = false;
for (int i = 0; i < 3; ++i) {
CodecSpecificInfo info;
CodecSpecificInfoVP8* vp8_info = &info.codecSpecific.VP8;
int flags = ConfigureFrame(false);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8,
vp8_info);
if (vp8_info->temporalIdx == 0) {
&info);
if (info.codecSpecific.VP8.temporalIdx == 0) {
// Bump TL0 to same quality as TL1.
bumped_tl0_quality = true;
} else {
if (bumped_tl0_quality) {
EXPECT_TRUE(vp8_info->layerSync);
EXPECT_TRUE(info.codecSpecific.VP8.layerSync);
EXPECT_EQ(kTl1SyncFlags, flags);
return;
}
@ -324,10 +318,9 @@ TEST_F(ScreenshareLayerTest, 2LayersToggling) {
int tl1_frames = 0;
for (int i = 0; i < 50; ++i) {
CodecSpecificInfo info;
const CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8;
EncodeFrame(false, &info);
timestamp_ += kTimestampDelta5Fps;
switch (vp8_info.temporalIdx) {
switch (info.codecSpecific.VP8.temporalIdx) {
case 0:
++tl0_frames;
break;
@ -348,11 +341,10 @@ TEST_F(ScreenshareLayerTest, AllFitsLayer0) {
// Insert 50 frames, small enough that all fits in TL0.
for (int i = 0; i < 50; ++i) {
CodecSpecificInfo info;
const CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8;
int flags = EncodeFrame(false, &info);
timestamp_ += kTimestampDelta5Fps;
EXPECT_EQ(kTl0Flags, flags);
EXPECT_EQ(0, vp8_info.temporalIdx);
EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx);
}
}
@ -365,13 +357,12 @@ TEST_F(ScreenshareLayerTest, TooHighBitrate) {
int dropped_frames = 0;
for (int i = 0; i < 100; ++i) {
CodecSpecificInfo info;
const CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8;
int flags = EncodeFrame(false, &info);
timestamp_ += kTimestampDelta5Fps;
if (flags == -1) {
++dropped_frames;
} else {
switch (vp8_info.temporalIdx) {
switch (info.codecSpecific.VP8.temporalIdx) {
case 0:
++tl0_frames;
break;
@ -428,7 +419,7 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) {
SkipUntilTl(0);
// Size 0 indicates dropped frame.
layers_->OnEncodeDone(timestamp_, 0, false, 0, IgnoredCodecSpecificInfoVp8());
layers_->OnEncodeDone(timestamp_, 0, false, 0, IgnoredCodecSpecificInfo());
// Re-encode frame (so don't advance timestamp).
int flags = EncodeFrame(false);
@ -441,19 +432,19 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) {
EXPECT_TRUE(config_updated_);
EXPECT_LT(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
timestamp_ += kTimestampDelta5Fps;
// ...then back to standard setup.
SkipUntilTl(0);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
timestamp_ += kTimestampDelta5Fps;
EXPECT_EQ(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
// Next drop in TL1.
SkipUntilTl(1);
layers_->OnEncodeDone(timestamp_, 0, false, 0, IgnoredCodecSpecificInfoVp8());
layers_->OnEncodeDone(timestamp_, 0, false, 0, IgnoredCodecSpecificInfo());
// Re-encode frame (so don't advance timestamp).
flags = EncodeFrame(false);
@ -466,14 +457,14 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) {
EXPECT_TRUE(config_updated_);
EXPECT_LT(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
timestamp_ += kTimestampDelta5Fps;
// ...and back to normal.
SkipUntilTl(1);
EXPECT_EQ(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
timestamp_ += kTimestampDelta5Fps;
}
@ -489,7 +480,7 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) {
EXPECT_EQ(kTl0Flags,
LibvpxVp8Encoder::EncodeFlags(UpdateLayerConfig(kStartTimestamp)));
layers_->OnEncodeDone(kStartTimestamp, kLargeFrameSizeBytes, false,
kDefaultQp, IgnoredCodecSpecificInfoVp8());
kDefaultQp, IgnoredCodecSpecificInfo());
const uint32_t kTwoSecondsLater =
kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90);
@ -539,15 +530,15 @@ TEST_F(ScreenshareLayerTest, UpdatesHistograms) {
if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) {
// Simulate a too large frame, to cause frame drop.
layers_->OnEncodeDone(timestamp, frame_size_ * 10, false, kTl0Qp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
trigger_drop = true;
} else {
layers_->OnEncodeDone(timestamp, frame_size_, false, kTl0Qp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
}
} else if (flags == kTl1Flags || flags == kTl1SyncFlags) {
layers_->OnEncodeDone(timestamp, frame_size_, false, kTl1Qp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
} else if (flags == -1) {
dropped_frame = true;
} else {
@ -614,7 +605,7 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) {
} else {
size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8;
layers_->OnEncodeDone(timestamp, frame_size_bytes, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
}
timestamp += kFrameIntervalsMs * 90;
clock_.AdvanceTime(TimeDelta::ms(kFrameIntervalsMs));
@ -632,7 +623,7 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) {
} else {
size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8;
layers_->OnEncodeDone(timestamp, frame_size_bytes, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
}
timestamp += kFrameIntervalsMs * 90 / 2;
clock_.AdvanceTime(TimeDelta::ms(kFrameIntervalsMs));
@ -657,10 +648,9 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAtOvershootDrop) {
config_updated_ = layers_->UpdateConfiguration(&cfg_);
EXPECT_EQ(kTl1SyncFlags, LibvpxVp8Encoder::EncodeFlags(tl_config_));
CodecSpecificInfo info;
CodecSpecificInfoVP8* vp8_info = &info.codecSpecific.VP8;
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, vp8_info);
EXPECT_TRUE(vp8_info->layerSync);
CodecSpecificInfo new_info;
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &new_info);
EXPECT_TRUE(new_info.codecSpecific.VP8.layerSync);
}
TEST_F(ScreenshareLayerTest, DropOnTooShortFrameInterval) {
@ -671,7 +661,7 @@ TEST_F(ScreenshareLayerTest, DropOnTooShortFrameInterval) {
timestamp_ += kTimestampDelta5Fps * 3;
EXPECT_FALSE(UpdateLayerConfig(timestamp_).drop_frame);
layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// Frame interval below 90% if desired time is not allowed, try inserting
// frame just before this limit.
@ -735,7 +725,7 @@ TEST_F(ScreenshareLayerTest, DISABLED_MaxQpRestoredAfterDoubleDrop) {
// Simulate re-encoded frame.
layers_->OnEncodeDone(timestamp_, 1, false, max_qp_,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// Next frame, expect boosted quality.
// Slightly alter bitrate between each frame.
@ -752,7 +742,7 @@ TEST_F(ScreenshareLayerTest, DISABLED_MaxQpRestoredAfterDoubleDrop) {
// Simulate re-encoded frame.
layers_->OnEncodeDone(timestamp_, frame_size_, false, max_qp_,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// A third frame, expect boosted quality.
layers_->OnRatesUpdated(kDefault2TlBitratesBps, kFrameRate);
@ -763,7 +753,7 @@ TEST_F(ScreenshareLayerTest, DISABLED_MaxQpRestoredAfterDoubleDrop) {
// Frame encoded.
layers_->OnEncodeDone(timestamp_, frame_size_, false, max_qp_,
IgnoredCodecSpecificInfoVp8());
IgnoredCodecSpecificInfo());
// A fourth frame, max qp should be restored.
layers_->OnRatesUpdated(kDefault2TlBitratesBpsAlt, kFrameRate);

20
modules/video_coding/include/video_codec_interface.cc Normal file
View File

@ -0,0 +1,20 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/include/video_codec_interface.h"
namespace webrtc {
CodecSpecificInfo::CodecSpecificInfo() : codecType(kVideoCodecGeneric) {
memset(&codecSpecific, 0, sizeof(codecSpecific));
}
CodecSpecificInfo::CodecSpecificInfo(const CodecSpecificInfo&) = default;
CodecSpecificInfo::~CodecSpecificInfo() = default;
} // namespace webrtc

12
modules/video_coding/include/video_codec_interface.h
View File

@ -13,10 +13,12 @@
#include <vector>
#include "absl/types/optional.h"
#include "api/video/video_frame.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_encoder.h"
#include "common_types.h" // NOLINT(build/include)
#include "common_video/generic_frame_descriptor/generic_frame_info.h"
#include "modules/include/module_common_types.h"
#include "modules/video_coding/include/video_error_codes.h"
@ -96,16 +98,18 @@ union CodecSpecificInfoUnion {
};
static_assert(std::is_pod<CodecSpecificInfoUnion>::value, "");
// Note: If any pointers are added to this struct or its sub-structs, it
// Note: if any pointers are added to this struct or its sub-structs, it
// must be fitted with a copy-constructor. This is because it is copied
// in the copy-constructor of VCMEncodedFrame.
struct CodecSpecificInfo {
CodecSpecificInfo() : codecType(kVideoCodecGeneric) {
memset(&codecSpecific, 0, sizeof(codecSpecific));
}
CodecSpecificInfo();
CodecSpecificInfo(const CodecSpecificInfo&);
~CodecSpecificInfo();
VideoCodecType codecType;
CodecSpecificInfoUnion codecSpecific;
absl::optional<GenericFrameInfo> generic_frame_info;
absl::optional<TemplateStructure> template_structure;
};
} // namespace webrtc

2
modules/video_coding/utility/simulcast_rate_allocator_unittest.cc
View File

@ -41,7 +41,7 @@ class MockTemporalLayers : public Vp8TemporalLayers {
MOCK_METHOD2(OnRatesUpdated, void(const std::vector<uint32_t>&, int));
MOCK_METHOD1(UpdateConfiguration, bool(Vp8EncoderConfig*));
MOCK_METHOD5(OnEncodeDone,
void(uint32_t, size_t, bool, int, CodecSpecificInfoVP8*));
void(uint32_t, size_t, bool, int, CodecSpecificInfo*));
MOCK_METHOD3(FrameEncoded, void(uint32_t, size_t, int));
MOCK_CONST_METHOD0(Tl0PicIdx, uint8_t());
MOCK_CONST_METHOD1(GetTemporalLayerId, int(const Vp8FrameConfig&));

7
test/fake_vp8_encoder.cc
View File

@ -106,11 +106,10 @@ void FakeVP8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
uint32_t timestamp) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
codec_specific->codecType = kVideoCodecVP8;
CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8);
vp8Info->keyIdx = kNoKeyIdx;
vp8Info->nonReference = false;
codec_specific->codecSpecific.VP8.keyIdx = kNoKeyIdx;
codec_specific->codecSpecific.VP8.nonReference = false;
temporal_layers_[stream_idx]->OnEncodeDone(
timestamp, size_bytes, frame_type == kVideoFrameKey, -1, vp8Info);
timestamp, size_bytes, frame_type == kVideoFrameKey, -1, codec_specific);
}
EncodedImageCallback::Result FakeVP8Encoder::OnEncodedImage(