zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merged FrameBuffer3 {Next,Last}DecodableTemporalUnitRtpTimestamp() function.

Browse Source 
Bug: webrtc:13343
Change-Id: Ic21eddd38466e6b5fd8b912b3ee2d9dc47a0ba35
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/260981
Reviewed-by: Evan Shrubsole <eshr@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#36756}
This commit is contained in:
philipel
2022-05-03 15:21:34 +02:00
committed by WebRTC LUCI CQ
parent 44be579b4a
commit 1bcd827e9b
5 changed files with 52 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
modules/video_coding/frame_buffer3.cc
View File

@ -11,9 +11,6 @@
#include "modules/video_coding/frame_buffer3.h" #include "modules/video_coding/frame_buffer3.h"
#include <algorithm> #include <algorithm>
#include <iterator>
#include <queue>
#include <utility>
#include "absl/algorithm/container.h" #include "absl/algorithm/container.h"
#include "absl/container/inlined_vector.h" #include "absl/container/inlined_vector.h"
@ -52,7 +49,7 @@ int64_t GetFrameId(const FrameIteratorT& it) {
} }
template <typename FrameIteratorT> template <typename FrameIteratorT>
int64_t GetTimestamp(const FrameIteratorT& it) { uint32_t GetTimestamp(const FrameIteratorT& it) {
return it->second.encoded_frame->Timestamp(); return it->second.encoded_frame->Timestamp();
} }
@ -158,17 +155,9 @@ absl::optional<int64_t> FrameBuffer::LastContinuousTemporalUnitFrameId() const {
return last_continuous_temporal_unit_frame_id_; return last_continuous_temporal_unit_frame_id_;
} }
absl::optional<uint32_t> FrameBuffer::NextDecodableTemporalUnitRtpTimestamp() absl::optional<FrameBuffer::DecodabilityInfo>
const { FrameBuffer::DecodableTemporalUnitsInfo() const {
if (!next_decodable_temporal_unit_) { return decodable_temporal_units_info_;
return absl::nullopt;
}
return GetTimestamp(next_decodable_temporal_unit_->first_frame);
}
absl::optional<uint32_t> FrameBuffer::LastDecodableTemporalUnitRtpTimestamp()
const {
return last_decodable_temporal_unit_timestamp_;
} }
int FrameBuffer::GetTotalNumberOfContinuousTemporalUnits() const { int FrameBuffer::GetTotalNumberOfContinuousTemporalUnits() const {
@ -221,7 +210,7 @@ void FrameBuffer::PropagateContinuity(const FrameIterator& frame_it) {
void FrameBuffer::FindNextAndLastDecodableTemporalUnit() { void FrameBuffer::FindNextAndLastDecodableTemporalUnit() {
next_decodable_temporal_unit_.reset(); next_decodable_temporal_unit_.reset();
last_decodable_temporal_unit_timestamp_.reset(); decodable_temporal_units_info_.reset();
if (!last_continuous_temporal_unit_frame_id_) { if (!last_continuous_temporal_unit_frame_id_) {
return; return;
@ -230,6 +219,7 @@ void FrameBuffer::FindNextAndLastDecodableTemporalUnit() {
FrameIterator first_frame_it = frames_.begin(); FrameIterator first_frame_it = frames_.begin();
FrameIterator last_frame_it = frames_.begin(); FrameIterator last_frame_it = frames_.begin();
absl::InlinedVector<int64_t, 4> frames_in_temporal_unit; absl::InlinedVector<int64_t, 4> frames_in_temporal_unit;
uint32_t last_decodable_temporal_unit_timestamp;
for (auto frame_it = frames_.begin(); frame_it != frames_.end();) { for (auto frame_it = frames_.begin(); frame_it != frames_.end();) {
if (GetFrameId(frame_it) > *last_continuous_temporal_unit_frame_id_) { if (GetFrameId(frame_it) > *last_continuous_temporal_unit_frame_id_) {
break; break;
@ -264,16 +254,23 @@ void FrameBuffer::FindNextAndLastDecodableTemporalUnit() {
next_decodable_temporal_unit_ = {first_frame_it, last_frame_it}; next_decodable_temporal_unit_ = {first_frame_it, last_frame_it};
} }
last_decodable_temporal_unit_timestamp_ = GetTimestamp(first_frame_it); last_decodable_temporal_unit_timestamp = GetTimestamp(first_frame_it);
} }
} }
} }
if (next_decodable_temporal_unit_) {
decodable_temporal_units_info_ = {
.next_rtp_timestamp =
GetTimestamp(next_decodable_temporal_unit_->first_frame),
.last_rtp_timestamp = last_decodable_temporal_unit_timestamp};
}
} }
void FrameBuffer::Clear() { void FrameBuffer::Clear() {
frames_.clear(); frames_.clear();
next_decodable_temporal_unit_.reset(); next_decodable_temporal_unit_.reset();
last_decodable_temporal_unit_timestamp_.reset(); decodable_temporal_units_info_.reset();
last_continuous_frame_id_.reset(); last_continuous_frame_id_.reset();
last_continuous_temporal_unit_frame_id_.reset(); last_continuous_temporal_unit_frame_id_.reset();
decoded_frame_history_.Clear(); decoded_frame_history_.Clear();

10
modules/video_coding/frame_buffer3.h
View File

@ -31,6 +31,11 @@ namespace webrtc {
// The FrameBuffer is thread-unsafe. // The FrameBuffer is thread-unsafe.
class FrameBuffer { class FrameBuffer {
public: public:
struct DecodabilityInfo {
uint32_t next_rtp_timestamp;
uint32_t last_rtp_timestamp;
};
// The `max_size` determines the maxmimum number of frames the buffer will // The `max_size` determines the maxmimum number of frames the buffer will
// store, and max_decode_history determines how far back (by frame ID) the // store, and max_decode_history determines how far back (by frame ID) the
// buffer will store if a frame was decoded or not. // buffer will store if a frame was decoded or not.
@ -56,8 +61,7 @@ class FrameBuffer {
absl::optional<int64_t> LastContinuousFrameId() const; absl::optional<int64_t> LastContinuousFrameId() const;
absl::optional<int64_t> LastContinuousTemporalUnitFrameId() const; absl::optional<int64_t> LastContinuousTemporalUnitFrameId() const;
absl::optional<uint32_t> NextDecodableTemporalUnitRtpTimestamp() const; absl::optional<DecodabilityInfo> DecodableTemporalUnitsInfo() const;
absl::optional<uint32_t> LastDecodableTemporalUnitRtpTimestamp() const;
int GetTotalNumberOfContinuousTemporalUnits() const; int GetTotalNumberOfContinuousTemporalUnits() const;
int GetTotalNumberOfDroppedFrames() const; int GetTotalNumberOfDroppedFrames() const;
@ -87,7 +91,7 @@ class FrameBuffer {
const size_t max_size_; const size_t max_size_;
FrameMap frames_; FrameMap frames_;
absl::optional<TemporalUnit> next_decodable_temporal_unit_; absl::optional<TemporalUnit> next_decodable_temporal_unit_;
absl::optional<uint32_t> last_decodable_temporal_unit_timestamp_; absl::optional<DecodabilityInfo> decodable_temporal_units_info_;
absl::optional<int64_t> last_continuous_frame_id_; absl::optional<int64_t> last_continuous_frame_id_;
absl::optional<int64_t> last_continuous_temporal_unit_frame_id_; absl::optional<int64_t> last_continuous_temporal_unit_frame_id_;
video_coding::DecodedFramesHistory decoded_frame_history_; video_coding::DecodedFramesHistory decoded_frame_history_;

19
modules/video_coding/frame_buffer3_unittest.cc
View File

@ -111,10 +111,9 @@ TEST(FrameBuffer3Test, NextDecodable) {
FrameBuffer buffer(/*max_frame_slots=*/10, /*max_decode_history=*/100, FrameBuffer buffer(/*max_frame_slots=*/10, /*max_decode_history=*/100,
field_trials); field_trials);
EXPECT_THAT(buffer.NextDecodableTemporalUnitRtpTimestamp(), EXPECT_THAT(buffer.DecodableTemporalUnitsInfo(), Eq(absl::nullopt));
Eq(absl::nullopt));
buffer.InsertFrame(test::FakeFrameBuilder().Time(10).Id(1).AsLast().Build()); buffer.InsertFrame(test::FakeFrameBuilder().Time(10).Id(1).AsLast().Build());
EXPECT_THAT(buffer.NextDecodableTemporalUnitRtpTimestamp(), Eq(10U)); EXPECT_THAT(buffer.DecodableTemporalUnitsInfo()->next_rtp_timestamp, Eq(10U));
} }
TEST(FrameBuffer3Test, AdvanceNextDecodableOnExtraction) { TEST(FrameBuffer3Test, AdvanceNextDecodableOnExtraction) {
@ -126,14 +125,14 @@ TEST(FrameBuffer3Test, AdvanceNextDecodableOnExtraction) {
buffer.InsertFrame(test::FakeFrameBuilder().Time(20).Id(2).AsLast().Build()); buffer.InsertFrame(test::FakeFrameBuilder().Time(20).Id(2).AsLast().Build());
buffer.InsertFrame( buffer.InsertFrame(
test::FakeFrameBuilder().Time(30).Id(3).Refs({2}).AsLast().Build()); test::FakeFrameBuilder().Time(30).Id(3).Refs({2}).AsLast().Build());
EXPECT_THAT(buffer.NextDecodableTemporalUnitRtpTimestamp(), Eq(10U)); EXPECT_THAT(buffer.DecodableTemporalUnitsInfo()->next_rtp_timestamp, Eq(10U));
EXPECT_THAT(buffer.ExtractNextDecodableTemporalUnit(), EXPECT_THAT(buffer.ExtractNextDecodableTemporalUnit(),
ElementsAre(FrameWithId(1))); ElementsAre(FrameWithId(1)));
EXPECT_THAT(buffer.NextDecodableTemporalUnitRtpTimestamp(), Eq(20U)); EXPECT_THAT(buffer.DecodableTemporalUnitsInfo()->next_rtp_timestamp, Eq(20U));
EXPECT_THAT(buffer.ExtractNextDecodableTemporalUnit(), EXPECT_THAT(buffer.ExtractNextDecodableTemporalUnit(),
ElementsAre(FrameWithId(2))); ElementsAre(FrameWithId(2)));
EXPECT_THAT(buffer.NextDecodableTemporalUnitRtpTimestamp(), Eq(30U)); EXPECT_THAT(buffer.DecodableTemporalUnitsInfo()->next_rtp_timestamp, Eq(30U));
EXPECT_THAT(buffer.ExtractNextDecodableTemporalUnit(), EXPECT_THAT(buffer.ExtractNextDecodableTemporalUnit(),
ElementsAre(FrameWithId(3))); ElementsAre(FrameWithId(3)));
} }
@ -148,11 +147,11 @@ TEST(FrameBuffer3Test, AdvanceLastDecodableOnExtraction) {
test::FakeFrameBuilder().Time(20).Id(2).Refs({1}).AsLast().Build()); test::FakeFrameBuilder().Time(20).Id(2).Refs({1}).AsLast().Build());
buffer.InsertFrame( buffer.InsertFrame(
test::FakeFrameBuilder().Time(30).Id(3).Refs({1}).AsLast().Build()); test::FakeFrameBuilder().Time(30).Id(3).Refs({1}).AsLast().Build());
EXPECT_THAT(buffer.LastDecodableTemporalUnitRtpTimestamp(), Eq(10U)); EXPECT_THAT(buffer.DecodableTemporalUnitsInfo()->last_rtp_timestamp, Eq(10U));
EXPECT_THAT(buffer.ExtractNextDecodableTemporalUnit(), EXPECT_THAT(buffer.ExtractNextDecodableTemporalUnit(),
ElementsAre(FrameWithId(1))); ElementsAre(FrameWithId(1)));
EXPECT_THAT(buffer.LastDecodableTemporalUnitRtpTimestamp(), Eq(30U)); EXPECT_THAT(buffer.DecodableTemporalUnitsInfo()->last_rtp_timestamp, Eq(30U));
} }
TEST(FrameBuffer3Test, FrameUpdatesNextDecodable) { TEST(FrameBuffer3Test, FrameUpdatesNextDecodable) {
@ -161,10 +160,10 @@ TEST(FrameBuffer3Test, FrameUpdatesNextDecodable) {
field_trials); field_trials);
buffer.InsertFrame(test::FakeFrameBuilder().Time(20).Id(2).AsLast().Build()); buffer.InsertFrame(test::FakeFrameBuilder().Time(20).Id(2).AsLast().Build());
EXPECT_THAT(buffer.NextDecodableTemporalUnitRtpTimestamp(), Eq(20U)); EXPECT_THAT(buffer.DecodableTemporalUnitsInfo()->next_rtp_timestamp, Eq(20U));
buffer.InsertFrame(test::FakeFrameBuilder().Time(10).Id(1).AsLast().Build()); buffer.InsertFrame(test::FakeFrameBuilder().Time(10).Id(1).AsLast().Build());
EXPECT_THAT(buffer.NextDecodableTemporalUnitRtpTimestamp(), Eq(10U)); EXPECT_THAT(buffer.DecodableTemporalUnitsInfo()->next_rtp_timestamp, Eq(10U));
} }
TEST(FrameBuffer3Test, KeyframeClearsFullBuffer) { TEST(FrameBuffer3Test, KeyframeClearsFullBuffer) {

12
test/fuzzers/frame_buffer3_fuzzer.cc
View File

@ -39,7 +39,7 @@ void FuzzOneInput(const uint8_t* data, size_t size) {
SeqNumUnwrapper<uint16_t, kFrameIdLength> unwrapper; SeqNumUnwrapper<uint16_t, kFrameIdLength> unwrapper;
while (helper.BytesLeft() > 0) { while (helper.BytesLeft() > 0) {
int action = helper.ReadOrDefaultValue<uint8_t>(0) % 7; int action = helper.ReadOrDefaultValue<uint8_t>(0) % 6;
switch (action) { switch (action) {
case 0: { case 0: {
@ -51,22 +51,18 @@ void FuzzOneInput(const uint8_t* data, size_t size) {
break; break;
} }
case 2: { case 2: {
buffer.NextDecodableTemporalUnitRtpTimestamp(); buffer.DecodableTemporalUnitsInfo();
break; break;
} }
case 3: { case 3: {
buffer.LastDecodableTemporalUnitRtpTimestamp();
break;
}
case 4: {
buffer.ExtractNextDecodableTemporalUnit(); buffer.ExtractNextDecodableTemporalUnit();
break; break;
} }
case 5: { case 4: {
buffer.DropNextDecodableTemporalUnit(); buffer.DropNextDecodableTemporalUnit();
break; break;
} }
case 6: { case 5: {
auto frame = std::make_unique<FuzzyFrameObject>(); auto frame = std::make_unique<FuzzyFrameObject>();
frame->SetTimestamp(helper.ReadOrDefaultValue<uint32_t>(0)); frame->SetTimestamp(helper.ReadOrDefaultValue<uint32_t>(0));
int64_t wire_id = int64_t wire_id =

37
video/frame_buffer_proxy.cc
View File

@ -384,13 +384,11 @@ class FrameBuffer3Proxy : public FrameBufferProxy {
private: private:
void FrameReadyForDecode(uint32_t rtp_timestamp, Timestamp render_time) { void FrameReadyForDecode(uint32_t rtp_timestamp, Timestamp render_time) {
RTC_DCHECK_RUN_ON(&worker_sequence_checker_); RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
RTC_DCHECK(buffer_->NextDecodableTemporalUnitRtpTimestamp() == auto frames = buffer_->ExtractNextDecodableTemporalUnit();
rtp_timestamp) RTC_DCHECK(frames[0]->Timestamp() == rtp_timestamp)
<< "Frame buffer's next decodable frame was not the one sent for " << "Frame buffer's next decodable frame was not the one sent for "
"extraction rtp=" "extraction rtp="
<< rtp_timestamp << " next=" << rtp_timestamp << " extracted rtp=" << frames[0]->Timestamp();
<< buffer_->NextDecodableTemporalUnitRtpTimestamp().value_or(-1);
auto frames = buffer_->ExtractNextDecodableTemporalUnit();
OnFrameReady(std::move(frames), render_time); OnFrameReady(std::move(frames), render_time);
} }
@ -431,7 +429,7 @@ class FrameBuffer3Proxy : public FrameBufferProxy {
RTC_DCHECK(keyframe_required_); RTC_DCHECK(keyframe_required_);
// Iterate through the frame buffer until there is a complete keyframe and // Iterate through the frame buffer until there is a complete keyframe and
// release this right away. // release this right away.
while (buffer_->NextDecodableTemporalUnitRtpTimestamp()) { while (buffer_->DecodableTemporalUnitsInfo()) {
auto next_frame = buffer_->ExtractNextDecodableTemporalUnit(); auto next_frame = buffer_->ExtractNextDecodableTemporalUnit();
if (next_frame.empty()) { if (next_frame.empty()) {
RTC_DCHECK_NOTREACHED() RTC_DCHECK_NOTREACHED()
@ -449,41 +447,40 @@ class FrameBuffer3Proxy : public FrameBufferProxy {
} }
void MaybeScheduleFrameForRelease() RTC_RUN_ON(&worker_sequence_checker_) { void MaybeScheduleFrameForRelease() RTC_RUN_ON(&worker_sequence_checker_) {
if (!decoder_ready_for_new_frame_ || auto decodable_tu_info = buffer_->DecodableTemporalUnitsInfo();
!buffer_->NextDecodableTemporalUnitRtpTimestamp()) if (!decoder_ready_for_new_frame_ || !decodable_tu_info) {
return; return;
}
if (keyframe_required_) { if (keyframe_required_) {
return ForceKeyFrameReleaseImmediately(); return ForceKeyFrameReleaseImmediately();
} }
// TODO(https://bugs.webrtc.org/13343): Make [next,last] decodable returned
// as an optional pair and remove this check.
RTC_CHECK(buffer_->LastDecodableTemporalUnitRtpTimestamp());
auto last_rtp = *buffer_->LastDecodableTemporalUnitRtpTimestamp();
// If already scheduled then abort. // If already scheduled then abort.
if (frame_decode_scheduler_->ScheduledRtpTimestamp() == if (frame_decode_scheduler_->ScheduledRtpTimestamp() ==
buffer_->NextDecodableTemporalUnitRtpTimestamp()) decodable_tu_info->next_rtp_timestamp) {
return; return;
}
absl::optional<FrameDecodeTiming::FrameSchedule> schedule; absl::optional<FrameDecodeTiming::FrameSchedule> schedule;
while (buffer_->NextDecodableTemporalUnitRtpTimestamp()) { while (decodable_tu_info) {
auto next_rtp = *buffer_->NextDecodableTemporalUnitRtpTimestamp(); schedule = decode_timing_.OnFrameBufferUpdated(
schedule = decode_timing_.OnFrameBufferUpdated(next_rtp, last_rtp, decodable_tu_info->next_rtp_timestamp,
IsTooManyFramesQueued()); decodable_tu_info->last_rtp_timestamp, IsTooManyFramesQueued());
if (schedule) { if (schedule) {
// Don't schedule if already waiting for the same frame. // Don't schedule if already waiting for the same frame.
if (frame_decode_scheduler_->ScheduledRtpTimestamp() != next_rtp) { if (frame_decode_scheduler_->ScheduledRtpTimestamp() !=
decodable_tu_info->next_rtp_timestamp) {
frame_decode_scheduler_->CancelOutstanding(); frame_decode_scheduler_->CancelOutstanding();
frame_decode_scheduler_->ScheduleFrame( frame_decode_scheduler_->ScheduleFrame(
next_rtp, *schedule, decodable_tu_info->next_rtp_timestamp, *schedule,
absl::bind_front(&FrameBuffer3Proxy::FrameReadyForDecode, this)); absl::bind_front(&FrameBuffer3Proxy::FrameReadyForDecode, this));
} }
return; return;
} }
// If no schedule for current rtp, drop and try again. // If no schedule for current rtp, drop and try again.
buffer_->DropNextDecodableTemporalUnit(); buffer_->DropNextDecodableTemporalUnit();
decodable_tu_info = buffer_->DecodableTemporalUnitsInfo();
} }
} }