zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added VideoContentType to OnDecodedFrame callback.

Browse Source 
Also added a FrameInfo struct to propagate various meta information along side it.

Bug: webrtc:9106
Change-Id: I1feb9f94c662c367f7c6e0a50d33705fdd5346bb
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/183880
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32079}
This commit is contained in:
philipel
2020-09-10 15:30:02 +02:00
committed by Commit Bot
parent 673027b4a5
commit 3dc4780d8e
4 changed files with 51 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
api/video/BUILD.gn
View File

@ -222,6 +222,7 @@ rtc_source_set("video_stream_decoder") {
deps = [
":encoded_frame",
":video_frame",
":video_rtp_headers",
"../task_queue",
"../units:time_delta",
"../video_codecs:video_codecs_api",

17
api/video/video_stream_decoder.h
View File

@ -17,6 +17,7 @@
#include "api/units/time_delta.h"
#include "api/video/encoded_frame.h"
#include "api/video/video_content_type.h"
#include "api/video/video_frame.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder_factory.h"
@ -29,6 +30,12 @@ class VideoStreamDecoderInterface {
public:
virtual ~Callbacks() = default;
struct FrameInfo {
absl::optional<int> decode_time_ms;
absl::optional<int> qp;
VideoContentType content_type;
};
// Called when the VideoStreamDecoder enters a non-decodable state.
virtual void OnNonDecodableState() = 0;
@ -37,9 +44,15 @@ class VideoStreamDecoderInterface {
const video_coding::VideoLayerFrameId& key) = 0;
// Called with the decoded frame.
virtual void OnDecodedFrame(VideoFrame decodedImage,
virtual void OnDecodedFrame(VideoFrame frame,
absl::optional<int> decode_time_ms,
absl::optional<int> qp) = 0;
absl::optional<int> qp) {}
// TODO(philipel): Make pure virtual and remove old function above when
// it is no longer used.
virtual void OnDecodedFrame(VideoFrame frame, const FrameInfo& meta_info) {
OnDecodedFrame(std::move(frame), meta_info.decode_time_ms, meta_info.qp);
}
};
virtual ~VideoStreamDecoderInterface() = default;

55
video/video_stream_decoder_impl.cc
View File

@ -26,7 +26,7 @@ VideoStreamDecoderImpl::VideoStreamDecoderImpl(
std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings)
: timing_(Clock::GetRealTimeClock()),
decode_callbacks_(this),
next_frame_timestamps_index_(0),
next_frame_info_index_(0),
callbacks_(callbacks),
keyframe_required_(true),
decoder_factory_(decoder_factory),
@ -39,7 +39,6 @@ VideoStreamDecoderImpl::VideoStreamDecoderImpl(
decode_queue_(task_queue_factory->CreateTaskQueue(
"video_stream_decoder_decode_queue",
TaskQueueFactory::Priority::NORMAL)) {
frame_timestamps_.fill({-1, -1, -1});
bookkeeping_queue_.PostTask([this]() {
RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
StartNextDecode();
@ -125,16 +124,15 @@ VideoDecoder* VideoStreamDecoderImpl::GetDecoder(int payload_type) {
return decoder_.get();
}
void VideoStreamDecoderImpl::SaveFrameTimestamps(
void VideoStreamDecoderImpl::SaveFrameInfo(
const video_coding::EncodedFrame& frame) {
FrameTimestamps* frame_timestamps =
&frame_timestamps_[next_frame_timestamps_index_];
frame_timestamps->timestamp = frame.Timestamp();
frame_timestamps->decode_start_time_ms = rtc::TimeMillis();
frame_timestamps->render_time_us = frame.RenderTimeMs() * 1000;
FrameInfo* frame_info = &frame_info_[next_frame_info_index_];
frame_info->timestamp = frame.Timestamp();
frame_info->decode_start_time_ms = rtc::TimeMillis();
frame_info->render_time_us = frame.RenderTimeMs() * 1000;
frame_info->content_type = frame.EncodedImage().content_type_;
next_frame_timestamps_index_ =
Add<kFrameTimestampsMemory>(next_frame_timestamps_index_, 1);
next_frame_info_index_ = Add<kFrameInfoMemory>(next_frame_info_index_, 1);
}
void VideoStreamDecoderImpl::StartNextDecode() {
@ -155,7 +153,7 @@ void VideoStreamDecoderImpl::OnNextFrameCallback(
switch (result) {
case video_coding::FrameBuffer::kFrameFound: {
RTC_DCHECK(frame);
SaveFrameTimestamps(*frame);
SaveFrameInfo(*frame);
MutexLock lock(&shut_down_mutex_);
if (shut_down_) {
@ -230,14 +228,14 @@ VideoStreamDecoderImpl::DecodeResult VideoStreamDecoderImpl::DecodeFrame(
}
}
VideoStreamDecoderImpl::FrameTimestamps*
VideoStreamDecoderImpl::GetFrameTimestamps(int64_t timestamp) {
int start_time_index = next_frame_timestamps_index_;
for (int i = 0; i < kFrameTimestampsMemory; ++i) {
start_time_index = Subtract<kFrameTimestampsMemory>(start_time_index, 1);
VideoStreamDecoderImpl::FrameInfo* VideoStreamDecoderImpl::GetFrameInfo(
int64_t timestamp) {
int start_time_index = next_frame_info_index_;
for (int i = 0; i < kFrameInfoMemory; ++i) {
start_time_index = Subtract<kFrameInfoMemory>(start_time_index, 1);
if (frame_timestamps_[start_time_index].timestamp == timestamp)
return &frame_timestamps_[start_time_index];
if (frame_info_[start_time_index].timestamp == timestamp)
return &frame_info_[start_time_index];
}
return nullptr;
@ -253,26 +251,27 @@ void VideoStreamDecoderImpl::OnDecodedFrameCallback(
decode_time_ms, qp]() {
RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
FrameTimestamps* frame_timestamps =
GetFrameTimestamps(decoded_image.timestamp());
if (!frame_timestamps) {
FrameInfo* frame_info = GetFrameInfo(decoded_image.timestamp());
if (!frame_info) {
RTC_LOG(LS_ERROR) << "No frame information found for frame with timestamp"
<< decoded_image.timestamp();
return;
}
absl::optional<int> casted_qp;
Callbacks::FrameInfo callback_info;
callback_info.content_type = frame_info->content_type;
if (qp)
casted_qp.emplace(*qp);
callback_info.qp.emplace(*qp);
absl::optional<int> casted_decode_time_ms(decode_time_ms.value_or(
decode_stop_time_ms - frame_timestamps->decode_start_time_ms));
callback_info.decode_time_ms = decode_time_ms.value_or(
decode_stop_time_ms - frame_info->decode_start_time_ms);
timing_.StopDecodeTimer(*casted_decode_time_ms, decode_stop_time_ms);
timing_.StopDecodeTimer(*callback_info.decode_time_ms, decode_stop_time_ms);
VideoFrame copy = decoded_image;
copy.set_timestamp_us(frame_timestamps->render_time_us);
callbacks_->OnDecodedFrame(copy, casted_decode_time_ms, casted_qp);
copy.set_timestamp_us(frame_info->render_time_us);
callbacks_->OnDecodedFrame(copy, callback_info);
});
}

16
video/video_stream_decoder_impl.h
View File

@ -62,16 +62,16 @@ class VideoStreamDecoderImpl : public VideoStreamDecoderInterface {
kDecodeFailure,
};
struct FrameTimestamps {
int64_t timestamp;
struct FrameInfo {
int64_t timestamp = -1;
int64_t decode_start_time_ms;
int64_t render_time_us;
VideoContentType content_type;
};
void SaveFrameTimestamps(const video_coding::EncodedFrame& frame)
RTC_RUN_ON(bookkeeping_queue_);
FrameTimestamps* GetFrameTimestamps(int64_t timestamp)
void SaveFrameInfo(const video_coding::EncodedFrame& frame)
RTC_RUN_ON(bookkeeping_queue_);
FrameInfo* GetFrameInfo(int64_t timestamp) RTC_RUN_ON(bookkeeping_queue_);
void StartNextDecode() RTC_RUN_ON(bookkeeping_queue_);
void OnNextFrameCallback(std::unique_ptr<video_coding::EncodedFrame> frame,
video_coding::FrameBuffer::ReturnReason res)
@ -90,10 +90,10 @@ class VideoStreamDecoderImpl : public VideoStreamDecoderInterface {
// Some decoders are pipelined so it is not sufficient to save frame info
// for the last frame only.
static constexpr int kFrameTimestampsMemory = 8;
std::array<FrameTimestamps, kFrameTimestampsMemory> frame_timestamps_
static constexpr int kFrameInfoMemory = 8;
std::array<FrameInfo, kFrameInfoMemory> frame_info_
RTC_GUARDED_BY(bookkeeping_queue_);
int next_frame_timestamps_index_ RTC_GUARDED_BY(bookkeeping_queue_);
int next_frame_info_index_ RTC_GUARDED_BY(bookkeeping_queue_);
VideoStreamDecoderInterface::Callbacks* const callbacks_
RTC_PT_GUARDED_BY(bookkeeping_queue_);
video_coding::VideoLayerFrameId last_continuous_id_