zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added VideoContentType to OnDecodedFrame callback.

Browse Source 
Also added a FrameInfo struct to propagate various meta information along side it.

Bug: webrtc:9106
Change-Id: I1feb9f94c662c367f7c6e0a50d33705fdd5346bb
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/183880
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32079}
This commit is contained in:
philipel
2020-09-10 15:30:02 +02:00
committed by Commit Bot
parent 673027b4a5
commit 3dc4780d8e
4 changed files with 51 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
api/video/BUILD.gn
View File

@ -222,6 +222,7 @@ rtc_source_set("video_stream_decoder") {
deps = [ deps = [
":encoded_frame", ":encoded_frame",
":video_frame", ":video_frame",
":video_rtp_headers",
"../task_queue", "../task_queue",
"../units:time_delta", "../units:time_delta",
"../video_codecs:video_codecs_api", "../video_codecs:video_codecs_api",

17
api/video/video_stream_decoder.h
View File

@ -17,6 +17,7 @@
#include "api/units/time_delta.h" #include "api/units/time_delta.h"
#include "api/video/encoded_frame.h" #include "api/video/encoded_frame.h"
#include "api/video/video_content_type.h"
#include "api/video/video_frame.h" #include "api/video/video_frame.h"
#include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_decoder_factory.h"
@ -29,6 +30,12 @@ class VideoStreamDecoderInterface {
public: public:
virtual ~Callbacks() = default; virtual ~Callbacks() = default;
struct FrameInfo {
absl::optional<int> decode_time_ms;
absl::optional<int> qp;
VideoContentType content_type;
};
// Called when the VideoStreamDecoder enters a non-decodable state. // Called when the VideoStreamDecoder enters a non-decodable state.
virtual void OnNonDecodableState() = 0; virtual void OnNonDecodableState() = 0;
@ -37,9 +44,15 @@ class VideoStreamDecoderInterface {
const video_coding::VideoLayerFrameId& key) = 0; const video_coding::VideoLayerFrameId& key) = 0;
// Called with the decoded frame. // Called with the decoded frame.
virtual void OnDecodedFrame(VideoFrame decodedImage, virtual void OnDecodedFrame(VideoFrame frame,
absl::optional<int> decode_time_ms, absl::optional<int> decode_time_ms,
absl::optional<int> qp) = 0; absl::optional<int> qp) {}
// TODO(philipel): Make pure virtual and remove old function above when
// it is no longer used.
virtual void OnDecodedFrame(VideoFrame frame, const FrameInfo& meta_info) {
OnDecodedFrame(std::move(frame), meta_info.decode_time_ms, meta_info.qp);
}
}; };
virtual ~VideoStreamDecoderInterface() = default; virtual ~VideoStreamDecoderInterface() = default;

55
video/video_stream_decoder_impl.cc
View File

@ -26,7 +26,7 @@ VideoStreamDecoderImpl::VideoStreamDecoderImpl(
std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings) std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings)
: timing_(Clock::GetRealTimeClock()), : timing_(Clock::GetRealTimeClock()),
decode_callbacks_(this), decode_callbacks_(this),
next_frame_timestamps_index_(0), next_frame_info_index_(0),
callbacks_(callbacks), callbacks_(callbacks),
keyframe_required_(true), keyframe_required_(true),
decoder_factory_(decoder_factory), decoder_factory_(decoder_factory),
@ -39,7 +39,6 @@ VideoStreamDecoderImpl::VideoStreamDecoderImpl(
decode_queue_(task_queue_factory->CreateTaskQueue( decode_queue_(task_queue_factory->CreateTaskQueue(
"video_stream_decoder_decode_queue", "video_stream_decoder_decode_queue",
TaskQueueFactory::Priority::NORMAL)) { TaskQueueFactory::Priority::NORMAL)) {
frame_timestamps_.fill({-1, -1, -1});
bookkeeping_queue_.PostTask([this]() { bookkeeping_queue_.PostTask([this]() {
RTC_DCHECK_RUN_ON(&bookkeeping_queue_); RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
StartNextDecode(); StartNextDecode();
@ -125,16 +124,15 @@ VideoDecoder* VideoStreamDecoderImpl::GetDecoder(int payload_type) {
return decoder_.get(); return decoder_.get();
} }
void VideoStreamDecoderImpl::SaveFrameTimestamps( void VideoStreamDecoderImpl::SaveFrameInfo(
const video_coding::EncodedFrame& frame) { const video_coding::EncodedFrame& frame) {
FrameTimestamps* frame_timestamps = FrameInfo* frame_info = &frame_info_[next_frame_info_index_];
&frame_timestamps_[next_frame_timestamps_index_]; frame_info->timestamp = frame.Timestamp();
frame_timestamps->timestamp = frame.Timestamp(); frame_info->decode_start_time_ms = rtc::TimeMillis();
frame_timestamps->decode_start_time_ms = rtc::TimeMillis(); frame_info->render_time_us = frame.RenderTimeMs() * 1000;
frame_timestamps->render_time_us = frame.RenderTimeMs() * 1000; frame_info->content_type = frame.EncodedImage().content_type_;
next_frame_timestamps_index_ = next_frame_info_index_ = Add<kFrameInfoMemory>(next_frame_info_index_, 1);
Add<kFrameTimestampsMemory>(next_frame_timestamps_index_, 1);
} }
void VideoStreamDecoderImpl::StartNextDecode() { void VideoStreamDecoderImpl::StartNextDecode() {
@ -155,7 +153,7 @@ void VideoStreamDecoderImpl::OnNextFrameCallback(
switch (result) { switch (result) {
case video_coding::FrameBuffer::kFrameFound: { case video_coding::FrameBuffer::kFrameFound: {
RTC_DCHECK(frame); RTC_DCHECK(frame);
SaveFrameTimestamps(*frame); SaveFrameInfo(*frame);
MutexLock lock(&shut_down_mutex_); MutexLock lock(&shut_down_mutex_);
if (shut_down_) { if (shut_down_) {
@ -230,14 +228,14 @@ VideoStreamDecoderImpl::DecodeResult VideoStreamDecoderImpl::DecodeFrame(
} }
} }
VideoStreamDecoderImpl::FrameTimestamps* VideoStreamDecoderImpl::FrameInfo* VideoStreamDecoderImpl::GetFrameInfo(
VideoStreamDecoderImpl::GetFrameTimestamps(int64_t timestamp) { int64_t timestamp) {
int start_time_index = next_frame_timestamps_index_; int start_time_index = next_frame_info_index_;
for (int i = 0; i < kFrameTimestampsMemory; ++i) { for (int i = 0; i < kFrameInfoMemory; ++i) {
start_time_index = Subtract<kFrameTimestampsMemory>(start_time_index, 1); start_time_index = Subtract<kFrameInfoMemory>(start_time_index, 1);
if (frame_timestamps_[start_time_index].timestamp == timestamp) if (frame_info_[start_time_index].timestamp == timestamp)
return &frame_timestamps_[start_time_index]; return &frame_info_[start_time_index];
} }
return nullptr; return nullptr;
@ -253,26 +251,27 @@ void VideoStreamDecoderImpl::OnDecodedFrameCallback(
decode_time_ms, qp]() { decode_time_ms, qp]() {
RTC_DCHECK_RUN_ON(&bookkeeping_queue_); RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
FrameTimestamps* frame_timestamps = FrameInfo* frame_info = GetFrameInfo(decoded_image.timestamp());
GetFrameTimestamps(decoded_image.timestamp()); if (!frame_info) {
if (!frame_timestamps) {
RTC_LOG(LS_ERROR) << "No frame information found for frame with timestamp" RTC_LOG(LS_ERROR) << "No frame information found for frame with timestamp"
<< decoded_image.timestamp(); << decoded_image.timestamp();
return; return;
} }
absl::optional<int> casted_qp; Callbacks::FrameInfo callback_info;
callback_info.content_type = frame_info->content_type;
if (qp) if (qp)
casted_qp.emplace(*qp); callback_info.qp.emplace(*qp);
absl::optional<int> casted_decode_time_ms(decode_time_ms.value_or( callback_info.decode_time_ms = decode_time_ms.value_or(
decode_stop_time_ms - frame_timestamps->decode_start_time_ms)); decode_stop_time_ms - frame_info->decode_start_time_ms);
timing_.StopDecodeTimer(*casted_decode_time_ms, decode_stop_time_ms); timing_.StopDecodeTimer(*callback_info.decode_time_ms, decode_stop_time_ms);
VideoFrame copy = decoded_image; VideoFrame copy = decoded_image;
copy.set_timestamp_us(frame_timestamps->render_time_us); copy.set_timestamp_us(frame_info->render_time_us);
callbacks_->OnDecodedFrame(copy, casted_decode_time_ms, casted_qp); callbacks_->OnDecodedFrame(copy, callback_info);
}); });
} }

16
video/video_stream_decoder_impl.h
View File

@ -62,16 +62,16 @@ class VideoStreamDecoderImpl : public VideoStreamDecoderInterface {
kDecodeFailure, kDecodeFailure,
}; };
struct FrameTimestamps { struct FrameInfo {
int64_t timestamp; int64_t timestamp = -1;
int64_t decode_start_time_ms; int64_t decode_start_time_ms;
int64_t render_time_us; int64_t render_time_us;
VideoContentType content_type;
}; };
void SaveFrameTimestamps(const video_coding::EncodedFrame& frame) void SaveFrameInfo(const video_coding::EncodedFrame& frame)
RTC_RUN_ON(bookkeeping_queue_);
FrameTimestamps* GetFrameTimestamps(int64_t timestamp)
RTC_RUN_ON(bookkeeping_queue_); RTC_RUN_ON(bookkeeping_queue_);
FrameInfo* GetFrameInfo(int64_t timestamp) RTC_RUN_ON(bookkeeping_queue_);
void StartNextDecode() RTC_RUN_ON(bookkeeping_queue_); void StartNextDecode() RTC_RUN_ON(bookkeeping_queue_);
void OnNextFrameCallback(std::unique_ptr<video_coding::EncodedFrame> frame, void OnNextFrameCallback(std::unique_ptr<video_coding::EncodedFrame> frame,
video_coding::FrameBuffer::ReturnReason res) video_coding::FrameBuffer::ReturnReason res)
@ -90,10 +90,10 @@ class VideoStreamDecoderImpl : public VideoStreamDecoderInterface {
// Some decoders are pipelined so it is not sufficient to save frame info // Some decoders are pipelined so it is not sufficient to save frame info
// for the last frame only. // for the last frame only.
static constexpr int kFrameTimestampsMemory = 8; static constexpr int kFrameInfoMemory = 8;
std::array<FrameTimestamps, kFrameTimestampsMemory> frame_timestamps_ std::array<FrameInfo, kFrameInfoMemory> frame_info_
RTC_GUARDED_BY(bookkeeping_queue_); RTC_GUARDED_BY(bookkeeping_queue_);
int next_frame_timestamps_index_ RTC_GUARDED_BY(bookkeeping_queue_); int next_frame_info_index_ RTC_GUARDED_BY(bookkeeping_queue_);
VideoStreamDecoderInterface::Callbacks* const callbacks_ VideoStreamDecoderInterface::Callbacks* const callbacks_
RTC_PT_GUARDED_BY(bookkeeping_queue_); RTC_PT_GUARDED_BY(bookkeeping_queue_);
video_coding::VideoLayerFrameId last_continuous_id_ video_coding::VideoLayerFrameId last_continuous_id_