zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WebRtcVideoEngine: Enable encoded frame sink.

Browse Source 
This change ultimately enables wiring up VideoRtpReceiver::OnGenerateKeyFrame
and OnEncodedSinkEnabled into internal::VideoReceiveStream so that encoded
frames can flow to sinks installed in VideoTrackSourceInterface.

Bug: chromium:1013590
Change-Id: I136132c210e5811547f2522ddc371d0acac90664
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/161093
Commit-Queue: Markus Handell <handellm@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30001}
This commit is contained in:
Markus Handell
2019-12-04 10:58:17 +01:00
committed by Commit Bot
parent a9ad36f322
commit 32565f684b
7 changed files with 286 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
media/BUILD.gn
View File

@ -92,6 +92,7 @@ rtc_library("rtc_media_base") {
"../api/video:video_rtp_headers",
"../api/video_codecs:video_codecs_api",
"../call:call_interfaces",
"../call:video_stream_api",
"../common_video",
"../modules/audio_processing:audio_processing_statistics",
"../modules/rtp_rtcp:rtp_rtcp_format",
@ -172,6 +173,7 @@ rtc_library("rtc_simulcast_encoder_adapter") {
"../api/video:video_rtp_headers",
"../api/video_codecs:rtc_software_fallback_wrappers",
"../api/video_codecs:video_codecs_api",
"../call:video_stream_api",
"../modules/video_coding:video_codec_interface",
"../modules/video_coding:video_coding_utility",
"../rtc_base:checks",
@ -571,6 +573,7 @@ if (rtc_include_tests) {
"../rtc_base/experiments:min_video_bitrate_experiment",
"../rtc_base/third_party/sigslot",
"../test:audio_codec_mocks",
"../test:fake_video_codecs",
"../test:field_trial",
"../test:rtp_test_utils",
"../test:test_main",

10
media/base/fake_media_engine.cc
View File

@ -400,11 +400,21 @@ bool FakeVideoMediaChannel::SetOptions(const VideoOptions& options) {
options_ = options;
return true;
}
bool FakeVideoMediaChannel::SetMaxSendBandwidth(int bps) {
max_bps_ = bps;
return true;
}
void FakeVideoMediaChannel::SetRecordableEncodedFrameCallback(
uint32_t ssrc,
std::function<void(const webrtc::RecordableEncodedFrame&)> callback) {}
void FakeVideoMediaChannel::ClearRecordableEncodedFrameCallback(uint32_t ssrc) {
}
void FakeVideoMediaChannel::GenerateKeyFrame(uint32_t ssrc) {}
FakeDataMediaChannel::FakeDataMediaChannel(void* unused,
const DataOptions& options)
: send_blocked_(false), max_bps_(-1) {}

7
media/base/fake_media_engine.h
View File

@ -436,6 +436,13 @@ class FakeVideoMediaChannel : public RtpHelper<VideoMediaChannel> {
absl::optional<int> GetBaseMinimumPlayoutDelayMs(
uint32_t ssrc) const override;
void SetRecordableEncodedFrameCallback(
uint32_t ssrc,
std::function<void(const webrtc::RecordableEncodedFrame&)> callback)
override;
void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override;
void GenerateKeyFrame(uint32_t ssrc) override;
private:
bool SetRecvCodecs(const std::vector<VideoCodec>& codecs);
bool SetSendCodecs(const std::vector<VideoCodec>& codecs);

10
media/base/media_channel.h
View File

@ -31,6 +31,7 @@
#include "api/video/video_source_interface.h"
#include "api/video/video_timing.h"
#include "api/video_codecs/video_encoder_config.h"
#include "call/video_receive_stream.h"
#include "common_video/include/quality_limitation_reason.h"
#include "media/base/codec.h"
#include "media/base/delayable.h"
@ -41,6 +42,7 @@
#include "modules/rtp_rtcp/include/report_block_data.h"
#include "rtc_base/async_packet_socket.h"
#include "rtc_base/buffer.h"
#include "rtc_base/callback.h"
#include "rtc_base/copy_on_write_buffer.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/dscp.h"
@ -897,6 +899,14 @@ class VideoMediaChannel : public MediaChannel, public Delayable {
virtual void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) = 0;
// Gets quality stats for the channel.
virtual bool GetStats(VideoMediaInfo* info) = 0;
// Set recordable encoded frame callback for |ssrc|
virtual void SetRecordableEncodedFrameCallback(
uint32_t ssrc,
std::function<void(const webrtc::RecordableEncodedFrame&)> callback) = 0;
// Clear recordable encoded frame callback for |ssrc|
virtual void ClearRecordableEncodedFrameCallback(uint32_t ssrc) = 0;
// Cause generation of a keyframe for |ssrc|
virtual void GenerateKeyFrame(uint32_t ssrc) = 0;
virtual std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const = 0;
};

96
media/engine/webrtc_video_engine.cc
View File

@ -2632,8 +2632,12 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters(
void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateWebRtcVideoStream() {
absl::optional<int> base_minimum_playout_delay_ms;
absl::optional<webrtc::VideoReceiveStream::RecordingState> recording_state;
if (stream_) {
base_minimum_playout_delay_ms = stream_->GetBaseMinimumPlayoutDelayMs();
recording_state = stream_->SetAndGetRecordingState(
webrtc::VideoReceiveStream::RecordingState(),
/*generate_key_frame=*/false);
MaybeDissociateFlexfecFromVideo();
call_->DestroyVideoReceiveStream(stream_);
stream_ = nullptr;
@ -2646,6 +2650,10 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateWebRtcVideoStream() {
stream_->SetBaseMinimumPlayoutDelayMs(
base_minimum_playout_delay_ms.value());
}
if (recording_state) {
stream_->SetAndGetRecordingState(std::move(*recording_state),
/*generate_key_frame=*/false);
}
MaybeAssociateFlexfecWithVideo();
stream_->Start();
@ -2822,6 +2830,40 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::GetVideoReceiverInfo(
return info;
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::
SetRecordableEncodedFrameCallback(
std::function<void(const webrtc::RecordableEncodedFrame&)> callback) {
if (stream_) {
stream_->SetAndGetRecordingState(
webrtc::VideoReceiveStream::RecordingState(std::move(callback)),
/*generate_key_frame=*/true);
} else {
RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring setting encoded "
"frame sink";
}
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::
ClearRecordableEncodedFrameCallback() {
if (stream_) {
stream_->SetAndGetRecordingState(
webrtc::VideoReceiveStream::RecordingState(),
/*generate_key_frame=*/false);
} else {
RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring clearing encoded "
"frame sink";
}
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::GenerateKeyFrame() {
if (stream_) {
stream_->GenerateKeyFrame();
} else {
RTC_LOG(LS_ERROR)
<< "Absent receive stream; ignoring key frame generation request.";
}
}
WebRtcVideoChannel::VideoCodecSettings::VideoCodecSettings()
: flexfec_payload_type(-1), rtx_payload_type(-1) {}
@ -2968,6 +3010,60 @@ WebRtcVideoChannel::MapCodecs(const std::vector<VideoCodec>& codecs) {
return video_codecs;
}
WebRtcVideoChannel::WebRtcVideoReceiveStream*
WebRtcVideoChannel::FindReceiveStream(uint32_t ssrc) {
if (ssrc == 0) {
absl::optional<uint32_t> default_ssrc = GetDefaultReceiveStreamSsrc();
if (!default_ssrc) {
return nullptr;
}
ssrc = *default_ssrc;
}
auto it = receive_streams_.find(ssrc);
if (it != receive_streams_.end()) {
return it->second;
}
return nullptr;
}
void WebRtcVideoChannel::SetRecordableEncodedFrameCallback(
uint32_t ssrc,
std::function<void(const webrtc::RecordableEncodedFrame&)> callback) {
RTC_DCHECK_RUN_ON(&thread_checker_);
WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
if (stream) {
stream->SetRecordableEncodedFrameCallback(std::move(callback));
} else {
RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring setting encoded "
"frame sink for ssrc "
<< ssrc;
}
}
void WebRtcVideoChannel::ClearRecordableEncodedFrameCallback(uint32_t ssrc) {
RTC_DCHECK_RUN_ON(&thread_checker_);
WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
if (stream) {
stream->ClearRecordableEncodedFrameCallback();
} else {
RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring clearing encoded "
"frame sink for ssrc "
<< ssrc;
}
}
void WebRtcVideoChannel::GenerateKeyFrame(uint32_t ssrc) {
RTC_DCHECK_RUN_ON(&thread_checker_);
WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
if (stream) {
stream->GenerateKeyFrame();
} else {
RTC_LOG(LS_ERROR)
<< "Absent receive stream; ignoring key frame generation for ssrc "
<< ssrc;
}
}
// TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of
// EncoderStreamFactory and instead set this value individually for each stream
// in the VideoEncoderConfig.simulcast_layers.

17
media/engine/webrtc_video_engine.h
View File

@ -210,9 +210,21 @@ class WebRtcVideoChannel : public VideoMediaChannel,
void RequestEncoderFallback() override;
void RequestEncoderSwitch(
const EncoderSwitchRequestCallback::Config& conf) override;
void SetRecordableEncodedFrameCallback(
uint32_t ssrc,
std::function<void(const webrtc::RecordableEncodedFrame&)> callback)
override;
void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override;
void GenerateKeyFrame(uint32_t ssrc) override;
private:
class WebRtcVideoReceiveStream;
// Finds VideoReceiveStream corresponding to ssrc. Aware of unsignalled
// ssrc handling.
WebRtcVideoReceiveStream* FindReceiveStream(uint32_t ssrc)
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
struct VideoCodecSettings {
VideoCodecSettings();
@ -430,6 +442,11 @@ class WebRtcVideoChannel : public VideoMediaChannel,
VideoReceiverInfo GetVideoReceiverInfo(bool log_stats);
void SetRecordableEncodedFrameCallback(
std::function<void(const webrtc::RecordableEncodedFrame&)> callback);
void ClearRecordableEncodedFrameCallback();
void GenerateKeyFrame();
private:
void RecreateWebRtcVideoStream();
void MaybeRecreateWebRtcFlexfecStream();

143
media/engine/webrtc_video_engine_unittest.cc
View File

@ -26,6 +26,7 @@
#include "api/test/mock_video_bitrate_allocator_factory.h"
#include "api/test/mock_video_decoder_factory.h"
#include "api/test/mock_video_encoder_factory.h"
#include "api/test/video/function_video_decoder_factory.h"
#include "api/transport/field_trial_based_config.h"
#include "api/transport/media/media_transport_config.h"
#include "api/units/time_delta.h"
@ -57,6 +58,7 @@
#include "rtc_base/gunit.h"
#include "rtc_base/numerics/safe_conversions.h"
#include "rtc_base/time_utils.h"
#include "test/fake_decoder.h"
#include "test/field_trial.h"
#include "test/frame_generator.h"
#include "test/gmock.h"
@ -1307,6 +1309,147 @@ TEST_F(WebRtcVideoEngineTest, DISABLED_RecreatesEncoderOnContentTypeChange) {
EXPECT_EQ(0u, encoder_factory_->encoders().size());
}
class WebRtcVideoChannelEncodedFrameCallbackTest : public ::testing::Test {
protected:
webrtc::Call::Config GetCallConfig(
webrtc::RtcEventLogNull* event_log,
webrtc::TaskQueueFactory* task_queue_factory) {
webrtc::Call::Config call_config(event_log);
call_config.task_queue_factory = task_queue_factory;
call_config.trials = &field_trials_;
return call_config;
}
WebRtcVideoChannelEncodedFrameCallbackTest()
: task_queue_factory_(webrtc::CreateDefaultTaskQueueFactory()),
call_(absl::WrapUnique(webrtc::Call::Create(
GetCallConfig(&event_log_, task_queue_factory_.get())))),
video_bitrate_allocator_factory_(
webrtc::CreateBuiltinVideoBitrateAllocatorFactory()),
engine_(
webrtc::CreateBuiltinVideoEncoderFactory(),
std::make_unique<webrtc::test::FunctionVideoDecoderFactory>([]() {
return std::make_unique<webrtc::test::FakeDecoder>();
})),
channel_(absl::WrapUnique(static_cast<cricket::WebRtcVideoChannel*>(
engine_.CreateMediaChannel(
call_.get(),
cricket::MediaConfig(),
cricket::VideoOptions(),
webrtc::CryptoOptions(),
video_bitrate_allocator_factory_.get())))) {
network_interface_.SetDestination(channel_.get());
channel_->SetInterface(&network_interface_, webrtc::MediaTransportConfig());
cricket::VideoRecvParameters parameters;
parameters.codecs = engine_.codecs();
channel_->SetRecvParameters(parameters);
}
void DeliverKeyFrame(uint32_t ssrc) {
webrtc::RtpPacket packet;
packet.SetMarker(true);
packet.SetPayloadType(96); // VP8
packet.SetSsrc(ssrc);
// VP8 Keyframe + 1 byte payload
uint8_t* buf_ptr = packet.AllocatePayload(11);
memset(buf_ptr, 0, 11); // Pass MSAN (don't care about bytes 1-9)
buf_ptr[0] = 0x10; // Partition ID 0 + beginning of partition.
call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet.Buffer(),
/*packet_time_us=*/0);
}
void DeliverKeyFrameAndWait(uint32_t ssrc) {
DeliverKeyFrame(ssrc);
EXPECT_EQ_WAIT(1, renderer_.num_rendered_frames(), kTimeout);
EXPECT_EQ(0, renderer_.errors());
}
webrtc::FieldTrialBasedConfig field_trials_;
webrtc::RtcEventLogNull event_log_;
std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory_;
std::unique_ptr<webrtc::Call> call_;
std::unique_ptr<webrtc::VideoBitrateAllocatorFactory>
video_bitrate_allocator_factory_;
WebRtcVideoEngine engine_;
std::unique_ptr<WebRtcVideoChannel> channel_;
cricket::FakeNetworkInterface network_interface_;
cricket::FakeVideoRenderer renderer_;
};
TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest,
SetEncodedFrameBufferFunction_DefaultStream) {
testing::MockFunction<void(const webrtc::RecordableEncodedFrame&)> callback;
EXPECT_CALL(callback, Call);
EXPECT_TRUE(channel_->AddRecvStream(
cricket::StreamParams::CreateLegacy(kSsrc), /*is_default_stream=*/true));
channel_->SetRecordableEncodedFrameCallback(/*ssrc=*/0,
callback.AsStdFunction());
EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_));
DeliverKeyFrame(kSsrc);
EXPECT_EQ_WAIT(1, renderer_.num_rendered_frames(), kTimeout);
EXPECT_EQ(0, renderer_.errors());
channel_->RemoveRecvStream(kSsrc);
}
TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest,
SetEncodedFrameBufferFunction_MatchSsrcWithDefaultStream) {
testing::MockFunction<void(const webrtc::RecordableEncodedFrame&)> callback;
EXPECT_CALL(callback, Call);
EXPECT_TRUE(channel_->AddRecvStream(
cricket::StreamParams::CreateLegacy(kSsrc), /*is_default_stream=*/true));
EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_));
channel_->SetRecordableEncodedFrameCallback(kSsrc, callback.AsStdFunction());
DeliverKeyFrame(kSsrc);
EXPECT_EQ_WAIT(1, renderer_.num_rendered_frames(), kTimeout);
EXPECT_EQ(0, renderer_.errors());
channel_->RemoveRecvStream(kSsrc);
}
TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest,
SetEncodedFrameBufferFunction_MatchSsrc) {
testing::MockFunction<void(const webrtc::RecordableEncodedFrame&)> callback;
EXPECT_CALL(callback, Call);
EXPECT_TRUE(channel_->AddRecvStream(
cricket::StreamParams::CreateLegacy(kSsrc), /*is_default_stream=*/false));
EXPECT_TRUE(channel_->SetSink(kSsrc, &renderer_));
channel_->SetRecordableEncodedFrameCallback(kSsrc, callback.AsStdFunction());
DeliverKeyFrame(kSsrc);
EXPECT_EQ_WAIT(1, renderer_.num_rendered_frames(), kTimeout);
EXPECT_EQ(0, renderer_.errors());
channel_->RemoveRecvStream(kSsrc);
}
TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest,
SetEncodedFrameBufferFunction_MismatchSsrc) {
testing::StrictMock<
testing::MockFunction<void(const webrtc::RecordableEncodedFrame&)>>
callback;
EXPECT_TRUE(
channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc + 1),
/*is_default_stream=*/false));
EXPECT_TRUE(channel_->SetSink(kSsrc + 1, &renderer_));
channel_->SetRecordableEncodedFrameCallback(kSsrc, callback.AsStdFunction());
DeliverKeyFrame(kSsrc); // Expected to not cause function to fire.
DeliverKeyFrameAndWait(kSsrc + 1);
channel_->RemoveRecvStream(kSsrc + 1);
}
TEST_F(WebRtcVideoChannelEncodedFrameCallbackTest,
SetEncodedFrameBufferFunction_MismatchSsrcWithDefaultStream) {
testing::StrictMock<
testing::MockFunction<void(const webrtc::RecordableEncodedFrame&)>>
callback;
EXPECT_TRUE(
channel_->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc + 1),
/*is_default_stream=*/true));
EXPECT_TRUE(channel_->SetSink(kSsrc + 1, &renderer_));
channel_->SetRecordableEncodedFrameCallback(kSsrc, callback.AsStdFunction());
DeliverKeyFrame(kSsrc); // Expected to not cause function to fire.
DeliverKeyFrameAndWait(kSsrc + 1);
channel_->RemoveRecvStream(kSsrc + 1);
}
class WebRtcVideoChannelBaseTest : public ::testing::Test {
protected:
WebRtcVideoChannelBaseTest()