zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
a35df4260f2e51f35e94fa74d1e840f73b597369
platform-external-webrtc/webrtc/media/engine/webrtcvideoengine.h
magjed a35df4260f WebRtcVideoEngine: Encapsulate logic for unifying internal and external video codecs 
This CL encapsulates the logic for unifying the internal and external
video encoders into a helper class. The purpose is to prepare for
introducing a new video encoder factory interface that inherently
represents all encoders (i.e. both internal and external). A helper
interface EncoderFactoryAdapter is introduced that both the old
WebRtcVideoEncoderFactory and the new factory interface can implement
and serves as common point to leave the rest of the code unchanged.

BUG=webrtc:7925

Review-Url: https://codereview.webrtc.org/3006713002
Cr-Commit-Position: refs/heads/master@{#19600}
2017-08-30 11:21:30 +00:00

540 lines
21 KiB
C++
Raw Blame History

/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MEDIA_ENGINE_WEBRTCVIDEOENGINE_H_
#define WEBRTC_MEDIA_ENGINE_WEBRTCVIDEOENGINE_H_
#include <map>
#include <memory>
#include <set>
#include <string>
#include <vector>
#include "webrtc/api/call/transport.h"
#include "webrtc/api/video/video_frame.h"
#include "webrtc/call/call.h"
#include "webrtc/call/flexfec_receive_stream.h"
#include "webrtc/call/video_receive_stream.h"
#include "webrtc/call/video_send_stream.h"
#include "webrtc/media/base/mediaengine.h"
#include "webrtc/media/base/videosinkinterface.h"
#include "webrtc/media/base/videosourceinterface.h"
#include "webrtc/media/engine/webrtcvideodecoderfactory.h"
#include "webrtc/media/engine/webrtcvideoencoderfactory.h"
#include "webrtc/rtc_base/asyncinvoker.h"
#include "webrtc/rtc_base/criticalsection.h"
#include "webrtc/rtc_base/networkroute.h"
#include "webrtc/rtc_base/optional.h"
#include "webrtc/rtc_base/thread_annotations.h"
#include "webrtc/rtc_base/thread_checker.h"
namespace webrtc {
class VideoDecoder;
class VideoEncoder;
struct MediaConfig;
}
namespace rtc {
class Thread;
} // namespace rtc
namespace cricket {
class EncoderFactoryAdapter;
class VideoCapturer;
class VideoProcessor;
class VideoRenderer;
class VoiceMediaChannel;
class WebRtcDecoderObserver;
class WebRtcEncoderObserver;
class WebRtcLocalStreamInfo;
class WebRtcRenderAdapter;
class WebRtcVideoChannel;
class WebRtcVideoChannelRecvInfo;
class WebRtcVideoChannelSendInfo;
class WebRtcVoiceEngine;
class WebRtcVoiceMediaChannel;
struct Device;
class UnsignalledSsrcHandler {
public:
enum Action {
kDropPacket,
kDeliverPacket,
};
virtual Action OnUnsignalledSsrc(WebRtcVideoChannel* channel,
uint32_t ssrc) = 0;
virtual ~UnsignalledSsrcHandler() = default;
};
// TODO(pbos): Remove, use external handlers only.
class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler {
public:
DefaultUnsignalledSsrcHandler();
Action OnUnsignalledSsrc(WebRtcVideoChannel* channel,
uint32_t ssrc) override;
rtc::VideoSinkInterface<webrtc::VideoFrame>* GetDefaultSink() const;
void SetDefaultSink(WebRtcVideoChannel* channel,
rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
virtual ~DefaultUnsignalledSsrcHandler() = default;
private:
rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_;
};
// WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667).
class WebRtcVideoEngine {
public:
WebRtcVideoEngine();
virtual ~WebRtcVideoEngine();
// Basic video engine implementation.
void Init();
WebRtcVideoChannel* CreateChannel(webrtc::Call* call,
const MediaConfig& config,
const VideoOptions& options);
std::vector<VideoCodec> codecs() const;
RtpCapabilities GetCapabilities() const;
// Set a WebRtcVideoDecoderFactory for external decoding. Video engine does
// not take the ownership of |decoder_factory|. The caller needs to make sure
// that |decoder_factory| outlives the video engine.
void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory);
// Set a WebRtcVideoEncoderFactory for external encoding. Video engine does
// not take the ownership of |encoder_factory|. The caller needs to make sure
// that |encoder_factory| outlives the video engine.
virtual void SetExternalEncoderFactory(
WebRtcVideoEncoderFactory* encoder_factory);
private:
bool initialized_;
WebRtcVideoDecoderFactory* external_decoder_factory_;
std::unique_ptr<EncoderFactoryAdapter> encoder_factory_;
};
class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
public:
WebRtcVideoChannel(webrtc::Call* call,
const MediaConfig& config,
const VideoOptions& options,
const EncoderFactoryAdapter& encoder_factory,
WebRtcVideoDecoderFactory* external_decoder_factory);
~WebRtcVideoChannel() override;
// VideoMediaChannel implementation
rtc::DiffServCodePoint PreferredDscp() const override;
bool SetSendParameters(const VideoSendParameters& params) override;
bool SetRecvParameters(const VideoRecvParameters& params) override;
webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override;
bool SetRtpSendParameters(uint32_t ssrc,
const webrtc::RtpParameters& parameters) override;
webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override;
bool SetRtpReceiveParameters(
uint32_t ssrc,
const webrtc::RtpParameters& parameters) override;
bool GetSendCodec(VideoCodec* send_codec) override;
bool SetSend(bool send) override;
bool SetVideoSend(
uint32_t ssrc,
bool enable,
const VideoOptions* options,
rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override;
bool AddSendStream(const StreamParams& sp) override;
bool RemoveSendStream(uint32_t ssrc) override;
bool AddRecvStream(const StreamParams& sp) override;
bool AddRecvStream(const StreamParams& sp, bool default_stream);
bool RemoveRecvStream(uint32_t ssrc) override;
bool SetSink(uint32_t ssrc,
rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override;
bool GetStats(VideoMediaInfo* info) override;
void OnPacketReceived(rtc::CopyOnWriteBuffer* packet,
const rtc::PacketTime& packet_time) override;
void OnRtcpReceived(rtc::CopyOnWriteBuffer* packet,
const rtc::PacketTime& packet_time) override;
void OnReadyToSend(bool ready) override;
void OnNetworkRouteChanged(const std::string& transport_name,
const rtc::NetworkRoute& network_route) override;
void OnTransportOverheadChanged(int transport_overhead_per_packet) override;
void SetInterface(NetworkInterface* iface) override;
// Implemented for VideoMediaChannelTest.
bool sending() const { return sending_; }
rtc::Optional<uint32_t> GetDefaultReceiveStreamSsrc();
// AdaptReason is used for expressing why a WebRtcVideoSendStream request
// a lower input frame size than the currently configured camera input frame
// size. There can be more than one reason OR:ed together.
enum AdaptReason {
ADAPTREASON_NONE = 0,
ADAPTREASON_CPU = 1,
ADAPTREASON_BANDWIDTH = 2,
};
static constexpr int kDefaultQpMax = 56;
private:
class WebRtcVideoReceiveStream;
struct VideoCodecSettings {
VideoCodecSettings();
// Checks if all members of |*this| are equal to the corresponding members
// of |other|.
bool operator==(const VideoCodecSettings& other) const;
bool operator!=(const VideoCodecSettings& other) const;
// Checks if all members of |a|, except |flexfec_payload_type|, are equal
// to the corresponding members of |b|.
static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a,
const VideoCodecSettings& b);
VideoCodec codec;
webrtc::UlpfecConfig ulpfec;
int flexfec_payload_type;
int rtx_payload_type;
};
struct ChangedSendParameters {
// These optionals are unset if not changed.
rtc::Optional<VideoCodecSettings> codec;
rtc::Optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
rtc::Optional<int> max_bandwidth_bps;
rtc::Optional<bool> conference_mode;
rtc::Optional<webrtc::RtcpMode> rtcp_mode;
};
struct ChangedRecvParameters {
// These optionals are unset if not changed.
rtc::Optional<std::vector<VideoCodecSettings>> codec_settings;
rtc::Optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
// Keep track of the FlexFEC payload type separately from |codec_settings|.
// This allows us to recreate the FlexfecReceiveStream separately from the
// VideoReceiveStream when the FlexFEC payload type is changed.
rtc::Optional<int> flexfec_payload_type;
};
bool GetChangedSendParameters(const VideoSendParameters& params,
ChangedSendParameters* changed_params) const;
bool GetChangedRecvParameters(const VideoRecvParameters& params,
ChangedRecvParameters* changed_params) const;
void SetMaxSendBandwidth(int bps);
void ConfigureReceiverRtp(
webrtc::VideoReceiveStream::Config* config,
webrtc::FlexfecReceiveStream::Config* flexfec_config,
const StreamParams& sp) const;
bool ValidateSendSsrcAvailability(const StreamParams& sp) const
EXCLUSIVE_LOCKS_REQUIRED(stream_crit_);
bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const
EXCLUSIVE_LOCKS_REQUIRED(stream_crit_);
void DeleteReceiveStream(WebRtcVideoReceiveStream* stream)
EXCLUSIVE_LOCKS_REQUIRED(stream_crit_);
static std::string CodecSettingsVectorToString(
const std::vector<VideoCodecSettings>& codecs);
// Wrapper for the sender part.
class WebRtcVideoSendStream
: public rtc::VideoSourceInterface<webrtc::VideoFrame> {
public:
WebRtcVideoSendStream(
webrtc::Call* call,
const StreamParams& sp,
webrtc::VideoSendStream::Config config,
const VideoOptions& options,
const EncoderFactoryAdapter& encoder_factory,
bool enable_cpu_overuse_detection,
int max_bitrate_bps,
const rtc::Optional<VideoCodecSettings>& codec_settings,
const rtc::Optional<std::vector<webrtc::RtpExtension>>& rtp_extensions,
const VideoSendParameters& send_params);
virtual ~WebRtcVideoSendStream();
void SetSendParameters(const ChangedSendParameters& send_params);
bool SetRtpParameters(const webrtc::RtpParameters& parameters);
webrtc::RtpParameters GetRtpParameters() const;
// Implements rtc::VideoSourceInterface<webrtc::VideoFrame>.
// WebRtcVideoSendStream acts as a source to the webrtc::VideoSendStream
// in |stream_|. This is done to proxy VideoSinkWants from the encoder to
// the worker thread.
void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
const rtc::VideoSinkWants& wants) override;
void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
bool SetVideoSend(bool mute,
const VideoOptions* options,
rtc::VideoSourceInterface<webrtc::VideoFrame>* source);
void SetSend(bool send);
const std::vector<uint32_t>& GetSsrcs() const;
VideoSenderInfo GetVideoSenderInfo(bool log_stats);
void FillBitrateInfo(BandwidthEstimationInfo* bwe_info);
private:
// Parameters needed to reconstruct the underlying stream.
// webrtc::VideoSendStream doesn't support setting a lot of options on the
// fly, so when those need to be changed we tear down and reconstruct with
// similar parameters depending on which options changed etc.
struct VideoSendStreamParameters {
VideoSendStreamParameters(
webrtc::VideoSendStream::Config config,
const VideoOptions& options,
int max_bitrate_bps,
const rtc::Optional<VideoCodecSettings>& codec_settings);
webrtc::VideoSendStream::Config config;
VideoOptions options;
int max_bitrate_bps;
bool conference_mode;
rtc::Optional<VideoCodecSettings> codec_settings;
// Sent resolutions + bitrates etc. by the underlying VideoSendStream,
// typically changes when setting a new resolution or reconfiguring
// bitrates.
webrtc::VideoEncoderConfig encoder_config;
};
rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings>
ConfigureVideoEncoderSettings(const VideoCodec& codec);
void SetCodec(const VideoCodecSettings& codec,
bool force_encoder_allocation);
void RecreateWebRtcStream();
webrtc::VideoEncoderConfig CreateVideoEncoderConfig(
const VideoCodec& codec) const;
void ReconfigureEncoder();
bool ValidateRtpParameters(const webrtc::RtpParameters& parameters);
// Calls Start or Stop according to whether or not |sending_| is true,
// and whether or not the encoding in |rtp_parameters_| is active.
void UpdateSendState();
webrtc::VideoSendStream::DegradationPreference GetDegradationPreference()
const EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_);
rtc::ThreadChecker thread_checker_;
rtc::AsyncInvoker invoker_;
rtc::Thread* worker_thread_;
const std::vector<uint32_t> ssrcs_ ACCESS_ON(&thread_checker_);
const std::vector<SsrcGroup> ssrc_groups_ ACCESS_ON(&thread_checker_);
webrtc::Call* const call_;
const bool enable_cpu_overuse_detection_;
rtc::VideoSourceInterface<webrtc::VideoFrame>* source_
ACCESS_ON(&thread_checker_);
std::unique_ptr<EncoderFactoryAdapter> encoder_factory_
ACCESS_ON(&thread_checker_);
webrtc::VideoSendStream* stream_ ACCESS_ON(&thread_checker_);
rtc::VideoSinkInterface<webrtc::VideoFrame>* encoder_sink_
ACCESS_ON(&thread_checker_);
// Contains settings that are the same for all streams in the MediaChannel,
// such as codecs, header extensions, and the global bitrate limit for the
// entire channel.
VideoSendStreamParameters parameters_ ACCESS_ON(&thread_checker_);
// Contains settings that are unique for each stream, such as max_bitrate.
// Does *not* contain codecs, however.
// TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_.
// TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only
// one stream per MediaChannel.
webrtc::RtpParameters rtp_parameters_ ACCESS_ON(&thread_checker_);
std::unique_ptr<webrtc::VideoEncoder> allocated_encoder_
ACCESS_ON(&thread_checker_);
VideoCodec allocated_codec_ ACCESS_ON(&thread_checker_);
bool sending_ ACCESS_ON(&thread_checker_);
};
// Wrapper for the receiver part, contains configs etc. that are needed to
// reconstruct the underlying VideoReceiveStream.
class WebRtcVideoReceiveStream
: public rtc::VideoSinkInterface<webrtc::VideoFrame> {
public:
WebRtcVideoReceiveStream(
webrtc::Call* call,
const StreamParams& sp,
webrtc::VideoReceiveStream::Config config,
WebRtcVideoDecoderFactory* external_decoder_factory,
bool default_stream,
const std::vector<VideoCodecSettings>& recv_codecs,
const webrtc::FlexfecReceiveStream::Config& flexfec_config);
~WebRtcVideoReceiveStream();
const std::vector<uint32_t>& GetSsrcs() const;
rtc::Optional<uint32_t> GetFirstPrimarySsrc() const;
void SetLocalSsrc(uint32_t local_ssrc);
// TODO(deadbeef): Move these feedback parameters into the recv parameters.
void SetFeedbackParameters(bool nack_enabled,
bool remb_enabled,
bool transport_cc_enabled,
webrtc::RtcpMode rtcp_mode);
void SetRecvParameters(const ChangedRecvParameters& recv_params);
void OnFrame(const webrtc::VideoFrame& frame) override;
bool IsDefaultStream() const;
void SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
VideoReceiverInfo GetVideoReceiverInfo(bool log_stats);
private:
struct AllocatedDecoder {
AllocatedDecoder(webrtc::VideoDecoder* decoder,
webrtc::VideoCodecType type,
bool external);
webrtc::VideoDecoder* decoder;
// Decoder wrapped into a fallback decoder to permit software fallback.
webrtc::VideoDecoder* external_decoder;
webrtc::VideoCodecType type;
bool external;
};
void RecreateWebRtcVideoStream();
void MaybeRecreateWebRtcFlexfecStream();
void MaybeAssociateFlexfecWithVideo();
void MaybeDissociateFlexfecFromVideo();
void ConfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs,
std::vector<AllocatedDecoder>* old_codecs);
void ConfigureFlexfecCodec(int flexfec_payload_type);
AllocatedDecoder CreateOrReuseVideoDecoder(
std::vector<AllocatedDecoder>* old_decoder,
const VideoCodec& codec);
void ClearDecoders(std::vector<AllocatedDecoder>* allocated_decoders);
std::string GetCodecNameFromPayloadType(int payload_type);
webrtc::Call* const call_;
StreamParams stream_params_;
// Both |stream_| and |flexfec_stream_| are managed by |this|. They are
// destroyed by calling call_->DestroyVideoReceiveStream and
// call_->DestroyFlexfecReceiveStream, respectively.
webrtc::VideoReceiveStream* stream_;
const bool default_stream_;
webrtc::VideoReceiveStream::Config config_;
webrtc::FlexfecReceiveStream::Config flexfec_config_;
webrtc::FlexfecReceiveStream* flexfec_stream_;
WebRtcVideoDecoderFactory* const external_decoder_factory_;
std::vector<AllocatedDecoder> allocated_decoders_;
rtc::CriticalSection sink_lock_;
rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_ GUARDED_BY(sink_lock_);
// Expands remote RTP timestamps to int64_t to be able to estimate how long
// the stream has been running.
rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_
GUARDED_BY(sink_lock_);
int64_t first_frame_timestamp_ GUARDED_BY(sink_lock_);
// Start NTP time is estimated as current remote NTP time (estimated from
// RTCP) minus the elapsed time, as soon as remote NTP time is available.
int64_t estimated_remote_start_ntp_time_ms_ GUARDED_BY(sink_lock_);
};
void Construct(webrtc::Call* call, WebRtcVideoEngine* engine);
bool SendRtp(const uint8_t* data,
size_t len,
const webrtc::PacketOptions& options) override;
bool SendRtcp(const uint8_t* data, size_t len) override;
static std::vector<VideoCodecSettings> MapCodecs(
const std::vector<VideoCodec>& codecs);
// Select what video codec will be used for sending, i.e. what codec is used
// for local encoding, based on supported remote codecs. The first remote
// codec that is supported locally will be selected.
rtc::Optional<VideoCodecSettings> SelectSendVideoCodec(
const std::vector<VideoCodecSettings>& remote_mapped_codecs) const;
static bool NonFlexfecReceiveCodecsHaveChanged(
std::vector<VideoCodecSettings> before,
std::vector<VideoCodecSettings> after);
void FillSenderStats(VideoMediaInfo* info, bool log_stats);
void FillReceiverStats(VideoMediaInfo* info, bool log_stats);
void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats,
VideoMediaInfo* info);
void FillSendAndReceiveCodecStats(VideoMediaInfo* video_media_info);
rtc::ThreadChecker thread_checker_;
uint32_t rtcp_receiver_report_ssrc_;
bool sending_;
webrtc::Call* const call_;
DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_;
UnsignalledSsrcHandler* const unsignalled_ssrc_handler_;
const MediaConfig::Video video_config_;
rtc::CriticalSection stream_crit_;
// Using primary-ssrc (first ssrc) as key.
std::map<uint32_t, WebRtcVideoSendStream*> send_streams_
GUARDED_BY(stream_crit_);
std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_
GUARDED_BY(stream_crit_);
std::set<uint32_t> send_ssrcs_ GUARDED_BY(stream_crit_);
std::set<uint32_t> receive_ssrcs_ GUARDED_BY(stream_crit_);
rtc::Optional<VideoCodecSettings> send_codec_;
rtc::Optional<std::vector<webrtc::RtpExtension>> send_rtp_extensions_;
std::unique_ptr<EncoderFactoryAdapter> encoder_factory_;
WebRtcVideoDecoderFactory* const external_decoder_factory_;
std::vector<VideoCodecSettings> recv_codecs_;
std::vector<webrtc::RtpExtension> recv_rtp_extensions_;
// See reason for keeping track of the FlexFEC payload type separately in
// comment in WebRtcVideoChannel::ChangedRecvParameters.
int recv_flexfec_payload_type_;
webrtc::Call::Config::BitrateConfig bitrate_config_;
// TODO(deadbeef): Don't duplicate information between
// send_params/recv_params, rtp_extensions, options, etc.
VideoSendParameters send_params_;
VideoOptions default_send_options_;
VideoRecvParameters recv_params_;
int64_t last_stats_log_ms_;
};
class EncoderStreamFactory
: public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface {
public:
EncoderStreamFactory(std::string codec_name,
int max_qp,
int max_framerate,
bool is_screencast,
bool conference_mode);
private:
std::vector<webrtc::VideoStream> CreateEncoderStreams(
int width,
int height,
const webrtc::VideoEncoderConfig& encoder_config) override;
const std::string codec_name_;
const int max_qp_;
const int max_framerate_;
const bool is_screencast_;
const bool conference_mode_;
};
} // namespace cricket
#endif // WEBRTC_MEDIA_ENGINE_WEBRTCVIDEOENGINE_H_
Reference in New Issue View Git Blame Copy Permalink