diff --git a/audio/BUILD.gn b/audio/BUILD.gn index b6369e08b8..3754a41df7 100644 --- a/audio/BUILD.gn +++ b/audio/BUILD.gn @@ -28,8 +28,6 @@ rtc_static_library("audio") { "channel_receive.h", "channel_send.cc", "channel_send.h", - "channel_send_proxy.cc", - "channel_send_proxy.h", "conversion.h", "null_audio_poller.cc", "null_audio_poller.h", diff --git a/audio/audio_receive_stream.h b/audio/audio_receive_stream.h index 5b41f1ad01..86bcb1c6a2 100644 --- a/audio/audio_receive_stream.h +++ b/audio/audio_receive_stream.h @@ -101,7 +101,7 @@ class AudioReceiveStream final : public webrtc::AudioReceiveStream, rtc::ThreadChecker module_process_thread_checker_; webrtc::AudioReceiveStream::Config config_; rtc::scoped_refptr audio_state_; - std::unique_ptr channel_receive_; + const std::unique_ptr channel_receive_; AudioSendStream* associated_send_stream_ = nullptr; bool playing_ RTC_GUARDED_BY(worker_thread_checker_) = false; diff --git a/audio/audio_send_stream.cc b/audio/audio_send_stream.cc index c88a52ef2e..3e97ab23ec 100644 --- a/audio/audio_send_stream.cc +++ b/audio/audio_send_stream.cc @@ -22,7 +22,6 @@ #include "api/crypto/frameencryptorinterface.h" #include "audio/audio_state.h" #include "audio/channel_send.h" -#include "audio/channel_send_proxy.h" #include "audio/conversion.h" #include "call/rtp_config.h" #include "call/rtp_transport_controller_send_interface.h" @@ -50,31 +49,14 @@ constexpr size_t kPacketLossTrackerMaxWindowSizeMs = 15000; constexpr size_t kPacketLossRateMinNumAckedPackets = 50; constexpr size_t kRecoverablePacketLossRateMinNumAckedPairs = 40; -void CallEncoder(const std::unique_ptr& channel_proxy, +void CallEncoder(const std::unique_ptr& channel_send, rtc::FunctionView lambda) { - channel_proxy->ModifyEncoder([&](std::unique_ptr* encoder_ptr) { + channel_send->ModifyEncoder([&](std::unique_ptr* encoder_ptr) { RTC_DCHECK(encoder_ptr); lambda(encoder_ptr->get()); }); } -std::unique_ptr CreateChannelAndProxy( - rtc::TaskQueue* worker_queue, - ProcessThread* module_process_thread, - MediaTransportInterface* media_transport, - RtcpRttStats* rtcp_rtt_stats, - RtcEventLog* event_log, - FrameEncryptorInterface* frame_encryptor, - const webrtc::CryptoOptions& crypto_options, - bool extmap_allow_mixed, - int rtcp_report_interval_ms) { - return absl::make_unique( - absl::make_unique( - worker_queue, module_process_thread, media_transport, rtcp_rtt_stats, - event_log, frame_encryptor, crypto_options, extmap_allow_mixed, - rtcp_report_interval_ms)); -} - void UpdateEventLogStreamConfig(RtcEventLog* event_log, const AudioSendStream::Config& config, const AudioSendStream::Config* old_config) { @@ -152,15 +134,15 @@ AudioSendStream::AudioSendStream( rtcp_rtt_stats, suspended_rtp_state, overall_call_lifetime, - CreateChannelAndProxy(worker_queue, - module_process_thread, - config.media_transport, - rtcp_rtt_stats, - event_log, - config.frame_encryptor, - config.crypto_options, - config.rtp.extmap_allow_mixed, - config.rtcp_report_interval_ms)) {} + voe::CreateChannelSend(worker_queue, + module_process_thread, + config.media_transport, + rtcp_rtt_stats, + event_log, + config.frame_encryptor, + config.crypto_options, + config.rtp.extmap_allow_mixed, + config.rtcp_report_interval_ms)) {} AudioSendStream::AudioSendStream( const webrtc::AudioSendStream::Config& config, @@ -172,12 +154,12 @@ AudioSendStream::AudioSendStream( RtcpRttStats* rtcp_rtt_stats, const absl::optional& suspended_rtp_state, TimeInterval* overall_call_lifetime, - std::unique_ptr channel_proxy) + std::unique_ptr channel_send) : worker_queue_(worker_queue), config_(Config(/*send_transport=*/nullptr, /*media_transport=*/nullptr)), audio_state_(audio_state), - channel_proxy_(std::move(channel_proxy)), + channel_send_(std::move(channel_send)), event_log_(event_log), bitrate_allocator_(bitrate_allocator), rtp_transport_(rtp_transport), @@ -190,7 +172,7 @@ AudioSendStream::AudioSendStream( RTC_LOG(LS_INFO) << "AudioSendStream: " << config.rtp.ssrc; RTC_DCHECK(worker_queue_); RTC_DCHECK(audio_state_); - RTC_DCHECK(channel_proxy_); + RTC_DCHECK(channel_send_); RTC_DCHECK(bitrate_allocator_); // TODO(nisse): Eventually, we should have only media_transport. But for the // time being, we can have either. When media transport is injected, there @@ -199,7 +181,7 @@ AudioSendStream::AudioSendStream( RTC_DCHECK(rtp_transport || config.media_transport); RTC_DCHECK(overall_call_lifetime_); - rtp_rtcp_module_ = channel_proxy_->GetRtpRtcp(); + rtp_rtcp_module_ = channel_send_->GetRtpRtcp(); RTC_DCHECK(rtp_rtcp_module_); ConfigureStream(this, config, true); @@ -218,8 +200,8 @@ AudioSendStream::~AudioSendStream() { RTC_DCHECK(!sending_); if (rtp_transport_) { rtp_transport_->DeRegisterPacketFeedbackObserver(this); - channel_proxy_->RegisterTransport(nullptr); - channel_proxy_->ResetSenderCongestionControlObjects(); + channel_send_->RegisterTransport(nullptr); + channel_send_->ResetSenderCongestionControlObjects(); } // Lifetime can only be updated after deregistering // |timed_send_transport_adapter_| in the underlying channel object to avoid @@ -262,29 +244,29 @@ void AudioSendStream::ConfigureStream( UpdateEventLogStreamConfig(stream->event_log_, new_config, first_time ? nullptr : &stream->config_); - const auto& channel_proxy = stream->channel_proxy_; + const auto& channel_send = stream->channel_send_; const auto& old_config = stream->config_; if (first_time || old_config.rtp.ssrc != new_config.rtp.ssrc) { - channel_proxy->SetLocalSSRC(new_config.rtp.ssrc); + channel_send->SetLocalSSRC(new_config.rtp.ssrc); if (stream->suspended_rtp_state_) { stream->rtp_rtcp_module_->SetRtpState(*stream->suspended_rtp_state_); } } if (first_time || old_config.rtp.c_name != new_config.rtp.c_name) { - channel_proxy->SetRTCP_CNAME(new_config.rtp.c_name); + channel_send->SetRTCP_CNAME(new_config.rtp.c_name); } // TODO(solenberg): Config NACK history window (which is a packet count), // using the actual packet size for the configured codec. if (first_time || old_config.rtp.nack.rtp_history_ms != new_config.rtp.nack.rtp_history_ms) { - channel_proxy->SetNACKStatus(new_config.rtp.nack.rtp_history_ms != 0, - new_config.rtp.nack.rtp_history_ms / 20); + channel_send->SetNACKStatus(new_config.rtp.nack.rtp_history_ms != 0, + new_config.rtp.nack.rtp_history_ms / 20); } if (first_time || new_config.send_transport != old_config.send_transport) { if (old_config.send_transport) { - channel_proxy->RegisterTransport(nullptr); + channel_send->RegisterTransport(nullptr); } if (new_config.send_transport) { stream->timed_send_transport_adapter_.reset(new TimedTransport( @@ -292,26 +274,26 @@ void AudioSendStream::ConfigureStream( } else { stream->timed_send_transport_adapter_.reset(nullptr); } - channel_proxy->RegisterTransport( + channel_send->RegisterTransport( stream->timed_send_transport_adapter_.get()); } // Enable the frame encryptor if a new frame encryptor has been provided. if (first_time || new_config.frame_encryptor != old_config.frame_encryptor) { - channel_proxy->SetFrameEncryptor(new_config.frame_encryptor); + channel_send->SetFrameEncryptor(new_config.frame_encryptor); } if (first_time || new_config.rtp.extmap_allow_mixed != old_config.rtp.extmap_allow_mixed) { - channel_proxy->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed); + channel_send->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed); } const ExtensionIds old_ids = FindExtensionIds(old_config.rtp.extensions); const ExtensionIds new_ids = FindExtensionIds(new_config.rtp.extensions); // Audio level indication if (first_time || new_ids.audio_level != old_ids.audio_level) { - channel_proxy->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0, - new_ids.audio_level); + channel_send->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0, + new_ids.audio_level); } bool transport_seq_num_id_changed = new_ids.transport_sequence_number != old_ids.transport_sequence_number; @@ -319,7 +301,7 @@ void AudioSendStream::ConfigureStream( (transport_seq_num_id_changed && !webrtc::field_trial::IsEnabled("WebRTC-Audio-ForceNoTWCC"))) { if (!first_time) { - channel_proxy->ResetSenderCongestionControlObjects(); + channel_send->ResetSenderCongestionControlObjects(); } RtcpBandwidthObserver* bandwidth_observer = nullptr; @@ -327,7 +309,7 @@ void AudioSendStream::ConfigureStream( new_ids.transport_sequence_number != 0 && !webrtc::field_trial::IsEnabled("WebRTC-Audio-ForceNoTWCC"); if (has_transport_sequence_number) { - channel_proxy->EnableSendTransportSequenceNumber( + channel_send->EnableSendTransportSequenceNumber( new_ids.transport_sequence_number); // Probing in application limited region is only used in combination with // send side congestion control, wich depends on feedback packets which @@ -338,7 +320,7 @@ void AudioSendStream::ConfigureStream( } } if (stream->rtp_transport_) { - channel_proxy->RegisterSenderCongestionControlObjects( + channel_send->RegisterSenderCongestionControlObjects( stream->rtp_transport_, bandwidth_observer); } } @@ -346,7 +328,7 @@ void AudioSendStream::ConfigureStream( if ((first_time || new_ids.mid != old_ids.mid || new_config.rtp.mid != old_config.rtp.mid) && new_ids.mid != 0 && !new_config.rtp.mid.empty()) { - channel_proxy->SetMid(new_config.rtp.mid, new_ids.mid); + channel_send->SetMid(new_config.rtp.mid, new_ids.mid); } if (!ReconfigureSendCodec(stream, new_config)) { @@ -382,7 +364,7 @@ void AudioSendStream::Start() { } else { rtp_rtcp_module_->SetAsPartOfAllocation(false); } - channel_proxy_->StartSend(); + channel_send_->StartSend(); sending_ = true; audio_state()->AddSendingStream(this, encoder_sample_rate_hz_, encoder_num_channels_); @@ -395,14 +377,14 @@ void AudioSendStream::Stop() { } RemoveBitrateObserver(); - channel_proxy_->StopSend(); + channel_send_->StopSend(); sending_ = false; audio_state()->RemoveSendingStream(this); } void AudioSendStream::SendAudioData(std::unique_ptr audio_frame) { RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_); - channel_proxy_->ProcessAndEncodeAudio(std::move(audio_frame)); + channel_send_->ProcessAndEncodeAudio(std::move(audio_frame)); } bool AudioSendStream::SendTelephoneEvent(int payload_type, @@ -410,14 +392,14 @@ bool AudioSendStream::SendTelephoneEvent(int payload_type, int event, int duration_ms) { RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - return channel_proxy_->SetSendTelephoneEventPayloadType(payload_type, - payload_frequency) && - channel_proxy_->SendTelephoneEventOutband(event, duration_ms); + return channel_send_->SetSendTelephoneEventPayloadType(payload_type, + payload_frequency) && + channel_send_->SendTelephoneEventOutband(event, duration_ms); } void AudioSendStream::SetMuted(bool muted) { RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_proxy_->SetInputMute(muted); + channel_send_->SetInputMute(muted); } webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const { @@ -429,9 +411,9 @@ webrtc::AudioSendStream::Stats AudioSendStream::GetStats( RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); webrtc::AudioSendStream::Stats stats; stats.local_ssrc = config_.rtp.ssrc; - stats.target_bitrate_bps = channel_proxy_->GetBitrate(); + stats.target_bitrate_bps = channel_send_->GetBitrate(); - webrtc::CallSendStatistics call_stats = channel_proxy_->GetRTCPStatistics(); + webrtc::CallSendStatistics call_stats = channel_send_->GetRTCPStatistics(); stats.bytes_sent = call_stats.bytesSent; stats.packets_sent = call_stats.packetsSent; // RTT isn't known until a RTCP report is received. Until then, VoiceEngine @@ -445,7 +427,7 @@ webrtc::AudioSendStream::Stats AudioSendStream::GetStats( stats.codec_payload_type = spec.payload_type; // Get data from the last remote RTCP report. - for (const auto& block : channel_proxy_->GetRemoteRTCPReportBlocks()) { + for (const auto& block : channel_send_->GetRemoteRTCPReportBlocks()) { // Lookup report for send ssrc only. if (block.source_SSRC == stats.local_ssrc) { stats.packets_lost = block.cumulative_num_packets_lost; @@ -467,7 +449,7 @@ webrtc::AudioSendStream::Stats AudioSendStream::GetStats( stats.total_input_duration = input_stats.total_duration; stats.typing_noise_detected = audio_state()->typing_noise_detected(); - stats.ana_statistics = channel_proxy_->GetANAStatistics(); + stats.ana_statistics = channel_send_->GetANAStatistics(); RTC_DCHECK(audio_state_->audio_processing()); stats.apm_statistics = audio_state_->audio_processing()->GetStatistics(has_remote_tracks); @@ -484,7 +466,7 @@ bool AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) { // calls on the worker thread. We should move towards always using a network // thread. Then this check can be enabled. // RTC_DCHECK(!worker_thread_checker_.CalledOnValidThread()); - return channel_proxy_->ReceivedRTCPPacket(packet, length); + return channel_send_->ReceivedRTCPPacket(packet, length); } uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) { @@ -502,7 +484,7 @@ uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) { if (update.bitrate_bps > max_bitrate_bps) update.bitrate_bps = max_bitrate_bps; - channel_proxy_->SetBitrate(update.bitrate_bps, update.bwe_period_ms); + channel_send_->SetBitrate(update.bitrate_bps, update.bwe_period_ms); // The amount of audio protection is not exposed by the encoder, hence // always returning 0. @@ -536,25 +518,24 @@ void AudioSendStream::OnPacketFeedbackVector( // the previously sent value is no longer relevant. This will be taken care // of with some refactoring which is now being done. if (plr) { - channel_proxy_->OnTwccBasedUplinkPacketLossRate(*plr); + channel_send_->OnTwccBasedUplinkPacketLossRate(*plr); } if (rplr) { - channel_proxy_->OnRecoverableUplinkPacketLossRate(*rplr); + channel_send_->OnRecoverableUplinkPacketLossRate(*rplr); } } void AudioSendStream::SetTransportOverhead(int transport_overhead_per_packet) { RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_proxy_->SetTransportOverhead(transport_overhead_per_packet); + channel_send_->SetTransportOverhead(transport_overhead_per_packet); } RtpState AudioSendStream::GetRtpState() const { return rtp_rtcp_module_->GetRtpState(); } -const voe::ChannelSend* AudioSendStream::GetChannel() const { - RTC_DCHECK(channel_proxy_.get()); - return channel_proxy_->GetChannel(); +const voe::ChannelSendInterface* AudioSendStream::GetChannel() const { + return channel_send_.get(); } internal::AudioState* AudioSendStream::audio_state() { @@ -639,8 +620,8 @@ bool AudioSendStream::SetupSendCodec(AudioSendStream* stream, stream->StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels()); - stream->channel_proxy_->SetEncoder(new_config.send_codec_spec->payload_type, - std::move(encoder)); + stream->channel_send_->SetEncoder(new_config.send_codec_spec->payload_type, + std::move(encoder)); return true; } @@ -686,7 +667,7 @@ bool AudioSendStream::ReconfigureSendCodec(AudioSendStream* stream, if (!do_not_update_target_bitrate && new_target_bitrate_bps && new_target_bitrate_bps != old_config.send_codec_spec->target_bitrate_bps) { - CallEncoder(stream->channel_proxy_, [&](AudioEncoder* encoder) { + CallEncoder(stream->channel_send_, [&](AudioEncoder* encoder) { encoder->OnReceivedTargetAudioBitrate(*new_target_bitrate_bps); }); } @@ -704,7 +685,7 @@ void AudioSendStream::ReconfigureANA(AudioSendStream* stream, return; } if (new_config.audio_network_adaptor_config) { - CallEncoder(stream->channel_proxy_, [&](AudioEncoder* encoder) { + CallEncoder(stream->channel_send_, [&](AudioEncoder* encoder) { if (encoder->EnableAudioNetworkAdaptor( *new_config.audio_network_adaptor_config, stream->event_log_)) { RTC_DLOG(LS_INFO) << "Audio network adaptor enabled on SSRC " @@ -714,7 +695,7 @@ void AudioSendStream::ReconfigureANA(AudioSendStream* stream, } }); } else { - CallEncoder(stream->channel_proxy_, [&](AudioEncoder* encoder) { + CallEncoder(stream->channel_send_, [&](AudioEncoder* encoder) { encoder->DisableAudioNetworkAdaptor(); }); RTC_DLOG(LS_INFO) << "Audio network adaptor disabled on SSRC " @@ -738,7 +719,7 @@ void AudioSendStream::ReconfigureCNG(AudioSendStream* stream, } // Wrap or unwrap the encoder in an AudioEncoderCNG. - stream->channel_proxy_->ModifyEncoder( + stream->channel_send_->ModifyEncoder( [&](std::unique_ptr* encoder_ptr) { std::unique_ptr old_encoder(std::move(*encoder_ptr)); auto sub_encoders = old_encoder->ReclaimContainedEncoders(); diff --git a/audio/audio_send_stream.h b/audio/audio_send_stream.h index 5f7ea1498d..73e657e5d1 100644 --- a/audio/audio_send_stream.h +++ b/audio/audio_send_stream.h @@ -14,6 +14,7 @@ #include #include +#include "audio/channel_send.h" #include "audio/time_interval.h" #include "audio/transport_feedback_packet_loss_tracker.h" #include "call/audio_send_stream.h" @@ -30,11 +31,6 @@ class RtcpBandwidthObserver; class RtcpRttStats; class RtpTransportControllerSendInterface; -namespace voe { -class ChannelSend; -class ChannelSendProxy; -} // namespace voe - namespace internal { class AudioState; @@ -52,7 +48,7 @@ class AudioSendStream final : public webrtc::AudioSendStream, RtcpRttStats* rtcp_rtt_stats, const absl::optional& suspended_rtp_state, TimeInterval* overall_call_lifetime); - // For unit tests, which need to supply a mock channel proxy. + // For unit tests, which need to supply a mock ChannelSend. AudioSendStream(const webrtc::AudioSendStream::Config& config, const rtc::scoped_refptr& audio_state, rtc::TaskQueue* worker_queue, @@ -62,7 +58,7 @@ class AudioSendStream final : public webrtc::AudioSendStream, RtcpRttStats* rtcp_rtt_stats, const absl::optional& suspended_rtp_state, TimeInterval* overall_call_lifetime, - std::unique_ptr channel_proxy); + std::unique_ptr channel_send); ~AudioSendStream() override; // webrtc::AudioSendStream implementation. @@ -94,7 +90,7 @@ class AudioSendStream final : public webrtc::AudioSendStream, void SetTransportOverhead(int transport_overhead_per_packet); RtpState GetRtpState() const; - const voe::ChannelSend* GetChannel() const; + const voe::ChannelSendInterface* GetChannel() const; private: class TimedTransport; @@ -131,7 +127,7 @@ class AudioSendStream final : public webrtc::AudioSendStream, rtc::TaskQueue* worker_queue_; webrtc::AudioSendStream::Config config_; rtc::scoped_refptr audio_state_; - std::unique_ptr channel_proxy_; + const std::unique_ptr channel_send_; RtcEventLog* const event_log_; int encoder_sample_rate_hz_ = 0; diff --git a/audio/audio_send_stream_unittest.cc b/audio/audio_send_stream_unittest.cc index e226221715..6930cb1cd0 100644 --- a/audio/audio_send_stream_unittest.cc +++ b/audio/audio_send_stream_unittest.cc @@ -143,7 +143,7 @@ struct ConfigHelper { new rtc::RefCountedObject(); audio_state_ = AudioState::Create(config); - SetupDefaultChannelProxy(audio_bwe_enabled); + SetupDefaultChannelSend(audio_bwe_enabled); SetupMockForSetupSendCodec(expect_set_encoder_call); // Use ISAC as default codec so as to prevent unnecessary |channel_proxy_| @@ -169,7 +169,7 @@ struct ConfigHelper { stream_config_, audio_state_, &worker_queue_, &rtp_transport_, &bitrate_allocator_, &event_log_, &rtcp_rtt_stats_, absl::nullopt, &active_lifetime_, - std::unique_ptr(channel_proxy_))); + std::unique_ptr(channel_send_))); } AudioSendStream::Config& config() { return stream_config_; } @@ -177,7 +177,7 @@ struct ConfigHelper { return *static_cast( stream_config_.encoder_factory.get()); } - MockChannelSendProxy* channel_proxy() { return channel_proxy_; } + MockChannelSend* channel_send() { return channel_send_; } RtpTransportControllerSendInterface* transport() { return &rtp_transport_; } TimeInterval* active_lifetime() { return &active_lifetime_; } @@ -187,47 +187,46 @@ struct ConfigHelper { config->send_codec_spec->transport_cc_enabled = true; } - void SetupDefaultChannelProxy(bool audio_bwe_enabled) { - EXPECT_TRUE(channel_proxy_ == nullptr); - channel_proxy_ = new testing::StrictMock(); - EXPECT_CALL(*channel_proxy_, GetRtpRtcp()).WillRepeatedly(Invoke([this]() { + void SetupDefaultChannelSend(bool audio_bwe_enabled) { + EXPECT_TRUE(channel_send_ == nullptr); + channel_send_ = new testing::StrictMock(); + EXPECT_CALL(*channel_send_, GetRtpRtcp()).WillRepeatedly(Invoke([this]() { return &this->rtp_rtcp_; })); - EXPECT_CALL(*channel_proxy_, SetLocalSSRC(kSsrc)).Times(1); - EXPECT_CALL(*channel_proxy_, SetRTCP_CNAME(StrEq(kCName))).Times(1); - EXPECT_CALL(*channel_proxy_, SetNACKStatus(true, 10)).Times(1); - EXPECT_CALL(*channel_proxy_, SetFrameEncryptor(_)).Times(1); - EXPECT_CALL(*channel_proxy_, SetExtmapAllowMixed(false)).Times(1); - EXPECT_CALL(*channel_proxy_, + EXPECT_CALL(*channel_send_, SetLocalSSRC(kSsrc)).Times(1); + EXPECT_CALL(*channel_send_, SetRTCP_CNAME(StrEq(kCName))).Times(1); + EXPECT_CALL(*channel_send_, SetNACKStatus(true, 10)).Times(1); + EXPECT_CALL(*channel_send_, SetFrameEncryptor(_)).Times(1); + EXPECT_CALL(*channel_send_, SetExtmapAllowMixed(false)).Times(1); + EXPECT_CALL(*channel_send_, SetSendAudioLevelIndicationStatus(true, kAudioLevelId)) .Times(1); EXPECT_CALL(rtp_transport_, GetBandwidthObserver()) .WillRepeatedly(Return(&bandwidth_observer_)); if (audio_bwe_enabled) { - EXPECT_CALL(*channel_proxy_, + EXPECT_CALL(*channel_send_, EnableSendTransportSequenceNumber(kTransportSequenceNumberId)) .Times(1); - EXPECT_CALL(*channel_proxy_, + EXPECT_CALL(*channel_send_, RegisterSenderCongestionControlObjects( &rtp_transport_, Eq(&bandwidth_observer_))) .Times(1); } else { - EXPECT_CALL(*channel_proxy_, RegisterSenderCongestionControlObjects( - &rtp_transport_, Eq(nullptr))) + EXPECT_CALL(*channel_send_, RegisterSenderCongestionControlObjects( + &rtp_transport_, Eq(nullptr))) .Times(1); } - EXPECT_CALL(*channel_proxy_, ResetSenderCongestionControlObjects()) - .Times(1); + EXPECT_CALL(*channel_send_, ResetSenderCongestionControlObjects()).Times(1); { ::testing::InSequence unregister_on_destruction; - EXPECT_CALL(*channel_proxy_, RegisterTransport(_)).Times(1); - EXPECT_CALL(*channel_proxy_, RegisterTransport(nullptr)).Times(1); + EXPECT_CALL(*channel_send_, RegisterTransport(_)).Times(1); + EXPECT_CALL(*channel_send_, RegisterTransport(nullptr)).Times(1); } } void SetupMockForSetupSendCodec(bool expect_set_encoder_call) { if (expect_set_encoder_call) { - EXPECT_CALL(*channel_proxy_, SetEncoderForMock(_, _)) + EXPECT_CALL(*channel_send_, SetEncoderForMock(_, _)) .WillOnce(Invoke( [this](int payload_type, std::unique_ptr* encoder) { this->audio_encoder_ = std::move(*encoder); @@ -238,7 +237,7 @@ struct ConfigHelper { void SetupMockForModifyEncoder() { // Let ModifyEncoder to invoke mock audio encoder. - EXPECT_CALL(*channel_proxy_, ModifyEncoder(_)) + EXPECT_CALL(*channel_send_, ModifyEncoder(_)) .WillRepeatedly(Invoke( [this](rtc::FunctionView*)> modifier) { @@ -248,13 +247,13 @@ struct ConfigHelper { } void SetupMockForSendTelephoneEvent() { - EXPECT_TRUE(channel_proxy_); - EXPECT_CALL(*channel_proxy_, SetSendTelephoneEventPayloadType( - kTelephoneEventPayloadType, - kTelephoneEventPayloadFrequency)) + EXPECT_TRUE(channel_send_); + EXPECT_CALL(*channel_send_, SetSendTelephoneEventPayloadType( + kTelephoneEventPayloadType, + kTelephoneEventPayloadFrequency)) .WillOnce(Return(true)); EXPECT_CALL( - *channel_proxy_, + *channel_send_, SendTelephoneEventOutband(kTelephoneEventCode, kTelephoneEventDuration)) .WillOnce(Return(true)); } @@ -272,14 +271,14 @@ struct ConfigHelper { block.fraction_lost = 0; report_blocks.push_back(block); // Duplicate SSRC, bad fraction_lost. - EXPECT_TRUE(channel_proxy_); - EXPECT_CALL(*channel_proxy_, GetRTCPStatistics()) + EXPECT_TRUE(channel_send_); + EXPECT_CALL(*channel_send_, GetRTCPStatistics()) .WillRepeatedly(Return(kCallStats)); - EXPECT_CALL(*channel_proxy_, GetRemoteRTCPReportBlocks()) + EXPECT_CALL(*channel_send_, GetRemoteRTCPReportBlocks()) .WillRepeatedly(Return(report_blocks)); - EXPECT_CALL(*channel_proxy_, GetANAStatistics()) + EXPECT_CALL(*channel_send_, GetANAStatistics()) .WillRepeatedly(Return(ANAStats())); - EXPECT_CALL(*channel_proxy_, GetBitrate()).WillRepeatedly(Return(0)); + EXPECT_CALL(*channel_send_, GetBitrate()).WillRepeatedly(Return(0)); audio_processing_stats_.echo_return_loss = kEchoReturnLoss; audio_processing_stats_.echo_return_loss_enhancement = @@ -299,7 +298,7 @@ struct ConfigHelper { private: rtc::scoped_refptr audio_state_; AudioSendStream::Config stream_config_; - testing::StrictMock* channel_proxy_ = nullptr; + testing::StrictMock* channel_send_ = nullptr; rtc::scoped_refptr audio_processing_; AudioProcessingStats audio_processing_stats_; TimeInterval active_lifetime_; @@ -364,7 +363,7 @@ TEST(AudioSendStreamTest, SendTelephoneEvent) { TEST(AudioSendStreamTest, SetMuted) { ConfigHelper helper(false, true); auto send_stream = helper.CreateAudioSendStream(); - EXPECT_CALL(*helper.channel_proxy(), SetInputMute(true)); + EXPECT_CALL(*helper.channel_send(), SetInputMute(true)); send_stream->SetMuted(true); } @@ -454,7 +453,7 @@ TEST(AudioSendStreamTest, SendCodecCanApplyVad) { helper.config().send_codec_spec->cng_payload_type = 105; using ::testing::Invoke; std::unique_ptr stolen_encoder; - EXPECT_CALL(*helper.channel_proxy(), SetEncoderForMock(_, _)) + EXPECT_CALL(*helper.channel_send(), SetEncoderForMock(_, _)) .WillOnce( Invoke([&stolen_encoder](int payload_type, std::unique_ptr* encoder) { @@ -474,7 +473,7 @@ TEST(AudioSendStreamTest, SendCodecCanApplyVad) { TEST(AudioSendStreamTest, DoesNotPassHigherBitrateThanMaxBitrate) { ConfigHelper helper(false, true); auto send_stream = helper.CreateAudioSendStream(); - EXPECT_CALL(*helper.channel_proxy(), + EXPECT_CALL(*helper.channel_send(), SetBitrate(helper.config().max_bitrate_bps, _)); BitrateAllocationUpdate update; update.bitrate_bps = helper.config().max_bitrate_bps + 5000; @@ -487,7 +486,7 @@ TEST(AudioSendStreamTest, DoesNotPassHigherBitrateThanMaxBitrate) { TEST(AudioSendStreamTest, ProbingIntervalOnBitrateUpdated) { ConfigHelper helper(false, true); auto send_stream = helper.CreateAudioSendStream(); - EXPECT_CALL(*helper.channel_proxy(), SetBitrate(_, 5000)); + EXPECT_CALL(*helper.channel_send(), SetBitrate(_, 5000)); BitrateAllocationUpdate update; update.bitrate_bps = helper.config().max_bitrate_bps + 5000; update.fraction_loss = 0; @@ -504,7 +503,7 @@ TEST(AudioSendStreamTest, DontRecreateEncoder) { // to be correct, it's instead set-up manually here. Otherwise a simple change // to ConfigHelper (say to WillRepeatedly) would silently make this test // useless. - EXPECT_CALL(*helper.channel_proxy(), SetEncoderForMock(_, _)) + EXPECT_CALL(*helper.channel_send(), SetEncoderForMock(_, _)) .WillOnce(Return(true)); helper.config().send_codec_spec = @@ -519,15 +518,15 @@ TEST(AudioSendStreamTest, ReconfigureTransportCcResetsFirst) { auto send_stream = helper.CreateAudioSendStream(); auto new_config = helper.config(); ConfigHelper::AddBweToConfig(&new_config); - EXPECT_CALL(*helper.channel_proxy(), + EXPECT_CALL(*helper.channel_send(), EnableSendTransportSequenceNumber(kTransportSequenceNumberId)) .Times(1); { ::testing::InSequence seq; - EXPECT_CALL(*helper.channel_proxy(), ResetSenderCongestionControlObjects()) + EXPECT_CALL(*helper.channel_send(), ResetSenderCongestionControlObjects()) .Times(1); - EXPECT_CALL(*helper.channel_proxy(), RegisterSenderCongestionControlObjects( - helper.transport(), Ne(nullptr))) + EXPECT_CALL(*helper.channel_send(), RegisterSenderCongestionControlObjects( + helper.transport(), Ne(nullptr))) .Times(1); } send_stream->Reconfigure(new_config); @@ -543,11 +542,11 @@ TEST(AudioSendStreamTest, ReconfigureWithFrameEncryptor) { rtc::scoped_refptr mock_frame_encryptor_0( new rtc::RefCountedObject()); new_config.frame_encryptor = mock_frame_encryptor_0; - EXPECT_CALL(*helper.channel_proxy(), SetFrameEncryptor(Ne(nullptr))).Times(1); + EXPECT_CALL(*helper.channel_send(), SetFrameEncryptor(Ne(nullptr))).Times(1); send_stream->Reconfigure(new_config); // Not updating the frame encryptor shouldn't force it to reconfigure. - EXPECT_CALL(*helper.channel_proxy(), SetFrameEncryptor(_)).Times(0); + EXPECT_CALL(*helper.channel_send(), SetFrameEncryptor(_)).Times(0); send_stream->Reconfigure(new_config); // Updating frame encryptor to a new object should force a call to the proxy. @@ -555,7 +554,7 @@ TEST(AudioSendStreamTest, ReconfigureWithFrameEncryptor) { new rtc::RefCountedObject()); new_config.frame_encryptor = mock_frame_encryptor_1; new_config.crypto_options.sframe.require_frame_encryption = true; - EXPECT_CALL(*helper.channel_proxy(), SetFrameEncryptor(Ne(nullptr))).Times(1); + EXPECT_CALL(*helper.channel_send(), SetFrameEncryptor(Ne(nullptr))).Times(1); send_stream->Reconfigure(new_config); } @@ -568,7 +567,7 @@ TEST(AudioSendStreamTest, UpdateLifetime) { helper.config().send_transport = &mock_transport; Transport* registered_transport; - ON_CALL(*helper.channel_proxy(), RegisterTransport(_)) + ON_CALL(*helper.channel_send(), RegisterTransport(_)) .WillByDefault(Invoke([®istered_transport](Transport* transport) { registered_transport = transport; })); diff --git a/audio/channel_receive.cc b/audio/channel_receive.cc index e2ce7537bf..bdf3afef80 100644 --- a/audio/channel_receive.cc +++ b/audio/channel_receive.cc @@ -201,7 +201,7 @@ class ChannelReceive : public ChannelReceiveInterface, // Associate to a send channel. // Used for obtaining RTT for a receive-only channel. - void SetAssociatedSendChannel(const ChannelSend* channel) override; + void SetAssociatedSendChannel(const ChannelSendInterface* channel) override; std::vector GetSources() const override; @@ -294,7 +294,7 @@ class ChannelReceive : public ChannelReceiveInterface, // An associated send channel. rtc::CriticalSection assoc_send_channel_lock_; - const ChannelSend* associated_send_channel_ + const ChannelSendInterface* associated_send_channel_ RTC_GUARDED_BY(assoc_send_channel_lock_); PacketRouter* packet_router_ = nullptr; @@ -889,7 +889,8 @@ int ChannelReceive::ResendPackets(const uint16_t* sequence_numbers, return _rtpRtcpModule->SendNACK(sequence_numbers, length); } -void ChannelReceive::SetAssociatedSendChannel(const ChannelSend* channel) { +void ChannelReceive::SetAssociatedSendChannel( + const ChannelSendInterface* channel) { RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); rtc::CritScope lock(&assoc_send_channel_lock_); associated_send_channel_ = channel; @@ -1012,7 +1013,6 @@ int64_t ChannelReceive::GetRTT() const { return 0; } - RtcpMode method = _rtpRtcpModule->RTCP(); if (method == RtcpMode::kOff) { return 0; diff --git a/audio/channel_receive.h b/audio/channel_receive.h index fbfd7f038b..ffdd196b99 100644 --- a/audio/channel_receive.h +++ b/audio/channel_receive.h @@ -62,7 +62,7 @@ struct CallReceiveStatistics { namespace voe { -class ChannelSend; +class ChannelSendInterface; // Interface class needed for AudioReceiveStream tests that use a // MockChannelReceive. @@ -120,7 +120,8 @@ class ChannelReceiveInterface : public RtpPacketSinkInterface { // Associate to a send channel. // Used for obtaining RTT for a receive-only channel. - virtual void SetAssociatedSendChannel(const ChannelSend* channel) = 0; + virtual void SetAssociatedSendChannel( + const ChannelSendInterface* channel) = 0; virtual std::vector GetSources() const = 0; }; diff --git a/audio/channel_send.cc b/audio/channel_send.cc index e96ef4afde..8126adc14b 100644 --- a/audio/channel_send.cc +++ b/audio/channel_send.cc @@ -19,12 +19,16 @@ #include "absl/memory/memory.h" #include "api/array_view.h" +#include "api/call/transport.h" #include "api/crypto/frameencryptorinterface.h" #include "audio/utility/audio_frame_operations.h" #include "call/rtp_transport_controller_send_interface.h" +#include "common_types.h" // NOLINT(build/include) #include "logging/rtc_event_log/events/rtc_event_audio_playout.h" #include "logging/rtc_event_log/rtc_event_log.h" #include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/audio_processing/rms_level.h" #include "modules/pacing/packet_router.h" #include "modules/utility/include/process_thread.h" #include "rtc_base/checks.h" @@ -34,6 +38,7 @@ #include "rtc_base/location.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/race_checker.h" #include "rtc_base/rate_limiter.h" #include "rtc_base/task_queue.h" #include "rtc_base/thread_checker.h" @@ -67,7 +72,284 @@ MediaTransportFrameTypeForWebrtcFrameType(webrtc::FrameType frame_type) { } } -} // namespace +class RtpPacketSenderProxy; +class TransportFeedbackProxy; +class TransportSequenceNumberProxy; +class VoERtcpObserver; + +// Helper class to simplify locking scheme for members that are accessed from +// multiple threads. +// Example: a member can be set on thread T1 and read by an internal audio +// thread T2. Accessing the member via this class ensures that we are +// safe and also avoid TSan v2 warnings. +class ChannelSendState { + public: + struct State { + bool sending = false; + }; + + ChannelSendState() {} + virtual ~ChannelSendState() {} + + void Reset() { + rtc::CritScope lock(&lock_); + state_ = State(); + } + + State Get() const { + rtc::CritScope lock(&lock_); + return state_; + } + + void SetSending(bool enable) { + rtc::CritScope lock(&lock_); + state_.sending = enable; + } + + private: + rtc::CriticalSection lock_; + State state_; +}; + +class ChannelSend + : public ChannelSendInterface, + public Transport, + public OverheadObserver, + public AudioPacketizationCallback, // receive encoded packets from the + // ACM + public TargetTransferRateObserver { + public: + // TODO(nisse): Make OnUplinkPacketLossRate public, and delete friend + // declaration. + friend class VoERtcpObserver; + + ChannelSend(rtc::TaskQueue* encoder_queue, + ProcessThread* module_process_thread, + MediaTransportInterface* media_transport, + RtcpRttStats* rtcp_rtt_stats, + RtcEventLog* rtc_event_log, + FrameEncryptorInterface* frame_encryptor, + const webrtc::CryptoOptions& crypto_options, + bool extmap_allow_mixed, + int rtcp_report_interval_ms); + + ~ChannelSend() override; + + // Send using this encoder, with this payload type. + bool SetEncoder(int payload_type, + std::unique_ptr encoder) override; + void ModifyEncoder(rtc::FunctionView*)> + modifier) override; + + // API methods + + void StartSend() override; + void StopSend() override; + + // Codecs + void SetBitrate(int bitrate_bps, int64_t probing_interval_ms) override; + int GetBitrate() const override; + + // Network + void RegisterTransport(Transport* transport) override; + + bool ReceivedRTCPPacket(const uint8_t* data, size_t length) override; + + // Muting, Volume and Level. + void SetInputMute(bool enable) override; + + // Stats. + ANAStats GetANAStatistics() const override; + + // Used by AudioSendStream. + RtpRtcp* GetRtpRtcp() const override; + + // DTMF. + bool SendTelephoneEventOutband(int event, int duration_ms) override; + bool SetSendTelephoneEventPayloadType(int payload_type, + int payload_frequency) override; + + // RTP+RTCP + void SetLocalSSRC(uint32_t ssrc) override; + + void SetMid(const std::string& mid, int extension_id) override; + void SetExtmapAllowMixed(bool extmap_allow_mixed) override; + void SetSendAudioLevelIndicationStatus(bool enable, int id) override; + void EnableSendTransportSequenceNumber(int id) override; + + void RegisterSenderCongestionControlObjects( + RtpTransportControllerSendInterface* transport, + RtcpBandwidthObserver* bandwidth_observer) override; + void ResetSenderCongestionControlObjects() override; + void SetRTCP_CNAME(absl::string_view c_name) override; + std::vector GetRemoteRTCPReportBlocks() const override; + CallSendStatistics GetRTCPStatistics() const override; + void SetNACKStatus(bool enable, int max_packets) override; + + // ProcessAndEncodeAudio() posts a task on the shared encoder task queue, + // which in turn calls (on the queue) ProcessAndEncodeAudioOnTaskQueue() where + // the actual processing of the audio takes place. The processing mainly + // consists of encoding and preparing the result for sending by adding it to a + // send queue. + // The main reason for using a task queue here is to release the native, + // OS-specific, audio capture thread as soon as possible to ensure that it + // can go back to sleep and be prepared to deliver an new captured audio + // packet. + void ProcessAndEncodeAudio(std::unique_ptr audio_frame) override; + + void SetTransportOverhead(size_t transport_overhead_per_packet) override; + + // The existence of this function alongside OnUplinkPacketLossRate is + // a compromise. We want the encoder to be agnostic of the PLR source, but + // we also don't want it to receive conflicting information from TWCC and + // from RTCP-XR. + void OnTwccBasedUplinkPacketLossRate(float packet_loss_rate) override; + + void OnRecoverableUplinkPacketLossRate( + float recoverable_packet_loss_rate) override; + + int64_t GetRTT() const override; + + // E2EE Custom Audio Frame Encryption + void SetFrameEncryptor( + rtc::scoped_refptr frame_encryptor) override; + + private: + class ProcessAndEncodeAudioTask; + + // From AudioPacketizationCallback in the ACM + int32_t SendData(FrameType frameType, + uint8_t payloadType, + uint32_t timeStamp, + const uint8_t* payloadData, + size_t payloadSize, + const RTPFragmentationHeader* fragmentation) override; + + // From Transport (called by the RTP/RTCP module) + bool SendRtp(const uint8_t* data, + size_t len, + const PacketOptions& packet_options) override; + bool SendRtcp(const uint8_t* data, size_t len) override; + + bool Sending() const { return channel_state_.Get().sending; } + + // From OverheadObserver in the RTP/RTCP module + void OnOverheadChanged(size_t overhead_bytes_per_packet) override; + + void OnUplinkPacketLossRate(float packet_loss_rate); + bool InputMute() const; + + int ResendPackets(const uint16_t* sequence_numbers, int length); + + int SetSendRtpHeaderExtension(bool enable, RTPExtensionType type, int id); + + void UpdateOverheadForEncoder() + RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_); + + int GetRtpTimestampRateHz() const; + + int32_t SendRtpAudio(FrameType frameType, + uint8_t payloadType, + uint32_t timeStamp, + rtc::ArrayView payload, + const RTPFragmentationHeader* fragmentation); + + int32_t SendMediaTransportAudio(FrameType frameType, + uint8_t payloadType, + uint32_t timeStamp, + rtc::ArrayView payload, + const RTPFragmentationHeader* fragmentation); + + // Return media transport or nullptr if using RTP. + MediaTransportInterface* media_transport() { return media_transport_; } + + // Called on the encoder task queue when a new input audio frame is ready + // for encoding. + void ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input); + + void OnReceivedRtt(int64_t rtt_ms); + + void OnTargetTransferRate(TargetTransferRate) override; + + // Thread checkers document and lock usage of some methods on voe::Channel to + // specific threads we know about. The goal is to eventually split up + // voe::Channel into parts with single-threaded semantics, and thereby reduce + // the need for locks. + rtc::ThreadChecker worker_thread_checker_; + rtc::ThreadChecker module_process_thread_checker_; + // Methods accessed from audio and video threads are checked for sequential- + // only access. We don't necessarily own and control these threads, so thread + // checkers cannot be used. E.g. Chromium may transfer "ownership" from one + // audio thread to another, but access is still sequential. + rtc::RaceChecker audio_thread_race_checker_; + + rtc::CriticalSection _callbackCritSect; + rtc::CriticalSection volume_settings_critsect_; + + ChannelSendState channel_state_; + + RtcEventLog* const event_log_; + + std::unique_ptr _rtpRtcpModule; + + std::unique_ptr audio_coding_; + uint32_t _timeStamp RTC_GUARDED_BY(encoder_queue_); + + uint16_t send_sequence_number_; + + // uses + ProcessThread* _moduleProcessThreadPtr; + Transport* _transportPtr; // WebRtc socket or external transport + RmsLevel rms_level_ RTC_GUARDED_BY(encoder_queue_); + bool input_mute_ RTC_GUARDED_BY(volume_settings_critsect_); + bool previous_frame_muted_ RTC_GUARDED_BY(encoder_queue_); + // VoeRTP_RTCP + // TODO(henrika): can today be accessed on the main thread and on the + // task queue; hence potential race. + bool _includeAudioLevelIndication; + size_t transport_overhead_per_packet_ + RTC_GUARDED_BY(overhead_per_packet_lock_); + size_t rtp_overhead_per_packet_ RTC_GUARDED_BY(overhead_per_packet_lock_); + rtc::CriticalSection overhead_per_packet_lock_; + // RtcpBandwidthObserver + std::unique_ptr rtcp_observer_; + + PacketRouter* packet_router_ = nullptr; + std::unique_ptr feedback_observer_proxy_; + std::unique_ptr seq_num_allocator_proxy_; + std::unique_ptr rtp_packet_sender_proxy_; + std::unique_ptr retransmission_rate_limiter_; + + rtc::ThreadChecker construction_thread_; + + const bool use_twcc_plr_for_ana_; + + rtc::CriticalSection encoder_queue_lock_; + bool encoder_queue_is_active_ RTC_GUARDED_BY(encoder_queue_lock_) = false; + rtc::TaskQueue* encoder_queue_ = nullptr; + + MediaTransportInterface* const media_transport_; + int media_transport_sequence_number_ RTC_GUARDED_BY(encoder_queue_) = 0; + + rtc::CriticalSection media_transport_lock_; + // Currently set by SetLocalSSRC. + uint64_t media_transport_channel_id_ RTC_GUARDED_BY(&media_transport_lock_) = + 0; + // Cache payload type and sampling frequency from most recent call to + // SetEncoder. Needed to set MediaTransportEncodedAudioFrame metadata, and + // invalidate on encoder change. + int media_transport_payload_type_ RTC_GUARDED_BY(&media_transport_lock_); + int media_transport_sampling_frequency_ + RTC_GUARDED_BY(&media_transport_lock_); + + // E2EE Audio Frame Encryption + rtc::scoped_refptr frame_encryptor_; + // E2EE Frame Encryption Options + webrtc::CryptoOptions crypto_options_; + + rtc::CriticalSection bitrate_crit_section_; + int configured_bitrate_bps_ RTC_GUARDED_BY(bitrate_crit_section_) = 0; +}; const int kTelephoneEventAttenuationdB = 10; @@ -476,6 +758,8 @@ ChannelSend::ChannelSend(rtc::TaskQueue* encoder_queue, crypto_options_(crypto_options) { RTC_DCHECK(module_process_thread); RTC_DCHECK(encoder_queue); + module_process_thread_checker_.DetachFromThread(); + audio_coding_.reset(AudioCodingModule::Create(AudioCodingModule::Config())); RtpRtcp::Configuration configuration; @@ -556,6 +840,7 @@ ChannelSend::~ChannelSend() { } void ChannelSend::StartSend() { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); RTC_DCHECK(!channel_state_.Get().sending); channel_state_.SetSending(true); @@ -575,6 +860,7 @@ void ChannelSend::StartSend() { } void ChannelSend::StopSend() { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); if (!channel_state_.Get().sending) { return; } @@ -616,6 +902,7 @@ void ChannelSend::StopSend() { bool ChannelSend::SetEncoder(int payload_type, std::unique_ptr encoder) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); RTC_DCHECK_GE(payload_type, 0); RTC_DCHECK_LE(payload_type, 127); // TODO(ossu): Make CodecInsts up, for now: one for the RTP/RTCP module and @@ -659,11 +946,19 @@ bool ChannelSend::SetEncoder(int payload_type, void ChannelSend::ModifyEncoder( rtc::FunctionView*)> modifier) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); audio_coding_->ModifyEncoder(modifier); } -void ChannelSend::SetBitRate(int bitrate_bps, int64_t probing_interval_ms) { +void ChannelSend::SetBitrate(int bitrate_bps, int64_t probing_interval_ms) { + // This method can be called on the worker thread, module process thread + // or on a TaskQueue via VideoSendStreamImpl::OnEncoderConfigurationChanged. + // TODO(solenberg): Figure out a good way to check this or enforce calling + // rules. + // RTC_DCHECK(worker_thread_checker_.CalledOnValidThread() || + // module_process_thread_checker_.CalledOnValidThread()); rtc::CritScope lock(&bitrate_crit_section_); + audio_coding_->ModifyEncoder([&](std::unique_ptr* encoder) { if (*encoder) { (*encoder)->OnReceivedUplinkBandwidth(bitrate_bps, probing_interval_ms); @@ -673,12 +968,13 @@ void ChannelSend::SetBitRate(int bitrate_bps, int64_t probing_interval_ms) { configured_bitrate_bps_ = bitrate_bps; } -int ChannelSend::GetBitRate() const { +int ChannelSend::GetBitrate() const { rtc::CritScope lock(&bitrate_crit_section_); return configured_bitrate_bps_; } void ChannelSend::OnTwccBasedUplinkPacketLossRate(float packet_loss_rate) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); if (!use_twcc_plr_for_ana_) return; audio_coding_->ModifyEncoder([&](std::unique_ptr* encoder) { @@ -690,6 +986,7 @@ void ChannelSend::OnTwccBasedUplinkPacketLossRate(float packet_loss_rate) { void ChannelSend::OnRecoverableUplinkPacketLossRate( float recoverable_packet_loss_rate) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); audio_coding_->ModifyEncoder([&](std::unique_ptr* encoder) { if (*encoder) { (*encoder)->OnReceivedUplinkRecoverablePacketLossFraction( @@ -709,12 +1006,14 @@ void ChannelSend::OnUplinkPacketLossRate(float packet_loss_rate) { } void ChannelSend::RegisterTransport(Transport* transport) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); rtc::CritScope cs(&_callbackCritSect); _transportPtr = transport; } // TODO(nisse): Delete always-true return value. bool ChannelSend::ReceivedRTCPPacket(const uint8_t* data, size_t length) { + // May be called on either worker thread or network thread. if (media_transport_) { // Ignore RTCP packets while media transport is used. // Those packets should not arrive, but we are seeing occasional packets. @@ -743,6 +1042,7 @@ bool ChannelSend::ReceivedRTCPPacket(const uint8_t* data, size_t length) { } void ChannelSend::SetInputMute(bool enable) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); rtc::CritScope cs(&volume_settings_critsect_); input_mute_ = enable; } @@ -753,6 +1053,7 @@ bool ChannelSend::InputMute() const { } bool ChannelSend::SendTelephoneEventOutband(int event, int duration_ms) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); RTC_DCHECK_LE(0, event); RTC_DCHECK_GE(255, event); RTC_DCHECK_LE(0, duration_ms); @@ -770,6 +1071,7 @@ bool ChannelSend::SendTelephoneEventOutband(int event, int duration_ms) { bool ChannelSend::SetSendTelephoneEventPayloadType(int payload_type, int payload_frequency) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); RTC_DCHECK_LE(0, payload_type); RTC_DCHECK_GE(127, payload_type); CodecInst codec = {0}; @@ -788,7 +1090,8 @@ bool ChannelSend::SetSendTelephoneEventPayloadType(int payload_type, return true; } -void ChannelSend::SetLocalSSRC(unsigned int ssrc) { +void ChannelSend::SetLocalSSRC(uint32_t ssrc) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); RTC_DCHECK(!channel_state_.Get().sending); if (media_transport_) { @@ -799,22 +1102,26 @@ void ChannelSend::SetLocalSSRC(unsigned int ssrc) { } void ChannelSend::SetMid(const std::string& mid, int extension_id) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); int ret = SetSendRtpHeaderExtension(true, kRtpExtensionMid, extension_id); RTC_DCHECK_EQ(0, ret); _rtpRtcpModule->SetMid(mid); } void ChannelSend::SetExtmapAllowMixed(bool extmap_allow_mixed) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); _rtpRtcpModule->SetExtmapAllowMixed(extmap_allow_mixed); } void ChannelSend::SetSendAudioLevelIndicationStatus(bool enable, int id) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); _includeAudioLevelIndication = enable; int ret = SetSendRtpHeaderExtension(enable, kRtpExtensionAudioLevel, id); RTC_DCHECK_EQ(0, ret); } void ChannelSend::EnableSendTransportSequenceNumber(int id) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); int ret = SetSendRtpHeaderExtension(true, kRtpExtensionTransportSequenceNumber, id); RTC_DCHECK_EQ(0, ret); @@ -823,6 +1130,7 @@ void ChannelSend::EnableSendTransportSequenceNumber(int id) { void ChannelSend::RegisterSenderCongestionControlObjects( RtpTransportControllerSendInterface* transport, RtcpBandwidthObserver* bandwidth_observer) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); RtpPacketSender* rtp_packet_sender = transport->packet_sender(); TransportFeedbackObserver* transport_feedback_observer = transport->transport_feedback_observer(); @@ -844,6 +1152,7 @@ void ChannelSend::RegisterSenderCongestionControlObjects( } void ChannelSend::ResetSenderCongestionControlObjects() { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); RTC_DCHECK(packet_router_); _rtpRtcpModule->SetStorePacketsStatus(false, 600); rtcp_observer_->SetBandwidthObserver(nullptr); @@ -855,6 +1164,7 @@ void ChannelSend::ResetSenderCongestionControlObjects() { } void ChannelSend::SetRTCP_CNAME(absl::string_view c_name) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); // Note: SetCNAME() accepts a c string of length at most 255. const std::string c_name_limited(c_name.substr(0, 255)); int ret = _rtpRtcpModule->SetCNAME(c_name_limited.c_str()) != 0; @@ -862,6 +1172,7 @@ void ChannelSend::SetRTCP_CNAME(absl::string_view c_name) { } std::vector ChannelSend::GetRemoteRTCPReportBlocks() const { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); // Get the report blocks from the latest received RTCP Sender or Receiver // Report. Each element in the vector contains the sender's SSRC and a // report block according to RFC 3550. @@ -890,6 +1201,7 @@ std::vector ChannelSend::GetRemoteRTCPReportBlocks() const { } CallSendStatistics ChannelSend::GetRTCPStatistics() const { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); CallSendStatistics stats = {0}; stats.rttMs = GetRTT(); @@ -908,10 +1220,11 @@ CallSendStatistics ChannelSend::GetRTCPStatistics() const { return stats; } -void ChannelSend::SetNACKStatus(bool enable, int maxNumberOfPackets) { +void ChannelSend::SetNACKStatus(bool enable, int max_packets) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); // None of these functions can fail. if (enable) - audio_coding_->EnableNack(maxNumberOfPackets); + audio_coding_->EnableNack(max_packets); else audio_coding_->DisableNack(); } @@ -923,6 +1236,7 @@ int ChannelSend::ResendPackets(const uint16_t* sequence_numbers, int length) { void ChannelSend::ProcessAndEncodeAudio( std::unique_ptr audio_frame) { + RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_); // Avoid posting any new tasks if sending was already stopped in StopSend(). rtc::CritScope cs(&encoder_queue_lock_); if (!encoder_queue_is_active_) { @@ -988,6 +1302,7 @@ void ChannelSend::UpdateOverheadForEncoder() { } void ChannelSend::SetTransportOverhead(size_t transport_overhead_per_packet) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); rtc::CritScope cs(&overhead_per_packet_lock_); transport_overhead_per_packet_ = transport_overhead_per_packet; UpdateOverheadForEncoder(); @@ -1001,10 +1316,12 @@ void ChannelSend::OnOverheadChanged(size_t overhead_bytes_per_packet) { } ANAStats ChannelSend::GetANAStatistics() const { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); return audio_coding_->GetANAStats(); } RtpRtcp* ChannelSend::GetRtpRtcp() const { + RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread()); return _rtpRtcpModule.get(); } @@ -1046,7 +1363,6 @@ int64_t ChannelSend::GetRTT() const { return 0; } - RtcpMode method = _rtpRtcpModule->RTCP(); if (method == RtcpMode::kOff) { return 0; @@ -1073,6 +1389,7 @@ int64_t ChannelSend::GetRTT() const { void ChannelSend::SetFrameEncryptor( rtc::scoped_refptr frame_encryptor) { + RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); rtc::CritScope cs(&encoder_queue_lock_); if (encoder_queue_is_active_) { encoder_queue_->PostTask([this, frame_encryptor]() { @@ -1098,5 +1415,23 @@ void ChannelSend::OnReceivedRtt(int64_t rtt_ms) { }); } +} // namespace + +std::unique_ptr CreateChannelSend( + rtc::TaskQueue* encoder_queue, + ProcessThread* module_process_thread, + MediaTransportInterface* media_transport, + RtcpRttStats* rtcp_rtt_stats, + RtcEventLog* rtc_event_log, + FrameEncryptorInterface* frame_encryptor, + const webrtc::CryptoOptions& crypto_options, + bool extmap_allow_mixed, + int rtcp_report_interval_ms) { + return absl::make_unique( + encoder_queue, module_process_thread, media_transport, rtcp_rtt_stats, + rtc_event_log, frame_encryptor, crypto_options, extmap_allow_mixed, + rtcp_report_interval_ms); +} + } // namespace voe } // namespace webrtc diff --git a/audio/channel_send.h b/audio/channel_send.h index 3d194a7ecd..a5f718509f 100644 --- a/audio/channel_send.h +++ b/audio/channel_send.h @@ -11,44 +11,26 @@ #ifndef AUDIO_CHANNEL_SEND_H_ #define AUDIO_CHANNEL_SEND_H_ -#include #include #include #include #include "api/audio/audio_frame.h" #include "api/audio_codecs/audio_encoder.h" -#include "api/call/transport.h" #include "api/crypto/cryptooptions.h" #include "api/media_transport_interface.h" -#include "common_types.h" // NOLINT(build/include) -#include "modules/audio_coding/include/audio_coding_module.h" -#include "modules/audio_processing/rms_level.h" #include "modules/rtp_rtcp/include/rtp_rtcp.h" -#include "rtc_base/criticalsection.h" +#include "rtc_base/function_view.h" #include "rtc_base/task_queue.h" -#include "rtc_base/thread_checker.h" - -// TODO(solenberg, nisse): This file contains a few NOLINT marks, to silence -// warnings about use of unsigned short, and non-const reference arguments. -// These need cleanup, in a separate cl. - -namespace rtc { -class TimestampWrapAroundHandler; -} namespace webrtc { class FrameEncryptorInterface; -class PacketRouter; class ProcessThread; -class RateLimiter; class RtcEventLog; class RtpRtcp; class RtpTransportControllerSendInterface; -struct SenderInfo; - struct CallSendStatistics { int64_t rttMs; size_t bytesSent; @@ -69,146 +51,48 @@ struct ReportBlock { namespace voe { -class RtpPacketSenderProxy; -class TransportFeedbackProxy; -class TransportSequenceNumberProxy; -class VoERtcpObserver; - -// Helper class to simplify locking scheme for members that are accessed from -// multiple threads. -// Example: a member can be set on thread T1 and read by an internal audio -// thread T2. Accessing the member via this class ensures that we are -// safe and also avoid TSan v2 warnings. -class ChannelSendState { +class ChannelSendInterface { public: - struct State { - bool sending = false; - }; + virtual ~ChannelSendInterface() = default; - ChannelSendState() {} - virtual ~ChannelSendState() {} + virtual void RegisterTransport(Transport* transport) = 0; + virtual bool ReceivedRTCPPacket(const uint8_t* packet, size_t length) = 0; - void Reset() { - rtc::CritScope lock(&lock_); - state_ = State(); - } + virtual CallSendStatistics GetRTCPStatistics() const = 0; + virtual void SetNACKStatus(bool enable, int max_packets) = 0; - State Get() const { - rtc::CritScope lock(&lock_); - return state_; - } + virtual bool SetEncoder(int payload_type, + std::unique_ptr encoder) = 0; + virtual void ModifyEncoder( + rtc::FunctionView*)> modifier) = 0; - void SetSending(bool enable) { - rtc::CritScope lock(&lock_); - state_.sending = enable; - } - - private: - rtc::CriticalSection lock_; - State state_; -}; - -class ChannelSend : public Transport, - public AudioPacketizationCallback, // receive encoded - // packets from the ACM - public OverheadObserver, - public TargetTransferRateObserver { - public: - // TODO(nisse): Make OnUplinkPacketLossRate public, and delete friend - // declaration. - friend class VoERtcpObserver; - - ChannelSend(rtc::TaskQueue* encoder_queue, - ProcessThread* module_process_thread, - MediaTransportInterface* media_transport, - RtcpRttStats* rtcp_rtt_stats, - RtcEventLog* rtc_event_log, - FrameEncryptorInterface* frame_encryptor, - const webrtc::CryptoOptions& crypto_options, - bool extmap_allow_mixed, - int rtcp_report_interval_ms); - - virtual ~ChannelSend(); - - // Send using this encoder, with this payload type. - bool SetEncoder(int payload_type, std::unique_ptr encoder); - void ModifyEncoder( - rtc::FunctionView*)> modifier); - - // API methods - - void StartSend(); - void StopSend(); - - // Codecs - void SetBitRate(int bitrate_bps, int64_t probing_interval_ms); - int GetBitRate() const; - - // Network - void RegisterTransport(Transport* transport); - - bool ReceivedRTCPPacket(const uint8_t* data, size_t length); - - // Muting, Volume and Level. - void SetInputMute(bool enable); - - // Stats. - ANAStats GetANAStatistics() const; - - // Used by AudioSendStream. - RtpRtcp* GetRtpRtcp() const; - - // DTMF. - bool SendTelephoneEventOutband(int event, int duration_ms); - bool SetSendTelephoneEventPayloadType(int payload_type, - int payload_frequency); - - // RTP+RTCP - void SetLocalSSRC(uint32_t ssrc); - - void SetMid(const std::string& mid, int extension_id); - void SetExtmapAllowMixed(bool extmap_allow_mixed); - void SetSendAudioLevelIndicationStatus(bool enable, int id); - void EnableSendTransportSequenceNumber(int id); - - void RegisterSenderCongestionControlObjects( + virtual void SetLocalSSRC(uint32_t ssrc) = 0; + virtual void SetMid(const std::string& mid, int extension_id) = 0; + virtual void SetRTCP_CNAME(absl::string_view c_name) = 0; + virtual void SetExtmapAllowMixed(bool extmap_allow_mixed) = 0; + virtual void SetSendAudioLevelIndicationStatus(bool enable, int id) = 0; + virtual void EnableSendTransportSequenceNumber(int id) = 0; + virtual void RegisterSenderCongestionControlObjects( RtpTransportControllerSendInterface* transport, - RtcpBandwidthObserver* bandwidth_observer); - void ResetSenderCongestionControlObjects(); - void SetRTCP_CNAME(absl::string_view c_name); - std::vector GetRemoteRTCPReportBlocks() const; - CallSendStatistics GetRTCPStatistics() const; - void SetNACKStatus(bool enable, int maxNumberOfPackets); + RtcpBandwidthObserver* bandwidth_observer) = 0; + virtual void ResetSenderCongestionControlObjects() = 0; + virtual std::vector GetRemoteRTCPReportBlocks() const = 0; + virtual ANAStats GetANAStatistics() const = 0; + virtual bool SetSendTelephoneEventPayloadType(int payload_type, + int payload_frequency) = 0; + virtual bool SendTelephoneEventOutband(int event, int duration_ms) = 0; + virtual void SetBitrate(int bitrate_bps, int64_t probing_interval_ms) = 0; + virtual int GetBitrate() const = 0; + virtual void SetInputMute(bool muted) = 0; - // ProcessAndEncodeAudio() posts a task on the shared encoder task queue, - // which in turn calls (on the queue) ProcessAndEncodeAudioOnTaskQueue() where - // the actual processing of the audio takes place. The processing mainly - // consists of encoding and preparing the result for sending by adding it to a - // send queue. - // The main reason for using a task queue here is to release the native, - // OS-specific, audio capture thread as soon as possible to ensure that it - // can go back to sleep and be prepared to deliver an new captured audio - // packet. - void ProcessAndEncodeAudio(std::unique_ptr audio_frame); - - public: - void SetTransportOverhead(size_t transport_overhead_per_packet); - - // The existence of this function alongside OnUplinkPacketLossRate is - // a compromise. We want the encoder to be agnostic of the PLR source, but - // we also don't want it to receive conflicting information from TWCC and - // from RTCP-XR. - void OnTwccBasedUplinkPacketLossRate(float packet_loss_rate); - - void OnRecoverableUplinkPacketLossRate(float recoverable_packet_loss_rate); - - // Returns the RTT in milliseconds. - int64_t GetRTT() const; - - // E2EE Custom Audio Frame Encryption - void SetFrameEncryptor( - rtc::scoped_refptr frame_encryptor); + virtual void ProcessAndEncodeAudio( + std::unique_ptr audio_frame) = 0; + virtual void SetTransportOverhead(size_t transport_overhead_per_packet) = 0; + virtual RtpRtcp* GetRtpRtcp() const = 0; + virtual void OnTwccBasedUplinkPacketLossRate(float packet_loss_rate) = 0; + virtual void OnRecoverableUplinkPacketLossRate( + float recoverable_packet_loss_rate) = 0; // In RTP we currently rely on RTCP packets (|ReceivedRTCPPacket|) to inform // about RTT. // In media transport we rely on the TargetTransferRateObserver instead. @@ -218,131 +102,27 @@ class ChannelSend : public Transport, // // In future, RTP media will move to the media transport implementation and // these conditions will be removed. - void OnTargetTransferRate(TargetTransferRate rate) override; + // Returns the RTT in milliseconds. + virtual int64_t GetRTT() const = 0; + virtual void StartSend() = 0; + virtual void StopSend() = 0; - private: - class ProcessAndEncodeAudioTask; - - // From AudioPacketizationCallback in the ACM - int32_t SendData(FrameType frameType, - uint8_t payloadType, - uint32_t timeStamp, - const uint8_t* payloadData, - size_t payloadSize, - const RTPFragmentationHeader* fragmentation) override; - - // From Transport (called by the RTP/RTCP module) - bool SendRtp(const uint8_t* data, - size_t len, - const PacketOptions& packet_options) override; - bool SendRtcp(const uint8_t* data, size_t len) override; - - bool Sending() const { return channel_state_.Get().sending; } - - // From OverheadObserver in the RTP/RTCP module - void OnOverheadChanged(size_t overhead_bytes_per_packet) override; - - void OnUplinkPacketLossRate(float packet_loss_rate); - bool InputMute() const; - - int ResendPackets(const uint16_t* sequence_numbers, int length); - - int SetSendRtpHeaderExtension(bool enable, RTPExtensionType type, int id); - - void UpdateOverheadForEncoder() - RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_); - - int GetRtpTimestampRateHz() const; - - int32_t SendRtpAudio(FrameType frameType, - uint8_t payloadType, - uint32_t timeStamp, - rtc::ArrayView payload, - const RTPFragmentationHeader* fragmentation); - - int32_t SendMediaTransportAudio(FrameType frameType, - uint8_t payloadType, - uint32_t timeStamp, - rtc::ArrayView payload, - const RTPFragmentationHeader* fragmentation); - - // Return media transport or nullptr if using RTP. - MediaTransportInterface* media_transport() { return media_transport_; } - - // Called on the encoder task queue when a new input audio frame is ready - // for encoding. - void ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input); - - void OnReceivedRtt(int64_t rtt_ms); - - rtc::CriticalSection _callbackCritSect; - rtc::CriticalSection volume_settings_critsect_; - - ChannelSendState channel_state_; - - RtcEventLog* const event_log_; - - std::unique_ptr _rtpRtcpModule; - - std::unique_ptr audio_coding_; - uint32_t _timeStamp RTC_GUARDED_BY(encoder_queue_); - - uint16_t send_sequence_number_; - - // uses - ProcessThread* _moduleProcessThreadPtr; - Transport* _transportPtr; // WebRtc socket or external transport - RmsLevel rms_level_ RTC_GUARDED_BY(encoder_queue_); - bool input_mute_ RTC_GUARDED_BY(volume_settings_critsect_); - bool previous_frame_muted_ RTC_GUARDED_BY(encoder_queue_); - // VoeRTP_RTCP - // TODO(henrika): can today be accessed on the main thread and on the - // task queue; hence potential race. - bool _includeAudioLevelIndication; - size_t transport_overhead_per_packet_ - RTC_GUARDED_BY(overhead_per_packet_lock_); - size_t rtp_overhead_per_packet_ RTC_GUARDED_BY(overhead_per_packet_lock_); - rtc::CriticalSection overhead_per_packet_lock_; - // RtcpBandwidthObserver - std::unique_ptr rtcp_observer_; - - PacketRouter* packet_router_ = nullptr; - std::unique_ptr feedback_observer_proxy_; - std::unique_ptr seq_num_allocator_proxy_; - std::unique_ptr rtp_packet_sender_proxy_; - std::unique_ptr retransmission_rate_limiter_; - - rtc::ThreadChecker construction_thread_; - - const bool use_twcc_plr_for_ana_; - - rtc::CriticalSection encoder_queue_lock_; - bool encoder_queue_is_active_ RTC_GUARDED_BY(encoder_queue_lock_) = false; - rtc::TaskQueue* encoder_queue_ = nullptr; - - MediaTransportInterface* const media_transport_; - int media_transport_sequence_number_ RTC_GUARDED_BY(encoder_queue_) = 0; - - rtc::CriticalSection media_transport_lock_; - // Currently set by SetLocalSSRC. - uint64_t media_transport_channel_id_ RTC_GUARDED_BY(&media_transport_lock_) = - 0; - // Cache payload type and sampling frequency from most recent call to - // SetEncoder. Needed to set MediaTransportEncodedAudioFrame metadata, and - // invalidate on encoder change. - int media_transport_payload_type_ RTC_GUARDED_BY(&media_transport_lock_); - int media_transport_sampling_frequency_ - RTC_GUARDED_BY(&media_transport_lock_); - - // E2EE Audio Frame Encryption - rtc::scoped_refptr frame_encryptor_; - // E2EE Frame Encryption Options - webrtc::CryptoOptions crypto_options_; - - rtc::CriticalSection bitrate_crit_section_; - int configured_bitrate_bps_ RTC_GUARDED_BY(bitrate_crit_section_) = 0; + // E2EE Custom Audio Frame Encryption (Optional) + virtual void SetFrameEncryptor( + rtc::scoped_refptr frame_encryptor) = 0; }; +std::unique_ptr CreateChannelSend( + rtc::TaskQueue* encoder_queue, + ProcessThread* module_process_thread, + MediaTransportInterface* media_transport, + RtcpRttStats* rtcp_rtt_stats, + RtcEventLog* rtc_event_log, + FrameEncryptorInterface* frame_encryptor, + const webrtc::CryptoOptions& crypto_options, + bool extmap_allow_mixed, + int rtcp_report_interval_ms); + } // namespace voe } // namespace webrtc diff --git a/audio/channel_send_proxy.cc b/audio/channel_send_proxy.cc deleted file mode 100644 index 961528a8e5..0000000000 --- a/audio/channel_send_proxy.cc +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "audio/channel_send_proxy.h" - -#include - -#include "api/crypto/frameencryptorinterface.h" -#include "call/rtp_transport_controller_send_interface.h" -#include "rtc_base/checks.h" - -namespace webrtc { -namespace voe { -ChannelSendProxy::ChannelSendProxy() {} - -ChannelSendProxy::ChannelSendProxy(std::unique_ptr channel) - : channel_(std::move(channel)) { - RTC_DCHECK(channel_); - module_process_thread_checker_.DetachFromThread(); -} - -ChannelSendProxy::~ChannelSendProxy() {} - -void ChannelSendProxy::SetLocalSSRC(uint32_t ssrc) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - return channel_->SetLocalSSRC(ssrc); -} - -void ChannelSendProxy::SetNACKStatus(bool enable, int max_packets) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->SetNACKStatus(enable, max_packets); -} - -CallSendStatistics ChannelSendProxy::GetRTCPStatistics() const { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - return channel_->GetRTCPStatistics(); -} - -void ChannelSendProxy::RegisterTransport(Transport* transport) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->RegisterTransport(transport); -} - -bool ChannelSendProxy::ReceivedRTCPPacket(const uint8_t* packet, - size_t length) { - // May be called on either worker thread or network thread. - return channel_->ReceivedRTCPPacket(packet, length); -} - -bool ChannelSendProxy::SetEncoder(int payload_type, - std::unique_ptr encoder) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - return channel_->SetEncoder(payload_type, std::move(encoder)); -} - -void ChannelSendProxy::ModifyEncoder( - rtc::FunctionView*)> modifier) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->ModifyEncoder(modifier); -} - -void ChannelSendProxy::SetMid(const std::string& mid, int extension_id) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->SetMid(mid, extension_id); -} - -void ChannelSendProxy::SetRTCP_CNAME(absl::string_view c_name) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->SetRTCP_CNAME(c_name); -} - -void ChannelSendProxy::SetExtmapAllowMixed(bool extmap_allow_mixed) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->SetExtmapAllowMixed(extmap_allow_mixed); -} - -void ChannelSendProxy::SetSendAudioLevelIndicationStatus(bool enable, int id) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->SetSendAudioLevelIndicationStatus(enable, id); -} - -void ChannelSendProxy::EnableSendTransportSequenceNumber(int id) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->EnableSendTransportSequenceNumber(id); -} - -void ChannelSendProxy::RegisterSenderCongestionControlObjects( - RtpTransportControllerSendInterface* transport, - RtcpBandwidthObserver* bandwidth_observer) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->RegisterSenderCongestionControlObjects(transport, - bandwidth_observer); -} - -void ChannelSendProxy::ResetSenderCongestionControlObjects() { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->ResetSenderCongestionControlObjects(); -} - -std::vector ChannelSendProxy::GetRemoteRTCPReportBlocks() const { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - return channel_->GetRemoteRTCPReportBlocks(); -} - -ANAStats ChannelSendProxy::GetANAStatistics() const { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - return channel_->GetANAStatistics(); -} - -bool ChannelSendProxy::SetSendTelephoneEventPayloadType(int payload_type, - int payload_frequency) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - return channel_->SetSendTelephoneEventPayloadType(payload_type, - payload_frequency); -} - -bool ChannelSendProxy::SendTelephoneEventOutband(int event, int duration_ms) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - return channel_->SendTelephoneEventOutband(event, duration_ms); -} - -void ChannelSendProxy::SetBitrate(int bitrate_bps, - int64_t probing_interval_ms) { - // This method can be called on the worker thread, module process thread - // or on a TaskQueue via VideoSendStreamImpl::OnEncoderConfigurationChanged. - // TODO(solenberg): Figure out a good way to check this or enforce calling - // rules. - // RTC_DCHECK(worker_thread_checker_.CalledOnValidThread() || - // module_process_thread_checker_.CalledOnValidThread()); - channel_->SetBitRate(bitrate_bps, probing_interval_ms); -} - -int ChannelSendProxy::GetBitrate() const { - return channel_->GetBitRate(); -} - -void ChannelSendProxy::SetInputMute(bool muted) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->SetInputMute(muted); -} - -void ChannelSendProxy::ProcessAndEncodeAudio( - std::unique_ptr audio_frame) { - RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_); - return channel_->ProcessAndEncodeAudio(std::move(audio_frame)); -} - -void ChannelSendProxy::SetTransportOverhead(int transport_overhead_per_packet) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->SetTransportOverhead(transport_overhead_per_packet); -} - -RtpRtcp* ChannelSendProxy::GetRtpRtcp() const { - RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread()); - return channel_->GetRtpRtcp(); -} - -void ChannelSendProxy::OnTwccBasedUplinkPacketLossRate(float packet_loss_rate) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->OnTwccBasedUplinkPacketLossRate(packet_loss_rate); -} - -void ChannelSendProxy::OnRecoverableUplinkPacketLossRate( - float recoverable_packet_loss_rate) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->OnRecoverableUplinkPacketLossRate(recoverable_packet_loss_rate); -} - -void ChannelSendProxy::StartSend() { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->StartSend(); -} - -void ChannelSendProxy::StopSend() { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->StopSend(); -} - -ChannelSend* ChannelSendProxy::GetChannel() const { - return channel_.get(); -} - -void ChannelSendProxy::SetFrameEncryptor( - rtc::scoped_refptr frame_encryptor) { - RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); - channel_->SetFrameEncryptor(frame_encryptor); -} - -} // namespace voe -} // namespace webrtc diff --git a/audio/channel_send_proxy.h b/audio/channel_send_proxy.h deleted file mode 100644 index a606ea1db0..0000000000 --- a/audio/channel_send_proxy.h +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef AUDIO_CHANNEL_SEND_PROXY_H_ -#define AUDIO_CHANNEL_SEND_PROXY_H_ - -#include -#include -#include - -#include "api/audio_codecs/audio_encoder.h" -#include "audio/channel_send.h" -#include "rtc_base/constructormagic.h" -#include "rtc_base/race_checker.h" -#include "rtc_base/thread_checker.h" - -namespace webrtc { - -class FrameEncryptorInterface; -class RtcpBandwidthObserver; -class RtpRtcp; -class RtpTransportControllerSendInterface; -class Transport; - -namespace voe { - -// This class provides the "view" of a voe::Channel that we need to implement -// webrtc::AudioSendStream. It serves two purposes: -// 1. Allow mocking just the interfaces used, instead of the entire -// voe::Channel class. -// 2. Provide a refined interface for the stream classes, including assumptions -// on return values and input adaptation. -class ChannelSendProxy { - public: - ChannelSendProxy(); - explicit ChannelSendProxy(std::unique_ptr channel); - virtual ~ChannelSendProxy(); - - // Shared with ChannelReceiveProxy - virtual void SetLocalSSRC(uint32_t ssrc); - virtual void SetNACKStatus(bool enable, int max_packets); - virtual CallSendStatistics GetRTCPStatistics() const; - virtual void RegisterTransport(Transport* transport); - virtual bool ReceivedRTCPPacket(const uint8_t* packet, size_t length); - - virtual bool SetEncoder(int payload_type, - std::unique_ptr encoder); - virtual void ModifyEncoder( - rtc::FunctionView*)> modifier); - - virtual void SetMid(const std::string& mid, int extension_id); - virtual void SetRTCP_CNAME(absl::string_view c_name); - virtual void SetExtmapAllowMixed(bool extmap_allow_mixed); - virtual void SetSendAudioLevelIndicationStatus(bool enable, int id); - virtual void EnableSendTransportSequenceNumber(int id); - virtual void RegisterSenderCongestionControlObjects( - RtpTransportControllerSendInterface* transport, - RtcpBandwidthObserver* bandwidth_observer); - virtual void ResetSenderCongestionControlObjects(); - virtual std::vector GetRemoteRTCPReportBlocks() const; - virtual ANAStats GetANAStatistics() const; - virtual bool SetSendTelephoneEventPayloadType(int payload_type, - int payload_frequency); - virtual bool SendTelephoneEventOutband(int event, int duration_ms); - virtual void SetBitrate(int bitrate_bps, int64_t probing_interval_ms); - virtual int GetBitrate() const; - virtual void SetInputMute(bool muted); - - virtual void ProcessAndEncodeAudio(std::unique_ptr audio_frame); - virtual void SetTransportOverhead(int transport_overhead_per_packet); - virtual RtpRtcp* GetRtpRtcp() const; - - virtual void OnTwccBasedUplinkPacketLossRate(float packet_loss_rate); - virtual void OnRecoverableUplinkPacketLossRate( - float recoverable_packet_loss_rate); - virtual void StartSend(); - virtual void StopSend(); - - // Needed by ChannelReceiveProxy::AssociateSendChannel. - virtual ChannelSend* GetChannel() const; - - // E2EE Custom Audio Frame Encryption (Optional) - virtual void SetFrameEncryptor( - rtc::scoped_refptr frame_encryptor); - - private: - // Thread checkers document and lock usage of some methods on voe::Channel to - // specific threads we know about. The goal is to eventually split up - // voe::Channel into parts with single-threaded semantics, and thereby reduce - // the need for locks. - rtc::ThreadChecker worker_thread_checker_; - rtc::ThreadChecker module_process_thread_checker_; - // Methods accessed from audio and video threads are checked for sequential- - // only access. We don't necessarily own and control these threads, so thread - // checkers cannot be used. E.g. Chromium may transfer "ownership" from one - // audio thread to another, but access is still sequential. - rtc::RaceChecker audio_thread_race_checker_; - rtc::RaceChecker video_capture_thread_race_checker_; - std::unique_ptr channel_; - - RTC_DISALLOW_COPY_AND_ASSIGN(ChannelSendProxy); -}; -} // namespace voe -} // namespace webrtc - -#endif // AUDIO_CHANNEL_SEND_PROXY_H_ diff --git a/audio/mock_voe_channel_proxy.h b/audio/mock_voe_channel_proxy.h index 84e96af06c..ea7a0c0db9 100644 --- a/audio/mock_voe_channel_proxy.h +++ b/audio/mock_voe_channel_proxy.h @@ -18,7 +18,7 @@ #include "api/test/mock_frame_encryptor.h" #include "audio/channel_receive.h" -#include "audio/channel_send_proxy.h" +#include "audio/channel_send.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "test/gmock.h" @@ -48,7 +48,7 @@ class MockChannelReceive : public voe::ChannelReceiveInterface { AudioFrame* audio_frame)); MOCK_CONST_METHOD0(PreferredSampleRate, int()); MOCK_METHOD1(SetAssociatedSendChannel, - void(const voe::ChannelSend* send_channel)); + void(const voe::ChannelSendInterface* send_channel)); MOCK_CONST_METHOD0(GetPlayoutTimestamp, uint32_t()); MOCK_CONST_METHOD0(GetSyncInfo, absl::optional()); MOCK_METHOD1(SetMinimumPlayoutDelay, void(int delay_ms)); @@ -60,7 +60,7 @@ class MockChannelReceive : public voe::ChannelReceiveInterface { MOCK_METHOD0(StopPlayout, void()); }; -class MockChannelSendProxy : public voe::ChannelSendProxy { +class MockChannelSend : public voe::ChannelSendInterface { public: // GMock doesn't like move-only types, like std::unique_ptr. virtual bool SetEncoder(int payload_type, @@ -72,6 +72,7 @@ class MockChannelSendProxy : public voe::ChannelSendProxy { MOCK_METHOD1( ModifyEncoder, void(rtc::FunctionView*)> modifier)); + MOCK_METHOD2(SetMid, void(const std::string& mid, int extension_id)); MOCK_METHOD1(SetLocalSSRC, void(uint32_t ssrc)); MOCK_METHOD1(SetRTCP_CNAME, void(absl::string_view c_name)); MOCK_METHOD2(SetNACKStatus, void(bool enable, int max_packets)); @@ -98,12 +99,14 @@ class MockChannelSendProxy : public voe::ChannelSendProxy { } MOCK_METHOD1(ProcessAndEncodeAudioForMock, void(std::unique_ptr* audio_frame)); - MOCK_METHOD1(SetTransportOverhead, void(int transport_overhead_per_packet)); + MOCK_METHOD1(SetTransportOverhead, + void(size_t transport_overhead_per_packet)); MOCK_CONST_METHOD0(GetRtpRtcp, RtpRtcp*()); MOCK_CONST_METHOD0(GetBitrate, int()); MOCK_METHOD1(OnTwccBasedUplinkPacketLossRate, void(float packet_loss_rate)); MOCK_METHOD1(OnRecoverableUplinkPacketLossRate, void(float recoverable_packet_loss_rate)); + MOCK_CONST_METHOD0(GetRTT, int64_t()); MOCK_METHOD0(StartSend, void()); MOCK_METHOD0(StopSend, void()); MOCK_METHOD1(