diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index 4207f7bb98..3589d6a890 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -17,6 +17,7 @@ #include #include +#include "absl/memory/memory.h" #include "modules/rtp_rtcp/source/rtcp_packet/dlrr.h" #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" #include "rtc_base/checks.h" @@ -83,7 +84,6 @@ ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) : kDefaultVideoReportInterval), this), clock_(configuration.clock), - audio_(configuration.audio), keepalive_config_(configuration.keepalive_config), last_bitrate_process_time_(clock_->TimeInMilliseconds()), last_rtt_process_time_(clock_->TimeInMilliseconds()), @@ -101,7 +101,9 @@ ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) rtp_sender_.reset(new RTPSender( configuration.audio, configuration.clock, configuration.outgoing_transport, configuration.paced_sender, - configuration.flexfec_sender, + configuration.flexfec_sender + ? absl::make_optional(configuration.flexfec_sender->ssrc()) + : absl::nullopt, configuration.transport_sequence_number_allocator, configuration.transport_feedback_callback, configuration.send_bitrate_observer, @@ -112,6 +114,14 @@ ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) configuration.populate_network2_timestamp, configuration.frame_encryptor, configuration.require_frame_encryption, configuration.extmap_allow_mixed)); + if (configuration.audio) { + audio_ = absl::make_unique(clock_, rtp_sender_.get()); + } else { + video_ = absl::make_unique( + clock_, rtp_sender_.get(), configuration.flexfec_sender, + configuration.frame_encryptor, + configuration.require_frame_encryption); + } // Make sure rtcp sender use same timestamp offset as rtp sender. rtcp_sender_.SetTimestampOffset(rtp_sender_->TimestampOffset()); @@ -268,22 +278,21 @@ void ModuleRtpRtcpImpl::RegisterAudioSendPayload(int payload_type, int frequency, int channels, int rate) { + RTC_DCHECK(audio_); rtcp_sender_.SetRtpClockRate(payload_type, frequency); - RTC_CHECK_EQ(0, - rtp_sender_->RegisterPayload(payload_name, payload_type, - frequency, channels, rate)); + RTC_CHECK_EQ(0, audio_->RegisterAudioPayload(payload_name, payload_type, + frequency, channels, rate)); } void ModuleRtpRtcpImpl::RegisterVideoSendPayload(int payload_type, const char* payload_name) { + RTC_DCHECK(video_); rtcp_sender_.SetRtpClockRate(payload_type, kVideoPayloadTypeFrequency); - RTC_CHECK_EQ(0, - rtp_sender_->RegisterPayload(payload_name, payload_type, - kVideoPayloadTypeFrequency, 0, 0)); + video_->RegisterPayloadType(payload_type, payload_name); } int32_t ModuleRtpRtcpImpl::DeRegisterSendPayload(const int8_t payload_type) { - return rtp_sender_->DeRegisterSendPayload(payload_type); + return 0; } uint32_t ModuleRtpRtcpImpl::StartTimestamp() const { @@ -446,10 +455,22 @@ bool ModuleRtpRtcpImpl::SendOutgoingData( expected_retransmission_time_ms = kDefaultExpectedRetransmissionTimeMs; } } - return rtp_sender_->SendOutgoingData( - frame_type, payload_type, time_stamp, capture_time_ms, payload_data, - payload_size, fragmentation, rtp_video_header, transport_frame_id_out, - expected_retransmission_time_ms); + + const uint32_t rtp_timestamp = time_stamp + rtp_sender_->TimestampOffset(); + if (transport_frame_id_out) + *transport_frame_id_out = rtp_timestamp; + + if (audio_) { + RTC_DCHECK(fragmentation == nullptr); + + return audio_->SendAudio(frame_type, payload_type, rtp_timestamp, + payload_data, payload_size); + } else { + return video_->SendVideo(frame_type, payload_type, rtp_timestamp, + capture_time_ms, payload_data, payload_size, + fragmentation, rtp_video_header, + expected_retransmission_time_ms); + } } bool ModuleRtpRtcpImpl::TimeToSendPacket(uint32_t ssrc, @@ -764,11 +785,11 @@ bool ModuleRtpRtcpImpl::SendFeedbackPacket( int32_t ModuleRtpRtcpImpl::SendTelephoneEventOutband(const uint8_t key, const uint16_t time_ms, const uint8_t level) { - return rtp_sender_->SendTelephoneEvent(key, time_ms, level); + return audio_ ? audio_->SendTelephoneEvent(key, time_ms, level) : -1; } int32_t ModuleRtpRtcpImpl::SetAudioLevel(const uint8_t level_d_bov) { - return rtp_sender_->SetAudioLevel(level_d_bov); + return audio_ ? audio_->SetAudioLevel(level_d_bov) : -1; } int32_t ModuleRtpRtcpImpl::SetKeyFrameRequestMethod( @@ -789,13 +810,18 @@ int32_t ModuleRtpRtcpImpl::RequestKeyFrame() { void ModuleRtpRtcpImpl::SetUlpfecConfig(int red_payload_type, int ulpfec_payload_type) { - rtp_sender_->SetUlpfecConfig(red_payload_type, ulpfec_payload_type); + RTC_DCHECK(video_); + video_->SetUlpfecConfig(red_payload_type, ulpfec_payload_type); } bool ModuleRtpRtcpImpl::SetFecParameters( const FecProtectionParams& delta_params, const FecProtectionParams& key_params) { - return rtp_sender_->SetFecParameters(delta_params, key_params); + if (!video_) { + return false; + } + video_->SetFecParameters(delta_params, key_params); + return true; } void ModuleRtpRtcpImpl::SetRemoteSSRC(const uint32_t ssrc) { @@ -809,13 +835,13 @@ void ModuleRtpRtcpImpl::BitrateSent(uint32_t* total_rate, uint32_t* fec_rate, uint32_t* nack_rate) const { *total_rate = rtp_sender_->BitrateSent(); - *video_rate = rtp_sender_->VideoBitrateSent(); - *fec_rate = rtp_sender_->FecOverheadRate(); + *video_rate = video_ ? video_->VideoBitrateSent() : 0; + *fec_rate = video_ ? video_->FecOverheadRate() : 0; *nack_rate = rtp_sender_->NackOverheadRate(); } uint32_t ModuleRtpRtcpImpl::PacketizationOverheadBps() const { - return rtp_sender_->PacketizationOverheadBps(); + return video_ ? video_->PacketizationOverheadBps() : 0; } void ModuleRtpRtcpImpl::OnRequestSendReport() { @@ -843,8 +869,15 @@ void ModuleRtpRtcpImpl::OnReceivedNack( void ModuleRtpRtcpImpl::OnReceivedRtcpReportBlocks( const ReportBlockList& report_blocks) { - if (rtp_sender_) - rtp_sender_->OnReceivedRtcpReportBlocks(report_blocks); + if (video_) { + uint32_t ssrc = SSRC(); + + for (const RTCPReportBlock& report_block : report_blocks) { + if (ssrc == report_block.source_ssrc) { + video_->OnReceivedAck(report_block.extended_highest_sequence_number); + } + } + } } bool ModuleRtpRtcpImpl::LastReceivedNTP( diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl.h b/modules/rtp_rtcp/source/rtp_rtcp_impl.h index 61adb314b5..114e897c88 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_impl.h +++ b/modules/rtp_rtcp/source/rtp_rtcp_impl.h @@ -32,6 +32,8 @@ #include "modules/rtp_rtcp/source/rtcp_receiver.h" #include "modules/rtp_rtcp/source/rtcp_sender.h" #include "modules/rtp_rtcp/source/rtp_sender.h" +#include "modules/rtp_rtcp/source/rtp_sender_audio.h" +#include "modules/rtp_rtcp/source/rtp_sender_video.h" #include "rtc_base/critical_section.h" #include "rtc_base/gtest_prod_util.h" @@ -336,13 +338,13 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { bool TimeToSendFullNackList(int64_t now) const; std::unique_ptr rtp_sender_; + std::unique_ptr audio_; + std::unique_ptr video_; RTCPSender rtcp_sender_; RTCPReceiver rtcp_receiver_; Clock* const clock_; - const bool audio_; - const RtpKeepAliveConfig keepalive_config_; int64_t last_bitrate_process_time_; int64_t last_rtt_process_time_; diff --git a/modules/rtp_rtcp/source/rtp_sender.cc b/modules/rtp_rtcp/source/rtp_sender.cc index 7f42a62e96..ad7e63157b 100644 --- a/modules/rtp_rtcp/source/rtp_sender.cc +++ b/modules/rtp_rtcp/source/rtp_sender.cc @@ -20,14 +20,11 @@ #include "api/array_view.h" #include "logging/rtc_event_log/events/rtc_event_rtp_packet_outgoing.h" #include "logging/rtc_event_log/rtc_event_log.h" -#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h" #include "modules/rtp_rtcp/include/rtp_cvo.h" #include "modules/rtp_rtcp/source/byte_io.h" #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" -#include "modules/rtp_rtcp/source/rtp_sender_audio.h" -#include "modules/rtp_rtcp/source/rtp_sender_video.h" #include "modules/rtp_rtcp/source/time_util.h" #include "rtc_base/arraysize.h" #include "rtc_base/checks.h" @@ -35,7 +32,6 @@ #include "rtc_base/numerics/safe_minmax.h" #include "rtc_base/rate_limiter.h" #include "rtc_base/time_utils.h" -#include "rtc_base/trace_event.h" #include "system_wrappers/include/field_trial.h" namespace webrtc { @@ -87,21 +83,6 @@ constexpr RtpExtensionSize kVideoExtensionSizes[] = { RtpGenericFrameDescriptorExtension::kMaxSizeBytes}, }; -const char* FrameTypeToString(FrameType frame_type) { - switch (frame_type) { - case kEmptyFrame: - return "empty"; - case kAudioFrameSpeech: - return "audio_speech"; - case kAudioFrameCN: - return "audio_cn"; - case kVideoFrameKey: - return "video_key"; - case kVideoFrameDelta: - return "video_delta"; - } - return ""; -} } // namespace RTPSender::RTPSender( @@ -109,7 +90,7 @@ RTPSender::RTPSender( Clock* clock, Transport* transport, RtpPacketSender* paced_sender, - FlexfecSender* flexfec_sender, + absl::optional flexfec_ssrc, TransportSequenceNumberAllocator* sequence_number_allocator, TransportFeedbackObserver* transport_feedback_observer, BitrateStatisticsObserver* bitrate_callback, @@ -127,13 +108,7 @@ RTPSender::RTPSender( clock_delta_ms_(clock_->TimeInMilliseconds() - rtc::TimeMillis()), random_(clock_->TimeInMicroseconds()), audio_configured_(audio), - audio_(audio ? new RTPSenderAudio(clock, this) : nullptr), - video_(audio ? nullptr - : new RTPSenderVideo(clock, - this, - flexfec_sender, - frame_encryptor, - require_frame_encryption)), + flexfec_ssrc_(flexfec_ssrc), paced_sender_(paced_sender), transport_sequence_number_allocator_(sequence_number_allocator), transport_feedback_observer_(transport_feedback_observer), @@ -180,7 +155,7 @@ RTPSender::RTPSender( // Store FlexFEC packets in the packet history data structure, so they can // be found when paced. - if (flexfec_sender) { + if (flexfec_ssrc_) { flexfec_packet_history_.SetStorePacketsStatus( RtpPacketHistory::StorageMode::kStore, kMinFlexfecPacketsToStoreForPacing); @@ -216,29 +191,11 @@ uint16_t RTPSender::ActualSendBitrateKbit() const { 1000); } -uint32_t RTPSender::VideoBitrateSent() const { - if (video_) { - return video_->VideoBitrateSent(); - } - return 0; -} - -uint32_t RTPSender::FecOverheadRate() const { - if (video_) { - return video_->FecOverheadRate(); - } - return 0; -} - uint32_t RTPSender::NackOverheadRate() const { rtc::CritScope cs(&statistics_crit_); return nack_bitrate_sent_.Rate(clock_->TimeInMilliseconds()).value_or(0); } -uint32_t RTPSender::PacketizationOverheadBps() const { - return video_ ? video_->PacketizationOverheadBps() : 0; -} - void RTPSender::SetExtmapAllowMixed(bool extmap_allow_mixed) { rtc::CritScope lock(&send_critsect_); rtp_header_extension_map_.SetExtmapAllowMixed(extmap_allow_mixed); @@ -265,29 +222,6 @@ int32_t RTPSender::DeregisterRtpHeaderExtension(RTPExtensionType type) { return rtp_header_extension_map_.Deregister(type); } -int32_t RTPSender::RegisterPayload(absl::string_view payload_name, - int8_t payload_number, - uint32_t frequency, - size_t channels, - uint32_t rate) { - rtc::CritScope lock(&send_critsect_); - - int32_t ret_val = 0; - if (audio_configured_) { - // TODO(mflodman): Change to CreateAudioPayload and make static. - ret_val = audio_->RegisterAudioPayload(payload_name, payload_number, - frequency, channels, rate); - } else { - video_->RegisterPayloadType(payload_number, payload_name); - } - - return ret_val; -} - -int32_t RTPSender::DeRegisterSendPayload(int8_t /* payload_type */) { - return 0; -} - void RTPSender::SetMaxRtpPacketSize(size_t max_packet_size) { RTC_DCHECK_GE(max_packet_size, 100); RTC_DCHECK_LE(max_packet_size, IP_PACKET_SIZE); @@ -333,67 +267,6 @@ void RTPSender::SetRtxPayloadType(int payload_type, rtx_payload_type_map_[associated_payload_type] = payload_type; } -bool RTPSender::SendOutgoingData(FrameType frame_type, - int8_t payload_type, - uint32_t capture_timestamp, - int64_t capture_time_ms, - const uint8_t* payload_data, - size_t payload_size, - const RTPFragmentationHeader* fragmentation, - const RTPVideoHeader* rtp_header, - uint32_t* transport_frame_id_out, - int64_t expected_retransmission_time_ms) { - uint16_t sequence_number; - uint32_t rtp_timestamp; - { - // Drop this packet if we're not sending media packets. - rtc::CritScope lock(&send_critsect_); - RTC_DCHECK(ssrc_); - - sequence_number = sequence_number_; - rtp_timestamp = timestamp_offset_ + capture_timestamp; - if (transport_frame_id_out) - *transport_frame_id_out = rtp_timestamp; - if (!sending_media_) - return true; - } - switch (frame_type) { - case kAudioFrameSpeech: - case kAudioFrameCN: - RTC_CHECK(audio_configured_); - break; - case kVideoFrameKey: - case kVideoFrameDelta: - RTC_CHECK(!audio_configured_); - break; - case kEmptyFrame: - break; - } - - bool result; - if (audio_configured_) { - TRACE_EVENT_ASYNC_STEP1("webrtc", "Audio", rtp_timestamp, "Send", "type", - FrameTypeToString(frame_type)); - // The only known way to produce of RTPFragmentationHeader for audio is - // to use the AudioCodingModule directly. - RTC_DCHECK(fragmentation == nullptr); - result = audio_->SendAudio(frame_type, payload_type, rtp_timestamp, - payload_data, payload_size); - } else { - TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms, "Send", "type", - FrameTypeToString(frame_type)); - if (frame_type == kEmptyFrame) - return true; - - result = video_->SendVideo(frame_type, payload_type, rtp_timestamp, - capture_time_ms, payload_data, payload_size, - fragmentation, rtp_header, - expected_retransmission_time_ms); - } - - return result; -} - size_t RTPSender::TrySendRedundantPayloads(size_t bytes_to_send, const PacedPacketInfo& pacing_info) { { @@ -643,26 +516,6 @@ void RTPSender::OnReceivedNack( } } -void RTPSender::OnReceivedRtcpReportBlocks( - const ReportBlockList& report_blocks) { - if (!video_) { - return; - } - uint32_t ssrc; - { - rtc::CritScope lock(&send_critsect_); - if (!ssrc_) - return; - ssrc = *ssrc_; - } - - for (const RTCPReportBlock& report_block : report_blocks) { - if (ssrc == report_block.source_ssrc) { - video_->OnReceivedAck(report_block.extended_highest_sequence_number); - } - } -} - // Called from pacer when we can send the packet. bool RTPSender::TimeToSendPacket(uint32_t ssrc, uint16_t sequence_number, @@ -809,29 +662,14 @@ bool RTPSender::SendToNetwork(std::unique_ptr packet, RTC_DCHECK(packet); int64_t now_ms = clock_->TimeInMilliseconds(); - if (video_) { - BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoTotBitrate_kbps", now_ms, - ActualSendBitrateKbit(), packet->Ssrc()); - BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoFecBitrate_kbps", now_ms, - FecOverheadRate() / 1000, packet->Ssrc()); - BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoNackBitrate_kbps", now_ms, - NackOverheadRate() / 1000, packet->Ssrc()); - } else { - BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "AudioTotBitrate_kbps", now_ms, - ActualSendBitrateKbit(), packet->Ssrc()); - BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "AudioNackBitrate_kbps", now_ms, - NackOverheadRate() / 1000, packet->Ssrc()); - } - uint32_t ssrc = packet->Ssrc(); - absl::optional flexfec_ssrc = FlexfecSsrc(); if (paced_sender_) { uint16_t seq_no = packet->SequenceNumber(); // Correct offset between implementations of millisecond time stamps in // TickTime and Clock. int64_t corrected_time_ms = packet->capture_time_ms() + clock_delta_ms_; size_t payload_length = packet->payload_size(); - if (ssrc == flexfec_ssrc) { + if (ssrc == FlexfecSsrc()) { // Store FlexFEC packets in the history here, so they can be found // when the pacer calls TimeToSendPacket. flexfec_packet_history_.PutRtpPacket(std::move(packet), storage, @@ -1164,10 +1002,7 @@ void RTPSender::SetMid(const std::string& mid) { } absl::optional RTPSender::FlexfecSsrc() const { - if (video_) { - return video_->FlexfecSsrc(); - } - return absl::nullopt; + return flexfec_ssrc_; } void RTPSender::SetCsrcs(const std::vector& csrcs) { @@ -1187,34 +1022,6 @@ uint16_t RTPSender::SequenceNumber() const { return sequence_number_; } -// Audio. -int32_t RTPSender::SendTelephoneEvent(uint8_t key, - uint16_t time_ms, - uint8_t level) { - if (!audio_configured_) { - return -1; - } - return audio_->SendTelephoneEvent(key, time_ms, level); -} - -int32_t RTPSender::SetAudioLevel(uint8_t level_d_bov) { - return audio_->SetAudioLevel(level_d_bov); -} - -void RTPSender::SetUlpfecConfig(int red_payload_type, int ulpfec_payload_type) { - RTC_DCHECK(!audio_configured_); - video_->SetUlpfecConfig(red_payload_type, ulpfec_payload_type); -} - -bool RTPSender::SetFecParameters(const FecProtectionParams& delta_params, - const FecProtectionParams& key_params) { - if (audio_configured_) { - return false; - } - video_->SetFecParameters(delta_params, key_params); - return true; -} - static std::unique_ptr CreateRtxPacket( const RtpPacketToSend& packet, RtpHeaderExtensionMap* extension_map) { diff --git a/modules/rtp_rtcp/source/rtp_sender.h b/modules/rtp_rtcp/source/rtp_sender.h index 401a8ff067..622f41a08d 100644 --- a/modules/rtp_rtcp/source/rtp_sender.h +++ b/modules/rtp_rtcp/source/rtp_sender.h @@ -41,8 +41,6 @@ class OverheadObserver; class RateLimiter; class RtcEventLog; class RtpPacketToSend; -class RTPSenderAudio; -class RTPSenderVideo; class RTPSender { public: @@ -50,9 +48,7 @@ class RTPSender { Clock* clock, Transport* transport, RtpPacketSender* paced_sender, - // TODO(brandtr): Remove |flexfec_sender| when that is hooked up - // to PacedSender instead. - FlexfecSender* flexfec_sender, + absl::optional flexfec_ssrc, TransportSequenceNumberAllocator* sequence_number_allocator, TransportFeedbackObserver* transport_feedback_callback, BitrateStatisticsObserver* bitrate_callback, @@ -72,18 +68,7 @@ class RTPSender { uint16_t ActualSendBitrateKbit() const; - uint32_t VideoBitrateSent() const; - uint32_t FecOverheadRate() const; uint32_t NackOverheadRate() const; - uint32_t PacketizationOverheadBps() const; - - int32_t RegisterPayload(absl::string_view payload_name, - const int8_t payload_type, - const uint32_t frequency, - const size_t channels, - const uint32_t rate); - - int32_t DeRegisterSendPayload(const int8_t payload_type); void SetSendingMediaStatus(bool enabled); bool SendingMedia() const; @@ -109,17 +94,6 @@ class RTPSender { void SetMaxRtpPacketSize(size_t max_packet_size); - bool SendOutgoingData(FrameType frame_type, - int8_t payload_type, - uint32_t timestamp, - int64_t capture_time_ms, - const uint8_t* payload_data, - size_t payload_size, - const RTPFragmentationHeader* fragmentation, - const RTPVideoHeader* rtp_header, - uint32_t* transport_frame_id_out, - int64_t expected_retransmission_time_ms); - void SetExtmapAllowMixed(bool extmap_allow_mixed); // RTP header extension @@ -145,10 +119,6 @@ class RTPSender { int32_t ReSendPacket(uint16_t packet_id); - // Feedback to decide when to stop sending the playout delay and MID header - // extensions. - void OnReceivedRtcpReportBlocks(const ReportBlockList& report_blocks); - // RTX. void SetRtxStatus(int mode); int RtxStatus() const; @@ -187,21 +157,6 @@ class RTPSender { StorageType storage, RtpPacketSender::Priority priority); - // Audio. - - // Send a DTMF tone using RFC 2833 (4733). - int32_t SendTelephoneEvent(uint8_t key, uint16_t time_ms, uint8_t level); - - // Store the audio level in d_bov for - // header-extension-for-audio-level-indication. - int32_t SetAudioLevel(uint8_t level_d_bov); - - // ULPFEC. - void SetUlpfecConfig(int red_payload_type, int ulpfec_payload_type); - - bool SetFecParameters(const FecProtectionParams& delta_params, - const FecProtectionParams& key_params); - // Called on update of RTP statistics. void RegisterRtpStatisticsCallback(StreamDataCountersCallback* callback); StreamDataCountersCallback* GetRtpStatisticsCallback() const; @@ -269,8 +224,8 @@ class RTPSender { Random random_ RTC_GUARDED_BY(send_critsect_); const bool audio_configured_; - const std::unique_ptr audio_; - const std::unique_ptr video_; + + const absl::optional flexfec_ssrc_; RtpPacketSender* const paced_sender_; TransportSequenceNumberAllocator* const transport_sequence_number_allocator_; diff --git a/modules/rtp_rtcp/source/rtp_sender_audio.cc b/modules/rtp_rtcp/source/rtp_sender_audio.cc index 9acc098555..56d0884e75 100644 --- a/modules/rtp_rtcp/source/rtp_sender_audio.cc +++ b/modules/rtp_rtcp/source/rtp_sender_audio.cc @@ -16,6 +16,7 @@ #include "absl/strings/match.h" #include "api/audio_codecs/audio_format.h" +#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/byte_io.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" @@ -27,6 +28,24 @@ namespace webrtc { +namespace { + +const char* FrameTypeToString(FrameType frame_type) { + switch (frame_type) { + case kEmptyFrame: + return "empty"; + case kAudioFrameSpeech: + return "audio_speech"; + case kAudioFrameCN: + return "audio_cn"; + default: + RTC_NOTREACHED(); + return ""; + } +} + +} // namespace + RTPSenderAudio::RTPSenderAudio(Clock* clock, RTPSender* rtp_sender) : clock_(clock), rtp_sender_(rtp_sender) {} @@ -115,6 +134,12 @@ bool RTPSenderAudio::SendAudio(FrameType frame_type, uint32_t rtp_timestamp, const uint8_t* payload_data, size_t payload_size) { + RTC_DCHECK(frame_type == kAudioFrameSpeech || frame_type == kAudioFrameCN || + frame_type == kEmptyFrame); + + TRACE_EVENT_ASYNC_STEP1("webrtc", "Audio", rtp_timestamp, "Send", "type", + FrameTypeToString(frame_type)); + // From RFC 4733: // A source has wide latitude as to how often it sends event updates. A // natural interval is the spacing between non-event audio packets. [...] @@ -233,7 +258,7 @@ bool RTPSenderAudio::SendAudio(FrameType frame_type, TRACE_EVENT_ASYNC_END2("webrtc", "Audio", rtp_timestamp, "timestamp", packet->Timestamp(), "seqnum", packet->SequenceNumber()); - bool send_result = rtp_sender_->SendToNetwork( + bool send_result = LogAndSendToNetwork( std::move(packet), kAllowRetransmission, RtpPacketSender::kHighPriority); if (first_packet_sent_()) { RTC_LOG(LS_INFO) << "First audio RTP packet sent to pacer"; @@ -317,11 +342,29 @@ bool RTPSenderAudio::SendTelephoneEventPacket(bool ended, dtmfbuffer[1] = E | R | volume; ByteWriter::WriteBigEndian(dtmfbuffer + 2, duration); - result = rtp_sender_->SendToNetwork(std::move(packet), kAllowRetransmission, - RtpPacketSender::kHighPriority); + result = LogAndSendToNetwork(std::move(packet), kAllowRetransmission, + RtpPacketSender::kHighPriority); send_count--; } while (send_count > 0 && result); return result; } + +bool RTPSenderAudio::LogAndSendToNetwork( + std::unique_ptr packet, + StorageType storage, + RtpPacketSender::Priority priority) { +#if BWE_TEST_LOGGING_COMPILE_TIME_ENABLE + int64_t now_ms = clock_->TimeInMilliseconds(); + BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "AudioTotBitrate_kbps", now_ms, + rtp_sender_->ActualSendBitrateKbit(), + packet->Ssrc()); + BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "AudioNackBitrate_kbps", now_ms, + rtp_sender_->NackOverheadRate() / 1000, + packet->Ssrc()); +#endif + + return rtp_sender_->SendToNetwork(std::move(packet), storage, priority); +} + } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_sender_audio.h b/modules/rtp_rtcp/source/rtp_sender_audio.h index f002023e9a..fa5894367d 100644 --- a/modules/rtp_rtcp/source/rtp_sender_audio.h +++ b/modules/rtp_rtcp/source/rtp_sender_audio.h @@ -14,6 +14,8 @@ #include #include +#include + #include "absl/strings/string_view.h" #include "common_types.h" // NOLINT(build/include) #include "modules/rtp_rtcp/source/dtmf_queue.h" @@ -61,6 +63,10 @@ class RTPSenderAudio { bool MarkerBit(FrameType frame_type, int8_t payload_type); private: + bool LogAndSendToNetwork(std::unique_ptr packet, + StorageType storage, + RtpPacketSender::Priority priority); + Clock* const clock_ = nullptr; RTPSender* const rtp_sender_ = nullptr; diff --git a/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc index b1c46c1cea..7f9d72f40a 100644 --- a/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc @@ -64,7 +64,7 @@ class RtpSenderAudioTest : public ::testing::Test { &fake_clock_, &transport_, nullptr, - nullptr, + absl::nullopt, nullptr, nullptr, nullptr, diff --git a/modules/rtp_rtcp/source/rtp_sender_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_unittest.cc index b13875acc0..714fac708a 100644 --- a/modules/rtp_rtcp/source/rtp_sender_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_unittest.cc @@ -184,7 +184,7 @@ class RtpSenderTest : public ::testing::TestWithParam { void SetUpRtpSender(bool pacer, bool populate_network2) { rtp_sender_.reset(new RTPSender( false, &fake_clock_, &transport_, pacer ? &mock_paced_sender_ : nullptr, - nullptr, &seq_num_allocator_, nullptr, nullptr, nullptr, + absl::nullopt, &seq_num_allocator_, nullptr, nullptr, nullptr, &mock_rtc_event_log_, &send_packet_observer_, &retransmission_rate_limiter_, nullptr, populate_network2, nullptr, false, false)); @@ -324,9 +324,10 @@ TEST_P(RtpSenderTest, AssignSequenceNumberAllowsPaddingOnAudio) { MockTransport transport; const bool kEnableAudio = true; rtp_sender_.reset(new RTPSender( - kEnableAudio, &fake_clock_, &transport, &mock_paced_sender_, nullptr, - nullptr, nullptr, nullptr, nullptr, &mock_rtc_event_log_, nullptr, - &retransmission_rate_limiter_, nullptr, false, nullptr, false, false)); + kEnableAudio, &fake_clock_, &transport, &mock_paced_sender_, + absl::nullopt, nullptr, nullptr, nullptr, nullptr, &mock_rtc_event_log_, + nullptr, &retransmission_rate_limiter_, nullptr, false, nullptr, false, + false)); rtp_sender_->SetTimestampOffset(0); rtp_sender_->SetSSRC(kSsrc); @@ -370,10 +371,10 @@ TEST_P(RtpSenderTestWithoutPacer, constexpr int kRtpOverheadBytesPerPacket = 12 + 8; testing::NiceMock mock_overhead_observer; rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, nullptr, nullptr, &seq_num_allocator_, - &feedback_observer_, nullptr, nullptr, &mock_rtc_event_log_, nullptr, - &retransmission_rate_limiter_, &mock_overhead_observer, false, nullptr, - false, false)); + false, &fake_clock_, &transport_, nullptr, absl::nullopt, + &seq_num_allocator_, &feedback_observer_, nullptr, nullptr, + &mock_rtc_event_log_, nullptr, &retransmission_rate_limiter_, + &mock_overhead_observer, false, nullptr, false, false)); rtp_sender_->SetSSRC(kSsrc); EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( kRtpExtensionTransportSequenceNumber, @@ -397,10 +398,10 @@ TEST_P(RtpSenderTestWithoutPacer, TEST_P(RtpSenderTestWithoutPacer, SendsPacketsWithTransportSequenceNumber) { rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, nullptr, nullptr, &seq_num_allocator_, - &feedback_observer_, nullptr, nullptr, &mock_rtc_event_log_, - &send_packet_observer_, &retransmission_rate_limiter_, nullptr, false, - nullptr, false, false)); + false, &fake_clock_, &transport_, nullptr, absl::nullopt, + &seq_num_allocator_, &feedback_observer_, nullptr, nullptr, + &mock_rtc_event_log_, &send_packet_observer_, + &retransmission_rate_limiter_, nullptr, false, nullptr, false, false)); rtp_sender_->SetSSRC(kSsrc); EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( kRtpExtensionTransportSequenceNumber, @@ -429,10 +430,10 @@ TEST_P(RtpSenderTestWithoutPacer, SendsPacketsWithTransportSequenceNumber) { TEST_P(RtpSenderTestWithoutPacer, PacketOptionsNoRetransmission) { rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, nullptr, nullptr, &seq_num_allocator_, - &feedback_observer_, nullptr, nullptr, &mock_rtc_event_log_, - &send_packet_observer_, &retransmission_rate_limiter_, nullptr, false, - nullptr, false, false)); + false, &fake_clock_, &transport_, nullptr, absl::nullopt, + &seq_num_allocator_, &feedback_observer_, nullptr, nullptr, + &mock_rtc_event_log_, &send_packet_observer_, + &retransmission_rate_limiter_, nullptr, false, nullptr, false, false)); rtp_sender_->SetSSRC(kSsrc); SendGenericPacket(); @@ -484,18 +485,20 @@ TEST_P(RtpSenderTestWithoutPacer, DoesnSetIncludedInAllocationByDefault) { TEST_P(RtpSenderTestWithoutPacer, OnSendSideDelayUpdated) { testing::StrictMock send_side_delay_observer_; rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, nullptr, nullptr, nullptr, nullptr, - nullptr, &send_side_delay_observer_, &mock_rtc_event_log_, nullptr, - nullptr, nullptr, false, nullptr, false, false)); + false, &fake_clock_, &transport_, nullptr, absl::nullopt, nullptr, + nullptr, nullptr, &send_side_delay_observer_, &mock_rtc_event_log_, + nullptr, nullptr, nullptr, false, nullptr, false, false)); rtp_sender_->SetSSRC(kSsrc); + RTPSenderVideo rtp_sender_video(&fake_clock_, rtp_sender_.get(), nullptr, + nullptr, false); const uint8_t kPayloadType = 127; - const uint32_t kCaptureTimeMsToRtpTimestamp = 90; // 90 kHz clock const char payload_name[] = "GENERIC"; + + rtp_sender_video.RegisterPayloadType(kPayloadType, payload_name); + + const uint32_t kCaptureTimeMsToRtpTimestamp = 90; // 90 kHz clock RTPVideoHeader video_header; - EXPECT_EQ(0, rtp_sender_->RegisterPayload(payload_name, kPayloadType, - 1000 * kCaptureTimeMsToRtpTimestamp, - 0, 1500)); // Send packet with 10 ms send-side delay. The average and max should be 10 // ms. @@ -503,10 +506,10 @@ TEST_P(RtpSenderTestWithoutPacer, OnSendSideDelayUpdated) { .Times(1); int64_t capture_time_ms = fake_clock_.TimeInMilliseconds(); fake_clock_.AdvanceTimeMilliseconds(10); - EXPECT_TRUE(rtp_sender_->SendOutgoingData( + EXPECT_TRUE(rtp_sender_video.SendVideo( kVideoFrameKey, kPayloadType, capture_time_ms * kCaptureTimeMsToRtpTimestamp, capture_time_ms, - kPayloadData, sizeof(kPayloadData), nullptr, &video_header, nullptr, + kPayloadData, sizeof(kPayloadData), nullptr, &video_header, kDefaultExpectedRetransmissionTimeMs)); // Send another packet with 20 ms delay. The average @@ -514,10 +517,10 @@ TEST_P(RtpSenderTestWithoutPacer, OnSendSideDelayUpdated) { EXPECT_CALL(send_side_delay_observer_, SendSideDelayUpdated(15, 20, kSsrc)) .Times(1); fake_clock_.AdvanceTimeMilliseconds(10); - EXPECT_TRUE(rtp_sender_->SendOutgoingData( + EXPECT_TRUE(rtp_sender_video.SendVideo( kVideoFrameKey, kPayloadType, capture_time_ms * kCaptureTimeMsToRtpTimestamp, capture_time_ms, - kPayloadData, sizeof(kPayloadData), nullptr, &video_header, nullptr, + kPayloadData, sizeof(kPayloadData), nullptr, &video_header, kDefaultExpectedRetransmissionTimeMs)); // Send another packet at the same time, which replaces the last packet. @@ -526,10 +529,10 @@ TEST_P(RtpSenderTestWithoutPacer, OnSendSideDelayUpdated) { EXPECT_CALL(send_side_delay_observer_, SendSideDelayUpdated(5, 10, kSsrc)) .Times(1); capture_time_ms = fake_clock_.TimeInMilliseconds(); - EXPECT_TRUE(rtp_sender_->SendOutgoingData( + EXPECT_TRUE(rtp_sender_video.SendVideo( kVideoFrameKey, kPayloadType, capture_time_ms * kCaptureTimeMsToRtpTimestamp, capture_time_ms, - kPayloadData, sizeof(kPayloadData), nullptr, &video_header, nullptr, + kPayloadData, sizeof(kPayloadData), nullptr, &video_header, kDefaultExpectedRetransmissionTimeMs)); // Send a packet 1 second later. The earlier packets should have timed @@ -539,10 +542,10 @@ TEST_P(RtpSenderTestWithoutPacer, OnSendSideDelayUpdated) { fake_clock_.AdvanceTimeMilliseconds(1); EXPECT_CALL(send_side_delay_observer_, SendSideDelayUpdated(1, 1, kSsrc)) .Times(1); - EXPECT_TRUE(rtp_sender_->SendOutgoingData( + EXPECT_TRUE(rtp_sender_video.SendVideo( kVideoFrameKey, kPayloadType, capture_time_ms * kCaptureTimeMsToRtpTimestamp, capture_time_ms, - kPayloadData, sizeof(kPayloadData), nullptr, &video_header, nullptr, + kPayloadData, sizeof(kPayloadData), nullptr, &video_header, kDefaultExpectedRetransmissionTimeMs)); } @@ -561,7 +564,7 @@ TEST_P(RtpSenderTestWithoutPacer, OnSendPacketUpdated) { TEST_P(RtpSenderTest, SendsPacketsWithTransportSequenceNumber) { rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, &mock_paced_sender_, nullptr, + false, &fake_clock_, &transport_, &mock_paced_sender_, absl::nullopt, &seq_num_allocator_, &feedback_observer_, nullptr, nullptr, &mock_rtc_event_log_, &send_packet_observer_, &retransmission_rate_limiter_, nullptr, false, nullptr, false, false)); @@ -946,7 +949,7 @@ TEST_P(RtpSenderTest, OnSendPacketNotUpdatedForRetransmits) { TEST_P(RtpSenderTest, OnSendPacketNotUpdatedWithoutSeqNumAllocator) { rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, &mock_paced_sender_, nullptr, + false, &fake_clock_, &transport_, &mock_paced_sender_, absl::nullopt, nullptr /* TransportSequenceNumberAllocator */, nullptr, nullptr, nullptr, nullptr, &send_packet_observer_, &retransmission_rate_limiter_, nullptr, false, nullptr, false, false)); @@ -973,8 +976,8 @@ TEST_P(RtpSenderTest, OnSendPacketNotUpdatedWithoutSeqNumAllocator) { TEST_P(RtpSenderTest, SendRedundantPayloads) { MockTransport transport; rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport, &mock_paced_sender_, nullptr, nullptr, - nullptr, nullptr, nullptr, &mock_rtc_event_log_, nullptr, + false, &fake_clock_, &transport, &mock_paced_sender_, absl::nullopt, + nullptr, nullptr, nullptr, nullptr, &mock_rtc_event_log_, nullptr, &retransmission_rate_limiter_, nullptr, false, nullptr, false, false)); rtp_sender_->SetSequenceNumber(kSeqNum); rtp_sender_->SetSSRC(kSsrc); @@ -1050,15 +1053,16 @@ TEST_P(RtpSenderTest, SendRedundantPayloads) { TEST_P(RtpSenderTestWithoutPacer, SendGenericVideo) { const char payload_name[] = "GENERIC"; const uint8_t payload_type = 127; - ASSERT_EQ(0, rtp_sender_->RegisterPayload(payload_name, payload_type, 90000, - 0, 1500)); + RTPSenderVideo rtp_sender_video(&fake_clock_, rtp_sender_.get(), nullptr, + nullptr, false); + rtp_sender_video.RegisterPayloadType(payload_type, payload_name); uint8_t payload[] = {47, 11, 32, 93, 89}; // Send keyframe RTPVideoHeader video_header; - ASSERT_TRUE(rtp_sender_->SendOutgoingData( + ASSERT_TRUE(rtp_sender_video.SendVideo( kVideoFrameKey, payload_type, 1234, 4321, payload, sizeof(payload), - nullptr, &video_header, nullptr, kDefaultExpectedRetransmissionTimeMs)); + nullptr, &video_header, kDefaultExpectedRetransmissionTimeMs)); auto sent_payload = transport_.last_sent_packet().payload(); uint8_t generic_header = sent_payload[0]; @@ -1071,9 +1075,9 @@ TEST_P(RtpSenderTestWithoutPacer, SendGenericVideo) { payload[1] = 42; payload[4] = 13; - ASSERT_TRUE(rtp_sender_->SendOutgoingData( + ASSERT_TRUE(rtp_sender_video.SendVideo( kVideoFrameDelta, payload_type, 1234, 4321, payload, sizeof(payload), - nullptr, &video_header, nullptr, kDefaultExpectedRetransmissionTimeMs)); + nullptr, &video_header, kDefaultExpectedRetransmissionTimeMs)); sent_payload = transport_.last_sent_packet().payload(); generic_header = sent_payload[0]; @@ -1096,7 +1100,7 @@ TEST_P(RtpSenderTest, SendFlexfecPackets) { // Reset |rtp_sender_| to use FlexFEC. rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, &mock_paced_sender_, &flexfec_sender, + false, &fake_clock_, &transport_, &mock_paced_sender_, kFlexfecSsrc, &seq_num_allocator_, nullptr, nullptr, nullptr, &mock_rtc_event_log_, &send_packet_observer_, &retransmission_rate_limiter_, nullptr, false, nullptr, false, false)); @@ -1167,10 +1171,10 @@ TEST_P(RtpSenderTest, NoFlexfecForTimingFrames) { // Reset |rtp_sender_| to use FlexFEC. rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, &mock_paced_sender_, &flexfec_sender, - &seq_num_allocator_, nullptr, nullptr, nullptr, &mock_rtc_event_log_, - &send_packet_observer_, &retransmission_rate_limiter_, nullptr, false, - nullptr, false, false)); + false, &fake_clock_, &transport_, &mock_paced_sender_, + flexfec_sender.ssrc(), &seq_num_allocator_, nullptr, nullptr, nullptr, + &mock_rtc_event_log_, &send_packet_observer_, + &retransmission_rate_limiter_, nullptr, false, nullptr, false, false)); rtp_sender_->SetSSRC(kMediaSsrc); rtp_sender_->SetSequenceNumber(kSeqNum); rtp_sender_->SetStorePacketsStatus(true, 10); @@ -1264,7 +1268,7 @@ TEST_P(RtpSenderTestWithoutPacer, SendFlexfecPackets) { // Reset |rtp_sender_| to use FlexFEC. rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, nullptr, &flexfec_sender, + false, &fake_clock_, &transport_, nullptr, flexfec_sender.ssrc(), &seq_num_allocator_, nullptr, nullptr, nullptr, &mock_rtc_event_log_, &send_packet_observer_, &retransmission_rate_limiter_, nullptr, false, nullptr, false, false)); @@ -1391,10 +1395,10 @@ TEST_P(RtpSenderTest, FecOverheadRate) { // Reset |rtp_sender_| to use FlexFEC. rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, &mock_paced_sender_, &flexfec_sender, - &seq_num_allocator_, nullptr, nullptr, nullptr, &mock_rtc_event_log_, - &send_packet_observer_, &retransmission_rate_limiter_, nullptr, false, - nullptr, false, false)); + false, &fake_clock_, &transport_, &mock_paced_sender_, + flexfec_sender.ssrc(), &seq_num_allocator_, nullptr, nullptr, nullptr, + &mock_rtc_event_log_, &send_packet_observer_, + &retransmission_rate_limiter_, nullptr, false, nullptr, false, false)); rtp_sender_->SetSSRC(kMediaSsrc); rtp_sender_->SetSequenceNumber(kSeqNum); @@ -1460,11 +1464,17 @@ TEST_P(RtpSenderTest, BitrateCallbacks) { uint32_t retransmit_bitrate_; } callback; rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, nullptr, nullptr, nullptr, nullptr, - &callback, nullptr, nullptr, nullptr, &retransmission_rate_limiter_, - nullptr, false, nullptr, false, false)); + false, &fake_clock_, &transport_, nullptr, absl::nullopt, nullptr, + nullptr, &callback, nullptr, nullptr, nullptr, + &retransmission_rate_limiter_, nullptr, false, nullptr, false, false)); rtp_sender_->SetSSRC(kSsrc); + RTPSenderVideo rtp_sender_video(&fake_clock_, rtp_sender_.get(), nullptr, + nullptr, false); + const char payload_name[] = "GENERIC"; + const uint8_t payload_type = 127; + rtp_sender_video.RegisterPayloadType(payload_type, payload_name); + // Simulate kNumPackets sent with kPacketInterval ms intervals, with the // number of packets selected so that we fill (but don't overflow) the one // second averaging window. @@ -1475,10 +1485,6 @@ TEST_P(RtpSenderTest, BitrateCallbacks) { // Overhead = 12 bytes RTP header + 1 byte generic header. const uint32_t kPacketOverhead = 13; - const char payload_name[] = "GENERIC"; - const uint8_t payload_type = 127; - ASSERT_EQ(0, rtp_sender_->RegisterPayload(payload_name, payload_type, 90000, - 0, 1500)); uint8_t payload[] = {47, 11, 32, 93, 89}; rtp_sender_->SetStorePacketsStatus(true, 1); uint32_t ssrc = rtp_sender_->SSRC(); @@ -1489,9 +1495,9 @@ TEST_P(RtpSenderTest, BitrateCallbacks) { // Send a few frames. RTPVideoHeader video_header; for (uint32_t i = 0; i < kNumPackets; ++i) { - ASSERT_TRUE(rtp_sender_->SendOutgoingData( + ASSERT_TRUE(rtp_sender_video.SendVideo( kVideoFrameKey, payload_type, 1234, 4321, payload, sizeof(payload), - nullptr, &video_header, nullptr, kDefaultExpectedRetransmissionTimeMs)); + nullptr, &video_header, kDefaultExpectedRetransmissionTimeMs)); fake_clock_.AdvanceTimeMilliseconds(kPacketInterval); } @@ -1548,8 +1554,9 @@ TEST_P(RtpSenderTestWithoutPacer, StreamDataCountersCallbacks) { const uint8_t kUlpfecPayloadType = 97; const char payload_name[] = "GENERIC"; const uint8_t payload_type = 127; - ASSERT_EQ(0, rtp_sender_->RegisterPayload(payload_name, payload_type, 90000, - 0, 1500)); + RTPSenderVideo rtp_sender_video(&fake_clock_, rtp_sender_.get(), nullptr, + nullptr, false); + rtp_sender_video.RegisterPayloadType(payload_type, payload_name); uint8_t payload[] = {47, 11, 32, 93, 89}; rtp_sender_->SetStorePacketsStatus(true, 1); uint32_t ssrc = rtp_sender_->SSRC(); @@ -1558,9 +1565,9 @@ TEST_P(RtpSenderTestWithoutPacer, StreamDataCountersCallbacks) { // Send a frame. RTPVideoHeader video_header; - ASSERT_TRUE(rtp_sender_->SendOutgoingData( + ASSERT_TRUE(rtp_sender_video.SendVideo( kVideoFrameKey, payload_type, 1234, 4321, payload, sizeof(payload), - nullptr, &video_header, nullptr, kDefaultExpectedRetransmissionTimeMs)); + nullptr, &video_header, kDefaultExpectedRetransmissionTimeMs)); StreamDataCounters expected; expected.transmitted.payload_bytes = 6; expected.transmitted.header_bytes = 12; @@ -1594,15 +1601,15 @@ TEST_P(RtpSenderTestWithoutPacer, StreamDataCountersCallbacks) { callback.Matches(ssrc, expected); // Send ULPFEC. - rtp_sender_->SetUlpfecConfig(kRedPayloadType, kUlpfecPayloadType); + rtp_sender_video.SetUlpfecConfig(kRedPayloadType, kUlpfecPayloadType); FecProtectionParams fec_params; fec_params.fec_mask_type = kFecMaskRandom; fec_params.fec_rate = 1; fec_params.max_fec_frames = 1; - rtp_sender_->SetFecParameters(fec_params, fec_params); - ASSERT_TRUE(rtp_sender_->SendOutgoingData( + rtp_sender_video.SetFecParameters(fec_params, fec_params); + ASSERT_TRUE(rtp_sender_video.SendVideo( kVideoFrameDelta, payload_type, 1234, 4321, payload, sizeof(payload), - nullptr, &video_header, nullptr, kDefaultExpectedRetransmissionTimeMs)); + nullptr, &video_header, kDefaultExpectedRetransmissionTimeMs)); expected.transmitted.payload_bytes = 40; expected.transmitted.header_bytes = 60; expected.transmitted.packets = 5; @@ -1613,22 +1620,14 @@ TEST_P(RtpSenderTestWithoutPacer, StreamDataCountersCallbacks) { } TEST_P(RtpSenderTestWithoutPacer, BytesReportedCorrectly) { - const char* kPayloadName = "GENERIC"; + // XXX const char* kPayloadName = "GENERIC"; const uint8_t kPayloadType = 127; rtp_sender_->SetSSRC(1234); rtp_sender_->SetRtxSsrc(4321); rtp_sender_->SetRtxPayloadType(kPayloadType - 1, kPayloadType); rtp_sender_->SetRtxStatus(kRtxRetransmitted | kRtxRedundantPayloads); - ASSERT_EQ(0, rtp_sender_->RegisterPayload(kPayloadName, kPayloadType, 90000, - 0, 1500)); - uint8_t payload[] = {47, 11, 32, 93, 89}; - - RTPVideoHeader video_header; - ASSERT_TRUE(rtp_sender_->SendOutgoingData( - kVideoFrameKey, kPayloadType, 1234, 4321, payload, sizeof(payload), - nullptr, &video_header, nullptr, kDefaultExpectedRetransmissionTimeMs)); - + SendGenericPacket(); // Will send 2 full-size padding packets. rtp_sender_->TimeToSendPadding(1, PacedPacketInfo()); rtp_sender_->TimeToSendPadding(1, PacedPacketInfo()); @@ -1637,9 +1636,9 @@ TEST_P(RtpSenderTestWithoutPacer, BytesReportedCorrectly) { StreamDataCounters rtx_stats; rtp_sender_->GetDataCounters(&rtp_stats, &rtx_stats); - // Payload + 1-byte generic header. + // Payload EXPECT_GT(rtp_stats.first_packet_time_ms, -1); - EXPECT_EQ(rtp_stats.transmitted.payload_bytes, sizeof(payload) + 1); + EXPECT_EQ(rtp_stats.transmitted.payload_bytes, sizeof(kPayloadData)); EXPECT_EQ(rtp_stats.transmitted.header_bytes, 12u); EXPECT_EQ(rtp_stats.transmitted.padding_bytes, 0u); EXPECT_EQ(rtx_stats.transmitted.payload_bytes, 0u); @@ -1694,10 +1693,11 @@ TEST_P(RtpSenderTestWithoutPacer, RespectsNackBitrateLimit) { TEST_P(RtpSenderTest, OnOverheadChanged) { MockOverheadObserver mock_overhead_observer; - rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, nullptr, nullptr, nullptr, nullptr, - nullptr, nullptr, nullptr, nullptr, &retransmission_rate_limiter_, - &mock_overhead_observer, false, nullptr, false, false)); + rtp_sender_.reset( + new RTPSender(false, &fake_clock_, &transport_, nullptr, absl::nullopt, + nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, + &retransmission_rate_limiter_, &mock_overhead_observer, + false, nullptr, false, false)); rtp_sender_->SetSSRC(kSsrc); // RTP overhead is 12B. @@ -1715,10 +1715,11 @@ TEST_P(RtpSenderTest, OnOverheadChanged) { TEST_P(RtpSenderTest, DoesNotUpdateOverheadOnEqualSize) { MockOverheadObserver mock_overhead_observer; - rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport_, nullptr, nullptr, nullptr, nullptr, - nullptr, nullptr, nullptr, nullptr, &retransmission_rate_limiter_, - &mock_overhead_observer, false, nullptr, false, false)); + rtp_sender_.reset( + new RTPSender(false, &fake_clock_, &transport_, nullptr, absl::nullopt, + nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, + &retransmission_rate_limiter_, &mock_overhead_observer, + false, nullptr, false, false)); rtp_sender_->SetSSRC(kSsrc); EXPECT_CALL(mock_overhead_observer, OnOverheadChanged(_)).Times(1); @@ -1729,7 +1730,7 @@ TEST_P(RtpSenderTest, DoesNotUpdateOverheadOnEqualSize) { TEST_P(RtpSenderTest, SendsKeepAlive) { MockTransport transport; rtp_sender_.reset(new RTPSender( - false, &fake_clock_, &transport, nullptr, nullptr, nullptr, nullptr, + false, &fake_clock_, &transport, nullptr, absl::nullopt, nullptr, nullptr, nullptr, nullptr, &mock_rtc_event_log_, nullptr, &retransmission_rate_limiter_, nullptr, false, nullptr, false, false)); rtp_sender_->SetSequenceNumber(kSeqNum); diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 949af14b7d..c63f0d7ea9 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -21,6 +21,7 @@ #include "absl/memory/memory.h" #include "absl/strings/match.h" #include "api/crypto/frame_encryptor_interface.h" +#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/byte_io.h" #include "modules/rtp_rtcp/source/rtp_format_video_generic.h" @@ -152,6 +153,20 @@ bool IsBaseLayer(const RTPVideoHeader& video_header) { return true; } +const char* FrameTypeToString(FrameType frame_type) { + switch (frame_type) { + case kEmptyFrame: + return "empty"; + case kVideoFrameKey: + return "video_key"; + case kVideoFrameDelta: + return "video_delta"; + default: + RTC_NOTREACHED(); + return ""; + } +} + } // namespace RTPSenderVideo::RTPSenderVideo(Clock* clock, @@ -207,8 +222,8 @@ void RTPSenderVideo::SendVideoPacket(std::unique_ptr packet, // Remember some values about the packet before sending it away. size_t packet_size = packet->size(); uint16_t seq_num = packet->SequenceNumber(); - if (!rtp_sender_->SendToNetwork(std::move(packet), storage, - RtpPacketSender::kLowPriority)) { + if (!LogAndSendToNetwork(std::move(packet), storage, + RtpPacketSender::kLowPriority)) { RTC_LOG(LS_WARNING) << "Failed to send video packet " << seq_num; return; } @@ -249,8 +264,8 @@ void RTPSenderVideo::SendVideoPacketAsRedMaybeWithUlpfec( } // Send |red_packet| instead of |packet| for allocated sequence number. size_t red_packet_size = red_packet->size(); - if (rtp_sender_->SendToNetwork(std::move(red_packet), media_packet_storage, - RtpPacketSender::kLowPriority)) { + if (LogAndSendToNetwork(std::move(red_packet), media_packet_storage, + RtpPacketSender::kLowPriority)) { rtc::CritScope cs(&stats_crit_); video_bitrate_.Update(red_packet_size, clock_->TimeInMilliseconds()); } else { @@ -265,8 +280,8 @@ void RTPSenderVideo::SendVideoPacketAsRedMaybeWithUlpfec( rtp_packet->set_capture_time_ms(media_packet->capture_time_ms()); rtp_packet->set_is_fec(true); uint16_t fec_sequence_number = rtp_packet->SequenceNumber(); - if (rtp_sender_->SendToNetwork(std::move(rtp_packet), kDontRetransmit, - RtpPacketSender::kLowPriority)) { + if (LogAndSendToNetwork(std::move(rtp_packet), kDontRetransmit, + RtpPacketSender::kLowPriority)) { rtc::CritScope cs(&stats_crit_); fec_bitrate_.Update(fec_packet->length(), clock_->TimeInMilliseconds()); } else { @@ -293,8 +308,8 @@ void RTPSenderVideo::SendVideoPacketWithFlexfec( for (auto& fec_packet : fec_packets) { size_t packet_length = fec_packet->size(); uint16_t seq_num = fec_packet->SequenceNumber(); - if (rtp_sender_->SendToNetwork(std::move(fec_packet), kDontRetransmit, - RtpPacketSender::kLowPriority)) { + if (LogAndSendToNetwork(std::move(fec_packet), kDontRetransmit, + RtpPacketSender::kLowPriority)) { rtc::CritScope cs(&stats_crit_); fec_bitrate_.Update(packet_length, clock_->TimeInMilliseconds()); } else { @@ -304,6 +319,24 @@ void RTPSenderVideo::SendVideoPacketWithFlexfec( } } +bool RTPSenderVideo::LogAndSendToNetwork( + std::unique_ptr packet, + StorageType storage, + RtpPacketSender::Priority priority) { +#if BWE_TEST_LOGGING_COMPILE_TIME_ENABLE + int64_t now_ms = clock_->TimeInMilliseconds(); + BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoTotBitrate_kbps", now_ms, + rtp_sender_->ActualSendBitrateKbit(), + packet->Ssrc()); + BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoFecBitrate_kbps", now_ms, + FecOverheadRate() / 1000, packet->Ssrc()); + BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoNackBitrate_kbps", now_ms, + rtp_sender_->NackOverheadRate() / 1000, + packet->Ssrc()); +#endif + return rtp_sender_->SendToNetwork(std::move(packet), storage, priority); +} + void RTPSenderVideo::SetUlpfecConfig(int red_payload_type, int ulpfec_payload_type) { // Sanity check. Per the definition of UlpfecConfig (see config.h), @@ -371,6 +404,15 @@ bool RTPSenderVideo::SendVideo(FrameType frame_type, const RTPFragmentationHeader* fragmentation, const RTPVideoHeader* video_header, int64_t expected_retransmission_time_ms) { + RTC_DCHECK(frame_type == kVideoFrameKey || frame_type == kVideoFrameDelta || + frame_type == kEmptyFrame); + + TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms, "Send", "type", + FrameTypeToString(frame_type)); + + if (frame_type == kEmptyFrame) + return true; + if (payload_size == 0) return false; RTC_CHECK(video_header); diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h index 8e26206699..d29934f92d 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/modules/rtp_rtcp/source/rtp_sender_video.h @@ -121,6 +121,10 @@ class RTPSenderVideo { StorageType media_packet_storage, bool protect_media_packet); + bool LogAndSendToNetwork(std::unique_ptr packet, + StorageType storage, + RtpPacketSender::Priority priority); + bool red_enabled() const RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_) { return red_payload_type_ >= 0; } diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc index 9b155e786c..85bbc80ac9 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc @@ -112,7 +112,7 @@ class RtpSenderVideoTest : public ::testing::TestWithParam { &fake_clock_, &transport_, nullptr, - nullptr, + absl::nullopt, nullptr, nullptr, nullptr,