From f17d9a39d57078a0f4cb7504e0828361b4b9cb13 Mon Sep 17 00:00:00 2001 From: Per Kjellander Date: Mon, 11 Oct 2021 14:27:09 +0200 Subject: [PATCH] Send VideoLayersAllocation with valid frame rate when frame rate change Sends a VideoLayersAllocation header extension if frame rate change more than 5fps since the last time it was sent with valid frame rate and resolution. Bug: webrtc:12000 Change-Id: I2572c966025cc2c22743bbe2187cec7cceb86d01 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/234752 Reviewed-by: Danil Chapovalov Commit-Queue: Per Kjellander Cr-Commit-Position: refs/heads/main@{#35180} --- modules/rtp_rtcp/source/rtp_sender_video.cc | 19 ++++- modules/rtp_rtcp/source/rtp_sender_video.h | 2 + .../source/rtp_sender_video_unittest.cc | 83 +++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 906f2bca76..9e909c984e 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -292,12 +292,26 @@ void RTPSenderVideo::SetVideoLayersAllocationAfterTransformation( void RTPSenderVideo::SetVideoLayersAllocationInternal( VideoLayersAllocation allocation) { RTC_DCHECK_RUNS_SERIALIZED(&send_checker_); - if (!allocation_ || allocation.active_spatial_layers.size() > + if (!allocation_ || allocation.active_spatial_layers.size() != allocation_->active_spatial_layers.size()) { send_allocation_ = SendVideoLayersAllocation::kSendWithResolution; } else if (send_allocation_ == SendVideoLayersAllocation::kDontSend) { send_allocation_ = SendVideoLayersAllocation::kSendWithoutResolution; } + if (send_allocation_ == SendVideoLayersAllocation::kSendWithoutResolution) { + // Check if frame rate changed more than 5fps since the last time the + // extension was sent with frame rate and resolution. + for (size_t i = 0; i < allocation.active_spatial_layers.size(); ++i) { + if (abs(static_cast( + allocation.active_spatial_layers[i].frame_rate_fps) - + static_cast( + last_full_sent_allocation_->active_spatial_layers[i] + .frame_rate_fps)) > 5) { + send_allocation_ = SendVideoLayersAllocation::kSendWithResolution; + break; + } + } + } allocation_ = std::move(allocation); } @@ -726,6 +740,9 @@ bool RTPSenderVideo::SendVideo( // This frame will likely be delivered, no need to populate playout // delay extensions until it changes again. playout_delay_pending_ = false; + if (send_allocation_ == SendVideoLayersAllocation::kSendWithResolution) { + last_full_sent_allocation_ = allocation_; + } send_allocation_ = SendVideoLayersAllocation::kDontSend; } diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h index 226c4062a1..f4f15f3ba4 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/modules/rtp_rtcp/source/rtp_sender_video.h @@ -210,6 +210,8 @@ class RTPSenderVideo { RTC_GUARDED_BY(send_checker_); // Flag indicating if we should send `allocation_`. SendVideoLayersAllocation send_allocation_ RTC_GUARDED_BY(send_checker_); + absl::optional last_full_sent_allocation_ + RTC_GUARDED_BY(send_checker_); // Current target playout delay. VideoPlayoutDelay current_playout_delay_ RTC_GUARDED_BY(send_checker_); diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc index 78e842da7e..d3782be43e 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc @@ -1008,6 +1008,89 @@ TEST_P(RtpSenderVideoTest, EXPECT_TRUE(sent_allocation.resolution_and_frame_rate_is_valid); } +TEST_P(RtpSenderVideoTest, + VideoLayersAllocationWithResolutionSentOnLargeFrameRateChange) { + const size_t kFrameSize = 100; + uint8_t kFrame[kFrameSize]; + rtp_module_->RegisterRtpHeaderExtension( + RtpVideoLayersAllocationExtension::Uri(), + kVideoLayersAllocationExtensionId); + + VideoLayersAllocation allocation; + allocation.resolution_and_frame_rate_is_valid = true; + VideoLayersAllocation::SpatialLayer layer; + layer.width = 360; + layer.height = 180; + layer.spatial_id = 0; + layer.frame_rate_fps = 10; + layer.target_bitrate_per_temporal_layer.push_back( + DataRate::KilobitsPerSec(50)); + allocation.active_spatial_layers.push_back(layer); + rtp_sender_video_->SetVideoLayersAllocation(allocation); + + RTPVideoHeader hdr; + hdr.frame_type = VideoFrameType::kVideoFrameKey; + rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr, + kDefaultExpectedRetransmissionTimeMs); + ASSERT_TRUE(transport_.last_sent_packet() + .HasExtension()); + + // Update frame rate only. + allocation.active_spatial_layers[0].frame_rate_fps = 20; + rtp_sender_video_->SetVideoLayersAllocation(allocation); + hdr.frame_type = VideoFrameType::kVideoFrameDelta; + rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr, + kDefaultExpectedRetransmissionTimeMs); + + VideoLayersAllocation sent_allocation; + EXPECT_TRUE( + transport_.last_sent_packet() + .GetExtension(&sent_allocation)); + ASSERT_TRUE(sent_allocation.resolution_and_frame_rate_is_valid); + EXPECT_EQ(sent_allocation.active_spatial_layers[0].frame_rate_fps, 20); +} + +TEST_P(RtpSenderVideoTest, + VideoLayersAllocationWithoutResolutionSentOnSmallFrameRateChange) { + const size_t kFrameSize = 100; + uint8_t kFrame[kFrameSize]; + rtp_module_->RegisterRtpHeaderExtension( + RtpVideoLayersAllocationExtension::Uri(), + kVideoLayersAllocationExtensionId); + + VideoLayersAllocation allocation; + allocation.resolution_and_frame_rate_is_valid = true; + VideoLayersAllocation::SpatialLayer layer; + layer.width = 360; + layer.height = 180; + layer.spatial_id = 0; + layer.frame_rate_fps = 10; + layer.target_bitrate_per_temporal_layer.push_back( + DataRate::KilobitsPerSec(50)); + allocation.active_spatial_layers.push_back(layer); + rtp_sender_video_->SetVideoLayersAllocation(allocation); + + RTPVideoHeader hdr; + hdr.frame_type = VideoFrameType::kVideoFrameKey; + rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr, + kDefaultExpectedRetransmissionTimeMs); + ASSERT_TRUE(transport_.last_sent_packet() + .HasExtension()); + + // Update frame rate slightly. + allocation.active_spatial_layers[0].frame_rate_fps = 9; + rtp_sender_video_->SetVideoLayersAllocation(allocation); + hdr.frame_type = VideoFrameType::kVideoFrameDelta; + rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr, + kDefaultExpectedRetransmissionTimeMs); + + VideoLayersAllocation sent_allocation; + EXPECT_TRUE( + transport_.last_sent_packet() + .GetExtension(&sent_allocation)); + EXPECT_FALSE(sent_allocation.resolution_and_frame_rate_is_valid); +} + TEST_P(RtpSenderVideoTest, VideoLayersAllocationSentOnDeltaFramesOnlyOnUpdate) { const size_t kFrameSize = 100; uint8_t kFrame[kFrameSize];